| 460 | } |
| 461 | |
| 462 | func (f FileOp) DownloadFileWithProcess(url, dst, key string, options DownloadOptions) error { |
| 463 | client, err := newDownloadHTTPClient(options) |
| 464 | if err != nil { |
| 465 | return err |
| 466 | } |
| 467 | defer client.CloseIdleConnections() |
| 468 | |
| 469 | request, err := http.NewRequest("GET", url, nil) |
| 470 | if err != nil { |
| 471 | return buserr.WithDetail("ErrWgetRemoteFailed", err.Error(), err) |
| 472 | } |
| 473 | request.Header.Set("Accept-Encoding", "identity") |
| 474 | |
| 475 | resp, err := client.Do(request) |
| 476 | if err != nil { |
| 477 | global.LOG.Errorf("get download file [%s] error, err %s", dst, err.Error()) |
| 478 | return buserr.WithDetail("ErrWgetRemoteFailed", err.Error(), err) |
| 479 | } |
| 480 | |
| 481 | if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { |
| 482 | _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64*1024)) |
| 483 | _ = resp.Body.Close() |
| 484 | global.LOG.Errorf("wget remote returned non-success status %s for url %s", resp.Status, url) |
| 485 | return buserr.WithDetail("ErrWgetRemoteFailed", resp.StatusCode, nil) |
| 486 | } |
| 487 | |
| 488 | ct := strings.ToLower(resp.Header.Get("Content-Type")) |
| 489 | dstExt := strings.ToLower(filepath.Ext(dst)) |
| 490 | if (strings.Contains(ct, "text/html") || strings.Contains(ct, "text/xml")) && |
| 491 | dstExt != ".html" && dstExt != ".htm" && dstExt != ".xml" && dstExt != ".svg" { |
| 492 | _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64*1024)) |
| 493 | _ = resp.Body.Close() |
| 494 | detail := fmt.Sprintf("Content-Type: %s", ct) |
| 495 | global.LOG.Errorf("wget got html/xml response for non-html file %s, url %s, %s", dst, url, detail) |
| 496 | return buserr.WithDetail("ErrWgetInvalidContentType", detail, nil) |
| 497 | } |
| 498 | |
| 499 | out, err := os.Create(dst) |
| 500 | if err != nil { |
| 501 | global.LOG.Errorf("create download file [%s] error, err %s", dst, err.Error()) |
| 502 | resp.Body.Close() |
| 503 | return err |
| 504 | } |
| 505 | |
| 506 | downloadMu.Lock() |
| 507 | downloadTasks[key] = &downloadTask{ |
| 508 | resp: resp, |
| 509 | file: out, |
| 510 | dst: dst, |
| 511 | } |
| 512 | downloadMu.Unlock() |
| 513 | |
| 514 | go func() { |
| 515 | defer func() { |
| 516 | out.Close() |
| 517 | resp.Body.Close() |
| 518 | |
| 519 | downloadMu.Lock() |