| 1 | module http |
| 2 | |
| 3 | // Downloader is the interface that you have to implement, if you need to customise |
| 4 | // how download_file_with_progress works, and what output it produces while a file |
| 5 | // is downloaded. |
| 6 | pub interface Downloader { |
| 7 | mut: |
| 8 | // Called once, at the start of the streaming download. You can do setup here, |
| 9 | // like opening a target file, changing request.stop_copying_limit to a different value, |
| 10 | // if you need it. |
| 11 | on_start(mut request Request, path string) ! |
| 12 | // Called many times, once a chunk of data is received |
| 13 | on_chunk(request &Request, chunk []u8, already_received u64, expected u64) ! |
| 14 | // Called once, at the end of the download attempt. Do cleanup here, |
| 15 | // like closing a file (opened in on_start), reporting stats etc. |
| 16 | // `response` will be empty when the request fails before a response is parsed. |
| 17 | on_finish(request &Request, response &Response) ! |
| 18 | } |
| 19 | |
| 20 | // DownloaderParams is similar to FetchConfig, but it also allows you to pass |
| 21 | // a `downloader: your_downloader_instance` parameter. |
| 22 | // See also http.SilentStreamingDownloader, and http.TerminalStreamingDownloader . |
| 23 | @[params] |
| 24 | pub struct DownloaderParams { |
| 25 | FetchConfig |
| 26 | pub mut: |
| 27 | downloader &Downloader = &TerminalStreamingDownloader{} |
| 28 | } |
| 29 | |
| 30 | // download_file_with_progress will save the URL `url` to the filepath `path` . |
| 31 | // Unlike download_file/2, it *does not* load the whole content in memory, but |
| 32 | // instead streams it chunk by chunk to the target `path`, as the chunks are received |
| 33 | // from the network. This makes it suitable for downloading big files, *without* increasing |
| 34 | // the memory consumption of your application. |
| 35 | // |
| 36 | // By default, it will also show a progress line, while the download happens. |
| 37 | // If you do not want a status line, you can call it like this: |
| 38 | // `http.download_file_with_progress(url, path, downloader: http.SilentStreamingDownloader{})`, |
| 39 | // or you can implement your own http.Downloader and pass that instead. |
| 40 | // |
| 41 | // Note: the returned response by this function, will have a truncated .body, after the first |
| 42 | // few KBs, because it does not accumulate all its data in memory, instead relying on the |
| 43 | // downloaders to save the received data chunk by chunk. You can parametrise this by |
| 44 | // using `stop_copying_limit:` but you need to pass a number that is big enough to fit |
| 45 | // at least all headers in the response, otherwise the parsing of the response at the end will |
| 46 | // fail, despite saving all the data in the file before that. The default is 65536 bytes. |
| 47 | pub fn download_file_with_progress(url string, path string, params DownloaderParams) !Response { |
| 48 | mut d := unsafe { params.downloader } |
| 49 | mut config := params.FetchConfig |
| 50 | config.url = url |
| 51 | config.user_ptr = voidptr(d) |
| 52 | config.on_progress_body = download_progres_cb |
| 53 | if config.stop_copying_limit == -1 { |
| 54 | // leave more than enough space for potential redirect headers |
| 55 | config.stop_copying_limit = 65536 |
| 56 | } |
| 57 | mut req := prepare(config)! |
| 58 | d.on_start(mut req, path)! |
| 59 | mut response := Response{} |
| 60 | defer { |
| 61 | d.on_finish(req, response) or {} |
| 62 | } |
| 63 | response = req.do()! |
| 64 | $if windows && !no_vschannel ? { |
| 65 | // TODO: remove this, when windows supports streaming properly through vschannel |
| 66 | // For now though, just ensure that the complete body is "received" in one big chunk: |
| 67 | body_len := u64(response.body.len) |
| 68 | d.on_chunk(req, response.body.bytes(), body_len, body_len)! |
| 69 | } |
| 70 | return response |
| 71 | } |
| 72 | |
| 73 | const zz = &Downloader(unsafe { nil }) |
| 74 | |
| 75 | fn download_progres_cb(request &Request, chunk []u8, body_so_far u64, expected_size u64, status_code int) ! { |
| 76 | // TODO: remove this hack, when `unsafe { &Downloader( request.user_ptr ) }` works reliably, |
| 77 | // by just casting, without trying to promote the argument to the heap at all. |
| 78 | mut d := unsafe { zz } |
| 79 | pd := unsafe { &voidptr(&d) } |
| 80 | unsafe { |
| 81 | *pd = request.user_ptr |
| 82 | } |
| 83 | if status_code == 200 { |
| 84 | // ignore redirects, we are interested in the chunks of the final file: |
| 85 | d.on_chunk(request, chunk, body_so_far, expected_size)! |
| 86 | } |
| 87 | } |
| 88 | |