v / vlib / net / http / download_progress.v
87 lines · 82 sloc · 3.83 KB · 2c62c6d0fe270138d8905050d13502c3b5246d8c
Raw
1module http
2
3// Downloader is the interface that you have to implement, if you need to customise
4// how download_file_with_progress works, and what output it produces while a file
5// is downloaded.
6pub interface Downloader {
7mut:
8 // Called once, at the start of the streaming download. You can do setup here,
9 // like opening a target file, changing request.stop_copying_limit to a different value,
10 // if you need it.
11 on_start(mut request Request, path string) !
12 // Called many times, once a chunk of data is received
13 on_chunk(request &Request, chunk []u8, already_received u64, expected u64) !
14 // Called once, at the end of the download attempt. Do cleanup here,
15 // like closing a file (opened in on_start), reporting stats etc.
16 // `response` will be empty when the request fails before a response is parsed.
17 on_finish(request &Request, response &Response) !
18}
19
20// DownloaderParams is similar to FetchConfig, but it also allows you to pass
21// a `downloader: your_downloader_instance` parameter.
22// See also http.SilentStreamingDownloader, and http.TerminalStreamingDownloader .
23@[params]
24pub struct DownloaderParams {
25 FetchConfig
26pub mut:
27 downloader &Downloader = &TerminalStreamingDownloader{}
28}
29
30// download_file_with_progress will save the URL `url` to the filepath `path` .
31// Unlike download_file/2, it *does not* load the whole content in memory, but
32// instead streams it chunk by chunk to the target `path`, as the chunks are received
33// from the network. This makes it suitable for downloading big files, *without* increasing
34// the memory consumption of your application.
35//
36// By default, it will also show a progress line, while the download happens.
37// If you do not want a status line, you can call it like this:
38// `http.download_file_with_progress(url, path, downloader: http.SilentStreamingDownloader{})`,
39// or you can implement your own http.Downloader and pass that instead.
40//
41// Note: the returned response by this function, will have a truncated .body, after the first
42// few KBs, because it does not accumulate all its data in memory, instead relying on the
43// downloaders to save the received data chunk by chunk. You can parametrise this by
44// using `stop_copying_limit:` but you need to pass a number that is big enough to fit
45// at least all headers in the response, otherwise the parsing of the response at the end will
46// fail, despite saving all the data in the file before that. The default is 65536 bytes.
47pub fn download_file_with_progress(url string, path string, params DownloaderParams) !Response {
48 mut d := unsafe { params.downloader }
49 mut config := params.FetchConfig
50 config.url = url
51 config.user_ptr = voidptr(d)
52 config.on_progress_body = download_progres_cb
53 if config.stop_copying_limit == -1 {
54 // leave more than enough space for potential redirect headers
55 config.stop_copying_limit = 65536
56 }
57 mut req := prepare(config)!
58 d.on_start(mut req, path)!
59 mut response := Response{}
60 defer {
61 d.on_finish(req, response) or {}
62 }
63 response = req.do()!
64 $if windows && !no_vschannel ? {
65 // TODO: remove this, when windows supports streaming properly through vschannel
66 // For now though, just ensure that the complete body is "received" in one big chunk:
67 body_len := u64(response.body.len)
68 d.on_chunk(req, response.body.bytes(), body_len, body_len)!
69 }
70 return response
71}
72
73const zz = &Downloader(unsafe { nil })
74
75fn download_progres_cb(request &Request, chunk []u8, body_so_far u64, expected_size u64, status_code int) ! {
76 // TODO: remove this hack, when `unsafe { &Downloader( request.user_ptr ) }` works reliably,
77 // by just casting, without trying to promote the argument to the heap at all.
78 mut d := unsafe { zz }
79 pd := unsafe { &voidptr(&d) }
80 unsafe {
81 *pd = request.user_ptr
82 }
83 if status_code == 200 {
84 // ignore redirects, we are interested in the chunks of the final file:
85 d.on_chunk(request, chunk, body_so_far, expected_size)!
86 }
87}
88