| 1 | import os |
| 2 | import log |
| 3 | import time |
| 4 | import flag |
| 5 | import net.http |
| 6 | import crypto.sha1 |
| 7 | import crypto.sha256 |
| 8 | import crypto.sha3 |
| 9 | |
| 10 | struct Context { |
| 11 | mut: |
| 12 | show_help bool |
| 13 | show_sha1 bool |
| 14 | show_sha256 bool |
| 15 | show_sha3_256 bool |
| 16 | target_folder string |
| 17 | output string |
| 18 | continue_on_failure bool |
| 19 | retries int |
| 20 | delay time.Duration |
| 21 | urls []string |
| 22 | should_run bool |
| 23 | delete_after_run bool |
| 24 | } |
| 25 | |
| 26 | const vexe = os.real_path(os.getenv_opt('VEXE') or { @VEXE }) |
| 27 | |
| 28 | fn main() { |
| 29 | log.use_stdout() |
| 30 | mut ctx := Context{} |
| 31 | mut fp := flag.new_flag_parser(os.args#[1..]) |
| 32 | fp.application(os.file_name(os.executable())) |
| 33 | fp.version('0.0.1') |
| 34 | fp.description('Download files from http/https servers, given their URLs.') |
| 35 | fp.arguments_description('URL1 URL2 ...') |
| 36 | fp.skip_executable() |
| 37 | fp.limit_free_args_to_at_least(1)! |
| 38 | ctx.show_help = fp.bool('help', `h`, false, 'Show this help screen.') |
| 39 | ctx.target_folder = fp.string('target-folder', `t`, '.', |
| 40 | 'The target folder, where the file will be stored. It will be created, if it does not exist. Default is current folder.') |
| 41 | ctx.output = fp.string('output', `o`, '', |
| 42 | 'Write output to the given file, instead of inferring it from the final part of the URL. All intermediate folders will be created, if they do not exist.') |
| 43 | ctx.show_sha1 = fp.bool('sha1', `1`, false, 'Show the SHA1 hash of the downloaded file.') |
| 44 | ctx.show_sha256 = fp.bool('sha256', `2`, false, 'Show the SHA256 hash of the downloaded file.') |
| 45 | ctx.show_sha3_256 = fp.bool('sha3-256', `3`, false, |
| 46 | 'Show the SHA3-256 (Keccak) hash of the downloaded file.') |
| 47 | ctx.continue_on_failure = fp.bool('continue', `c`, false, |
| 48 | 'Continue on download failures. If you download 5 URLs, and several of them fail, continue without error. False by default.') |
| 49 | ctx.retries = fp.int('retries', `r`, 10, |
| 50 | 'Number of retries, when an URL fails to download. The default is 10.') |
| 51 | ctx.delay = time.Duration(u64(fp.float('delay', `d`, 1.0, |
| 52 | 'Delay in seconds, after each retry. The default is 1 second.') * time.second)) |
| 53 | ctx.should_run = fp.bool('run', `R`, false, |
| 54 | 'Run, after the script/program is completely downloaded.') |
| 55 | ctx.delete_after_run = fp.bool('delete-after-run', `D`, false, |
| 56 | 'Delete the downloaded script/program, after it has been run.') |
| 57 | if ctx.show_help { |
| 58 | println(fp.usage()) |
| 59 | exit(0) |
| 60 | } |
| 61 | ctx.urls = fp.finalize() or { |
| 62 | eprintln('Error: ${err}') |
| 63 | exit(1) |
| 64 | } |
| 65 | if ctx.target_folder != '.' { |
| 66 | ctx.target_folder = ctx.target_folder.replace('\\', '/') |
| 67 | os.mkdir_all(ctx.target_folder) or { |
| 68 | eprintln('Can not create target folder `${ctx.target_folder}` . Error: ${err}.') |
| 69 | exit(1) |
| 70 | } |
| 71 | os.chdir(ctx.target_folder)! |
| 72 | } |
| 73 | if ctx.output != '' { |
| 74 | odir := os.dir(ctx.output) |
| 75 | os.mkdir_all(odir) or {} |
| 76 | } |
| 77 | sw := time.new_stopwatch() |
| 78 | mut errors := 0 |
| 79 | mut downloaded := 0 |
| 80 | downloader := if os.is_atty(1) > 0 { |
| 81 | &http.Downloader(http.TerminalStreamingDownloader{}) |
| 82 | } else { |
| 83 | &http.Downloader(http.SilentStreamingDownloader{}) |
| 84 | } |
| 85 | for idx, url in ctx.urls { |
| 86 | fname := if ctx.output != '' { ctx.output } else { url.all_after_last('/') } |
| 87 | fpath := if os.is_abs_path(fname) { fname } else { '${ctx.target_folder}/${fname}' } |
| 88 | mut file_errors := 0 |
| 89 | log.info('Downloading [${idx + 1}/${ctx.urls.len}] from url: ${url} to ${fpath} ...') |
| 90 | for retry in 0 .. ctx.retries { |
| 91 | http.download_file_with_progress(url, fname, downloader: downloader) or { |
| 92 | log.error(' retry ${retry + 1}/${ctx.retries}, failed downloading from url: ${url}. Error: ${err}.') |
| 93 | file_errors++ |
| 94 | time.sleep(ctx.delay) |
| 95 | continue |
| 96 | } |
| 97 | if fs := os.stat(fname) { |
| 98 | if fs.size == 0 { |
| 99 | log.error(' retry ${retry + 1}/${ctx.retries}, got an empty (0 byte) file from url: ${url}.') |
| 100 | file_errors++ |
| 101 | time.sleep(ctx.delay) |
| 102 | continue |
| 103 | } |
| 104 | } |
| 105 | downloaded++ |
| 106 | break |
| 107 | } |
| 108 | if file_errors == ctx.retries { |
| 109 | log.error('Failed to download from url: ${url}.') |
| 110 | errors++ |
| 111 | if ctx.continue_on_failure { |
| 112 | continue |
| 113 | } else { |
| 114 | break |
| 115 | } |
| 116 | } |
| 117 | fstat := os.stat(fname)! |
| 118 | log.info(' Finished downloading file: ${fpath} .') |
| 119 | log.info(' size: ${fstat.size} bytes') |
| 120 | |
| 121 | if ctx.should_run { |
| 122 | run_cmd := '${os.quoted_path(vexe)} run ${os.quoted_path(fpath)}' |
| 123 | log.info(' Executing: ${run_cmd}') |
| 124 | os.system(run_cmd) |
| 125 | } |
| 126 | if ctx.delete_after_run { |
| 127 | log.info(' Removing: ${fpath}') |
| 128 | os.rm(fpath) or {} |
| 129 | } |
| 130 | if !ctx.show_sha256 && !ctx.show_sha1 && !ctx.show_sha3_256 { |
| 131 | continue |
| 132 | } |
| 133 | fbytes := os.read_bytes(fname)! |
| 134 | if ctx.show_sha1 { |
| 135 | mut digest1 := sha1.new() |
| 136 | digest1.write(fbytes)! |
| 137 | mut sum1 := digest1.sum([]) |
| 138 | hash1 := sum1.hex() |
| 139 | log.info(' SHA1: ${hash1}') |
| 140 | } |
| 141 | if ctx.show_sha256 { |
| 142 | mut digest256 := sha256.new() |
| 143 | digest256.write(fbytes)! |
| 144 | mut sum256 := digest256.sum([]) |
| 145 | hash256 := sum256.hex() |
| 146 | log.info(' SHA256: ${hash256}') |
| 147 | } |
| 148 | if ctx.show_sha3_256 { |
| 149 | hash3_256 := sha3.sum256(fbytes).hex() |
| 150 | log.info(' SHA3-256: ${hash3_256}') |
| 151 | } |
| 152 | } |
| 153 | println('Downloaded: ${downloaded} file(s) . Elapsed time: ${sw.elapsed()} . Errors: ${errors} .') |
| 154 | if !ctx.continue_on_failure && errors > 0 { |
| 155 | exit(1) |
| 156 | } |
| 157 | } |
| 158 | |