v2 / cmd / tools / vpm / common.v
484 lines · 445 sloc · 14.3 KB · e2e5cf8db56f3562c7baa735061690be936bdf3e
Raw
1module main
2
3import os
4import net.http
5import net.urllib
6import v.vmod
7import json
8import term
9
10struct ModuleVpmInfo {
11 // id int
12 name string
13 url string
14 vcs string
15 nr_downloads int
16}
17
18struct VpmInstallServerSelector {
19mut:
20 selected_url string
21 candidate_urls []string
22}
23
24@[params]
25struct ErrorOptions {
26 details string
27 verbose bool // is used to only output the error message if the verbose setting is enabled.
28}
29
30const vexe = os.quoted_path(os.getenv('VEXE'))
31const home_dir = os.home_dir()
32const selected_server_url_env = 'VPM_SELECTED_SERVER_URL'
33
34fn merge_server_urls(default_urls []string, custom_urls []string) []string {
35 mut server_urls := default_urls.clone()
36 for url in custom_urls {
37 if url in server_urls {
38 continue
39 }
40 server_urls << url
41 }
42 return server_urls
43}
44
45fn get_server_urls() []string {
46 return merge_server_urls(vpm_server_urls, settings.server_urls)
47}
48
49fn selected_server_url(set bool, url string) string {
50 if set {
51 os.setenv(selected_server_url_env, url, true)
52 }
53 return os.getenv(selected_server_url_env)
54}
55
56fn active_server_urls() []string {
57 selected_url := selected_server_url(false, '')
58 if selected_url != '' {
59 return [selected_url]
60 }
61 return get_server_urls()
62}
63
64fn get_mod_vpm_info(name string) !ModuleVpmInfo {
65 mut selector := VpmInstallServerSelector{
66 candidate_urls: if settings.server_urls.len > 0 {
67 settings.server_urls
68 } else {
69 vpm_server_urls
70 }
71 }
72 return get_mod_vpm_info_with_selector(name, mut selector)
73}
74
75fn get_mod_vpm_info_with_selector(name string, mut selector VpmInstallServerSelector) !ModuleVpmInfo {
76 if name.len < 2 || (!name[0].is_digit() && !name[0].is_letter()) {
77 return error('invalid module name `${name}`.')
78 }
79 if selector.candidate_urls.len == 0 {
80 return error('no vpm server urls configured.')
81 }
82 mut errors := []string{}
83 is_initial_selection := selected_server_url(false, '') == ''
84 for url in selector.metadata_server_urls() {
85 modurl := url + '/api/packages/${name}'
86 verbose_println_more(@FILE_LINE, @FN,
87 'Retrieving metadata for `${name}` from `${modurl}` by making a GET request ...')
88 r := http.get(modurl) or {
89 errors << 'Http server did not respond to our request for `${modurl}`.'
90 errors << 'Error details: ${err}'
91 continue
92 }
93 if r.status_code == 404 || r.body.trim_space() == '404' {
94 errors << 'Skipping module `${name}`, since `${url}` reported that `${name}` does not exist.'
95 continue
96 }
97 if r.status_code != 200 {
98 errors << 'Skipping module `${name}`, since `${url}` responded with ${r.status_code} http status code. Please try again later.'
99 continue
100 }
101 s := r.body
102 if s.len > 0 && s[0] != `{` {
103 errors << 'Invalid json data'
104 errors << s.trim_space().limit(100) + '...'
105 continue
106 }
107 mod := json.decode(ModuleVpmInfo, s) or {
108 errors << 'Skipping module `${name}`, since its information is not in json format.'
109 continue
110 }
111 if '' == mod.url || '' == mod.name {
112 errors << 'Skipping module `${name}`, since it is missing name or url information.'
113 continue
114 }
115 if selector.selected_url == '' {
116 selector.selected_url = url
117 verbose_println_more(@FILE_LINE, @FN, 'Using `${url}` for this installation.')
118 }
119 if is_initial_selection {
120 selected_server_url(true, url)
121 }
122 verbose_println_more(@FILE_LINE, @FN, 'name: ${name}; mod: ${mod}')
123 return mod
124 }
125 final_error := errors.join_lines()
126 verbose_println_more(@FILE_LINE, @FN, 'failed due to these errors: ${final_error}')
127 return error(final_error)
128}
129
130fn new_install_server_selector() VpmInstallServerSelector {
131 return VpmInstallServerSelector{
132 candidate_urls: if settings.server_urls.len > 0 {
133 settings.server_urls
134 } else {
135 build_install_server_urls(vpm_server_urls, settings.mirror_urls)
136 }
137 }
138}
139
140fn build_install_server_urls(default_urls []string, mirror_urls []string) []string {
141 mut urls := []string{}
142 urls << default_urls
143 urls << mirror_urls
144 return unique_server_urls(urls)
145}
146
147fn (selector VpmInstallServerSelector) metadata_server_urls() []string {
148 return if selector.selected_url != '' {
149 [selector.selected_url]
150 } else {
151 selector.candidate_urls
152 }
153}
154
155fn get_ident_from_url(raw_url string) !(string, string) {
156 verbose_println_more(@FILE_LINE, @FN, 'raw_url: ${raw_url}')
157 // On Windows, absolute paths like `C:\...` are misinterpreted by urllib.parse
158 // (the drive letter `C:` is treated as a URL scheme). Handle local paths first.
159 if os.is_abs_path(raw_url) || raw_url.starts_with('./') || raw_url.starts_with('../')
160 || raw_url.starts_with('~/') || raw_url.starts_with('.\\') || raw_url.starts_with('..\\')
161 || raw_url.starts_with('file://') {
162 normalized := raw_url.trim_string_left('file://').replace('\\', '/').trim_left('/')
163 _, name := normalized.rsplit_once('/') or {
164 return '', normalized.trim_string_right('.git')
165 }
166 return '', name.trim_string_right('.git')
167 }
168 url := urllib.parse(raw_url) or { return error('failed to parse module URL `${raw_url}`.') }
169 normalized_path := url.path.trim_left('/').trim_space()
170 publisher, mut name := normalized_path.rsplit_once('/') or {
171 if settings.vcs == .hg && raw_url.count(':') > 1 {
172 verbose_println_more(@FILE_LINE, @FN, 'ok, publisher: "", name: "test_module"')
173 return '', 'test_module'
174 }
175 if url.scheme in ['file', ''] && normalized_path.len > 0 {
176 return '', normalized_path
177 }
178 final_error := 'failed to retrieve module name for `${url}`.'
179 verbose_println_more(@FILE_LINE, @FN, 'failed error: `${final_error}`')
180 return error(final_error)
181 }
182 name = name.trim_string_right('.git')
183 verbose_println_more(@FILE_LINE, @FN,
184 'raw_url: ${raw_url}; publisher: ${publisher}; name: ${name}')
185 return publisher, name
186}
187
188fn get_name_from_url(raw_url string) !string {
189 _, name := get_ident_from_url(raw_url)!
190 return name
191}
192
193fn normalize_mod_path(path string) string {
194 return path.replace('-', '_').to_lower()
195}
196
197fn get_all_modules_for_search() []string {
198 working_server_url := get_working_server_url()
199 verbose_println_more(@FILE_LINE, @FN, 'working_server_url: ${working_server_url}')
200 println('Search server: ${working_server_url} .')
201 return get_all_modules_for_search_from_server(working_server_url) or {
202 vpm_error(err.msg())
203 exit(1)
204 }
205}
206
207fn get_all_modules_for_search_with_selector(mut selector VpmInstallServerSelector) ![]string {
208 if selector.candidate_urls.len == 0 {
209 return error('no vpm server urls configured.')
210 }
211 mut errors := []string{}
212 is_initial_selection := selected_server_url(false, '') == ''
213 for url in selector.metadata_server_urls() {
214 modules := get_all_modules_for_search_from_server(url) or {
215 errors << err.msg()
216 continue
217 }
218 if selector.selected_url == '' {
219 selector.selected_url = url
220 }
221 if is_initial_selection {
222 selected_server_url(true, url)
223 }
224 return modules
225 }
226 return error(errors.join_lines())
227}
228
229fn get_all_modules_for_search_from_server(server_url string) ![]string {
230 search_url := '${server_url}/search'
231 verbose_println_more(@FILE_LINE, @FN, 'making a GET request to search_url: ${search_url} ...')
232 r := http.get(search_url) or {
233 return error('Http server did not respond to our request for `${search_url}`.\nError details: ${err}')
234 }
235 if r.status_code != 200 {
236 return error('failed to search through ${search_url}\nStatus code: ${r.status_code}')
237 }
238 modules := extract_modules_from_search_response(r.body)
239 verbose_println_more(@FILE_LINE, @FN, 'found modules: ${modules}')
240 return modules
241}
242
243fn extract_modules_from_search_response(s string) []string {
244 mut read_len := 0
245 mut modules := []string{}
246 for read_len < s.len {
247 mut start_token := '<a class="package-card__title hover:underline cursor-pointer" href="/packages/'
248 end_token := '">'
249 // get the start index of the module entry
250 mut start_index := s.index_after(start_token, read_len) or { -1 }
251 if start_index == -1 {
252 break
253 }
254 start_index += start_token.len
255
256 // get the index of the end of module entry
257 end_index := s.index_after(end_token, start_index) or { break }
258 m := s[start_index..end_index]
259 modules << m
260 read_len = end_index
261 if read_len >= s.len {
262 break
263 }
264 }
265 return modules
266}
267
268fn normalize_repo_lookup_url(raw_url string) !string {
269 normalized_url := if raw_url.starts_with('git@') {
270 'https://' + raw_url['git@'.len..].replace(':', '/')
271 } else {
272 raw_url
273 }
274 url := urllib.parse(normalized_url) or {
275 return error('failed to parse module URL `${raw_url}`.')
276 }
277 host := url.hostname().trim_space().to_lower()
278 path :=
279 url.path.trim_space().trim_right('/').trim_left('/').trim_string_right('.git').to_lower()
280 if host == '' || path == '' {
281 return error('failed to normalize module URL `${raw_url}`.')
282 }
283 return '${host}/${path}'
284}
285
286fn get_installed_modules() []string {
287 dirs := os.ls(settings.vmodules_path) or { return [] }
288 mut modules := []string{}
289 for dir in dirs {
290 adir := os.join_path(settings.vmodules_path, dir)
291 if dir in excluded_dirs || !os.is_dir(adir) {
292 continue
293 }
294 if os.exists(os.join_path(adir, 'v.mod')) && os.exists(os.join_path(adir, '.git', 'config')) {
295 // an official vlang module with a short module name, like `vsl`, `ui` or `markdown`
296 modules << dir
297 continue
298 }
299 author := dir
300 mods := os.ls(adir) or { continue }
301 for m in mods {
302 vcs_used_in_dir(os.join_path(adir, m)) or { continue }
303 modules << '${author}.${m}'
304 }
305 }
306 verbose_println_more(@FILE_LINE, @FN, 'found modules: ${modules}')
307 return modules
308}
309
310fn get_path_of_existing_module(mod_name string) ?string {
311 name := get_name_from_url(mod_name) or { mod_name.replace('-', '_').to_lower() }
312 path := os.real_path(os.join_path(settings.vmodules_path, name.replace('.', os.path_separator)))
313 if !os.exists(path) {
314 vpm_error('failed to find `${name}` at `${path}`.')
315 return none
316 }
317 if !os.is_dir(path) {
318 vpm_error('skipping `${path}`, since it is not a directory.')
319 return none
320 }
321 verbose_println_more(@FILE_LINE, @FN, 'mod_name: ${mod_name}, found path: ${path}')
322 return path
323}
324
325fn get_working_server_url() string {
326 is_initial_selection := selected_server_url(false, '') == ''
327 for url in active_server_urls() {
328 verbose_println('Trying server url: ${url}')
329 http.head(url) or {
330 vpm_error('failed to connect to server url `${url}`.', details: err.msg())
331 continue
332 }
333 if is_initial_selection {
334 selected_server_url(true, url)
335 }
336 verbose_println_more(@FILE_LINE, @FN, 'found url: ${url}')
337 return url
338 }
339 vpm_error('No responding vpm server found. Please check your network connectivity and try again later.')
340 exit(1)
341}
342
343fn ensure_vmodules_dir_exist() {
344 if !os.is_dir(settings.vmodules_path) {
345 println('Creating `${settings.vmodules_path}`...')
346 os.mkdir(settings.vmodules_path) or {
347 vpm_error(err.msg(), verbose: true)
348 exit(1)
349 }
350 }
351 verbose_println_more(@FILE_LINE, @FN, 'settings.vmodules_path: ${settings.vmodules_path}')
352}
353
354fn increment_module_download_count(name string, preferred_server_url string) ! {
355 if settings.no_dl_count_increment {
356 println('Skipping download count increment for `${name}`.')
357 return
358 }
359 server_urls := if preferred_server_url != '' {
360 unique_server_urls([preferred_server_url])
361 } else if settings.server_urls.len > 0 {
362 settings.server_urls
363 } else {
364 vpm_server_urls
365 }
366 if server_urls.len == 0 {
367 return error('no vpm server urls configured.')
368 }
369 mut errors := []string{}
370 is_initial_selection := selected_server_url(false, '') == ''
371 for url in server_urls {
372 modurl := url + '/api/packages/${name}/incr_downloads'
373 verbose_println_more(@FILE_LINE, @FN, 'making a POST request to modurl: ${modurl} ...')
374 r := http.post(modurl, '') or {
375 errors << 'Http server did not respond to our request for `${modurl}`.'
376 errors << 'Error details: ${err}'
377 continue
378 }
379 if r.status_code != 200 {
380 errors << 'Failed to increment the download count for module `${name}`, since `${url}` responded with ${r.status_code} http status code. Please try again later.'
381 continue
382 }
383 if is_initial_selection {
384 selected_server_url(true, url)
385 }
386 return
387 }
388 final_error := errors.join_lines()
389 verbose_println_more(@FILE_LINE, @FN, 'final_error: ${final_error}')
390 return error(final_error)
391}
392
393fn get_manifest(path string) ?vmod.Manifest {
394 return vmod.from_file(os.join_path(path, 'v.mod')) or { return none }
395}
396
397fn resolve_dependencies(manifest ?vmod.Manifest, modules []string) {
398 mod := manifest or { return }
399 // Filter out modules that are both contained in the input query and listed as
400 // dependencies in the mod file of the module that is supposed to be installed.
401 deps := mod.dependencies.filter(it !in modules)
402 verbose_println_more(@FILE_LINE, @FN, 'deps: ${deps}')
403 if deps.len > 0 {
404 println('Resolving ${deps.len} dependencies for module `${mod.name}`...')
405 verbose_println('Found dependencies: ${deps}')
406 vpm_install(deps)
407 }
408}
409
410fn verbose_println(msg string) {
411 if settings.is_verbose {
412 println(msg)
413 }
414}
415
416fn verbose_println_more(fline string, fname string, msg string) {
417 vpm_log(fline, fname, msg)
418 verbose_println(msg)
419}
420
421fn vpm_log_header(txt string) {
422 divider := '='.repeat(40 - txt.len / 2)
423 settings.logger.debug('\n${divider} ${txt} ${divider}\n')
424}
425
426fn vpm_log(line string, func string, msg string) {
427 settings.logger.debug('${line:-15s} fn: ${func:-30s} msg: ${msg}')
428}
429
430fn vpm_error(msg string, opts ErrorOptions) {
431 if opts.verbose && !settings.is_verbose {
432 return
433 }
434 eprintln(term.ecolorize(term.red, 'error: ') + msg)
435 if opts.details.len > 0 && settings.is_verbose {
436 eprint(term.ecolorize(term.cyan, 'details: '))
437 padding := ' '.repeat('details: '.len)
438 for i, line in opts.details.split_into_lines() {
439 if i > 0 {
440 eprint(padding)
441 }
442 eprintln(term.ecolorize(term.cyan, line))
443 }
444 }
445}
446
447fn vpm_warn(msg string, opts ErrorOptions) {
448 eprintln(term.ecolorize(term.yellow, 'warning: ') + msg)
449 if opts.details.len > 0 {
450 eprint(term.ecolorize(term.cyan, 'details: '))
451 padding := ' '.repeat('details: '.len)
452 for i, line in opts.details.split_into_lines() {
453 if i > 0 {
454 eprint(padding)
455 }
456 eprintln(term.ecolorize(term.cyan, line))
457 }
458 }
459}
460
461// Formatted version of the vmodules install path. E.g. `/home/user/.vmodules` -> `~/.vmodules`
462fn fmt_mod_path(path string) string {
463 if !path.contains(home_dir) {
464 return path
465 }
466 return $if windows {
467 path.replace(home_dir, '%USERPROFILE%')
468 } $else {
469 path.replace(home_dir, '~')
470 }
471}
472
473fn at_version(version string) string {
474 return if version != '' { '@${version}' } else { '' }
475}
476
477// FIXME: Workaround for failing `rmdir` commands on Windows.
478fn rmdir_all(path string) ! {
479 $if windows {
480 os.execute_opt('rd /s /q ${path}')!
481 } $else {
482 os.rmdir_all(path)!
483 }
484}
485