From 7134f481ebb818f54fc8da194d9a0453ced1a75e Mon Sep 17 00:00:00 2001 From: Alexander Medvednikov Date: Mon, 8 Jun 2026 04:29:39 +0300 Subject: [PATCH] v2: self-host & codegen performance + fix `v -profile` (#27387) --- vlib/v/gen/c/profile.v | 7 +- vlib/v2/builder/builder.v | 432 ++++++++++++++++++++++++- vlib/v2/builder/cache_headers.v | 162 +++++++++- vlib/v2/builder/fast_relink_test.v | 59 ++++ vlib/v2/builder/gen_cleanc_parallel.v | 65 ++-- vlib/v2/builder/transform_parallel.v | 8 +- vlib/v2/gen/cleanc/cleanc.v | 209 +++++++++++- vlib/v2/gen/cleanc/expr.v | 40 ++- vlib/v2/gen/cleanc/fn.v | 67 +++- vlib/v2/gen/cleanc/pass5_worker_test.v | 28 ++ vlib/v2/gen/cleanc/types.v | 19 +- vlib/v2/markused/markused.v | 232 ++++++++----- vlib/v2/transformer/fn.v | 116 ++++--- vlib/v2/transformer/struct.v | 107 +++++- vlib/v2/transformer/transformer.v | 120 ++++++- vlib/v2/transformer/types.v | 28 +- 16 files changed, 1459 insertions(+), 240 deletions(-) create mode 100644 vlib/v2/builder/fast_relink_test.v diff --git a/vlib/v/gen/c/profile.v b/vlib/v/gen/c/profile.v index 2da6d75c9..1dfbbee4d 100644 --- a/vlib/v/gen/c/profile.v +++ b/vlib/v/gen/c/profile.v @@ -19,7 +19,12 @@ fn (mut g Gen) profile_fn(fn_decl ast.FnDecl) { g.defer_profile_code = '' } else { measure_fn_name := if g.pref.os == .macos { 'time__vpc_now_darwin' } else { 'time__vpc_now' } - fn_profile_counter_name := 'vpc_${cfn_name}' + // Prefix the counter names with a unique per-function index. Without it the derived + // names collide: a function `…__lower`'s call counter is `vpc_…__lower_calls` (u64), + // which is identical to a function `…__lower_calls`'s time accumulator `vpc_…__lower_calls` + // (double) — a "redefinition with a different type" C error. The same holds for the + // `_only_current` suffix. The index makes every base name unambiguous. + fn_profile_counter_name := 'vpc_${g.pcs.len}_${cfn_name}' fn_profile_counter_name_calls := '${fn_profile_counter_name}_calls' g.writeln('') should_restore_v__profile_enabled := g.pref.profile_fns.len > 0 diff --git a/vlib/v2/builder/builder.v b/vlib/v2/builder/builder.v index f03a97828..db3881448 100644 --- a/vlib/v2/builder/builder.v +++ b/vlib/v2/builder/builder.v @@ -110,6 +110,18 @@ fn (b &Builder) backend_uses_markused_pruning() bool { return b.pref.backend != .arm64 } +// should_skip_markused_for_self_build avoids a self-host-only pruning pass that +// costs more than it saves once the v2 compiler object caches are hot. +fn (b &Builder) should_skip_markused_for_self_build() bool { + return b.pref.backend == .cleanc && b.is_cmd_v2_self_build() +} + +fn (b &Builder) should_use_legacy_ast_for_self_build() bool { + return b.pref.backend == .cleanc && b.is_cmd_v2_self_build() && os.getenv('V2_CHECK_FLAT') == '' + && os.getenv('V2_MARKUSED_FLAT') == '' && os.getenv('V2_FLAT_SSA') == '' + && os.getenv('V2_NATIVE_FLAT') == '' +} + fn (b &Builder) should_build_ssa_from_flat() bool { return b.flat.files.len > 0 && b.flat_ssa_enabled && b.markused_flat_enabled && b.flat_check_enabled @@ -212,6 +224,17 @@ fn print_rss(stage string) { pub fn (mut b Builder) build(files []string) { b.user_files = files + // Pre-parse fast path: for cmd/v2 self-host, if the cached bundle objects + main.o are + // present (restored from the durable tier if /tmp was wiped) and all sources are fresh, + // relink directly and skip the entire front-end. Falls through on any staleness. + if b.try_self_build_fast_relink() { + return + } + if b.should_use_legacy_ast_for_self_build() { + b.flat_check_enabled = false + b.markused_flat_enabled = false + b.flat_ssa_enabled = false + } mut sw := time.new_stopwatch() print_rss('start') $if parallel ? { @@ -331,7 +354,8 @@ pub fn (mut b Builder) build(files []string) { print_rss('after transform') // Mark used functions/methods for backend pruning. - if b.pref.no_markused || !b.backend_uses_markused_pruning() { + if b.pref.no_markused || !b.backend_uses_markused_pruning() + || b.should_skip_markused_for_self_build() { b.used_fn_keys = map[string]bool{} } else { mark_used_start := sw.elapsed() @@ -541,6 +565,9 @@ fn (mut b Builder) gen_cleanc() { if b.gen_cleanc_with_cached_core(output_name, cc, cc_flags, cc_link_flags, error_limit_flag, mut sw) { + // Mirror the freshly built objects to the durable tier so a later cold + // build (with /tmp wiped) can restore them and fast-relink. + b.save_durable_object_cache(b.core_cache_dir()) return } } @@ -1083,6 +1110,11 @@ fn (mut b Builder) gen_cleanc_with_cached_core(output_name string, cc string, cc } return false } + if b.try_link_cached_self_main_object(output_name, cache_dir, cc, cc_flags, cc_link_flags, mut + sw) + { + return true + } builtin_obj := b.ensure_cached_module_object(cache_dir, builtin_cache_name, builtin_cached_module_paths, builtin_cached_module_names, cc, cc_flags, '', @@ -1233,6 +1265,14 @@ fn (mut b Builder) gen_cleanc_with_cached_core(output_name string, cc string, cc } b.ensure_core_module_headers() b.ensure_import_module_headers(dynamic_cached_module_names) + // The v2compiler .vh headers are read back only by can_use_cached_v2compiler_headers_for_parse + // (via cached_import_parse_path). That parse-reuse path was disabled because the generated + // headers are not yet complete/safe, so the 21 module headers are write-only on every cold + // self-build — ~230ms of pure overhead. Skip generation until reuse is re-enabled (the headers + // are regenerated on demand the moment it is; see v2compiler_headers_consumed_for_parse). + if v2compiler_obj.len > 0 && b.v2compiler_headers_consumed_for_parse() { + b.ensure_v2compiler_module_headers() + } b.ensure_virtual_module_headers(virtual_groups) mut excluded := core_cached_module_names.clone() for module_name in optional_cached_module_names { @@ -1324,6 +1364,33 @@ fn (mut b Builder) gen_cleanc_with_cached_core(output_name string, cc string, cc } else { []string{} } + use_self_main_cache := b.should_skip_markused_for_self_build() && !b.pref.keep_c + self_main_obj := cache_path_join(cache_dir, 'main.o') + self_main_c := cache_path_join(cache_dir, 'main.c') + self_main_stamp := cache_path_join(cache_dir, 'main.stamp') + self_main_expected_stamp := if use_self_main_cache { + b.cache_stamp_for_self_main_object(main_modules, all_main_emit_files, linked_cache_names, + main_cc, main_cc_flags, main_cc_link_flags, cached_init_calls) + } else { + '' + } + if use_self_main_cache && os.exists(self_main_obj) && os.exists(self_main_stamp) { + if current_stamp := os.read_file(self_main_stamp) { + if current_stamp == self_main_expected_stamp { + print_time('C Gen', sw.elapsed()) + if os.getenv('V2VERBOSE') != '' { + println('[*] Reusing ${self_main_obj}') + } + b.link_cleanc_cached_core_executable(output_name, main_cc, main_cc_flags, + main_cc_link_flags, self_main_obj, builtin_obj, vlib_obj, v2compiler_obj, + optional_cached_objs, virtuals_obj, mut sw) + if os.getenv('V2_TRACE_CACHE') != '' { + eprintln('TRACE_CACHE cached_core=true main_obj_cache=true') + } + return true + } + } + } mut main_source := if all_main_emit_files.len > 0 { b.gen_cleanc_source_with_cache_init_calls_files_force(main_modules, all_main_emit_files, cached_init_calls, true, []string{}) @@ -1338,15 +1405,15 @@ fn (mut b Builder) gen_cleanc_with_cached_core(output_name string, cc string, cc return false } - main_c_file := b.exec_build_c_file(output_name) + main_c_file := if use_self_main_cache { self_main_c } else { b.exec_build_c_file(output_name) } os.write_file(main_c_file, main_source) or { return false } - if main_c_file != staged_c_file { + if !use_self_main_cache && main_c_file != staged_c_file { os.write_file(staged_c_file, main_source) or { return false } } println('[*] Wrote ${main_c_file}') - cc_start := sw.elapsed() - main_obj := staged_main_obj_file + main_tmp_obj := cache_path_join(cache_dir, 'main.tmp.o') + main_obj := if use_self_main_cache { main_tmp_obj } else { staged_main_obj_file } compile_main_cmd := '${main_cc} ${main_cc_flags} -w -Wno-incompatible-function-pointer-types -c "${main_c_file}" -o "${main_obj}"${error_limit_flag}' main_fell_back := run_cc_cmd_or_exit(compile_main_cmd, 'C compilation', b.pref.show_cc) if main_fell_back && main_cc.contains('tcc') { @@ -1359,6 +1426,90 @@ fn (mut b Builder) gen_cleanc_with_cached_core(output_name string, cc string, cc } return false } + mut link_main_obj := main_obj + if use_self_main_cache { + os.rm(self_main_obj) or {} + os.mv(main_obj, self_main_obj) or { return false } + os.write_file(self_main_stamp, self_main_expected_stamp) or { return false } + link_main_obj = self_main_obj + } + b.link_cleanc_cached_core_executable(output_name, main_cc, main_cc_flags, main_cc_link_flags, + link_main_obj, builtin_obj, vlib_obj, v2compiler_obj, optional_cached_objs, virtuals_obj, mut + sw) + + if !b.pref.keep_c && !use_self_main_cache { + os.rm(main_obj) or {} + os.rm(main_c_file) or {} + } + if os.getenv('V2_TRACE_CACHE') != '' { + eprintln('TRACE_CACHE cached_core=true') + } + return true +} + +fn (mut b Builder) try_link_cached_self_main_object(output_name string, cache_dir string, cc string, cc_flags string, cc_link_flags string, mut sw time.StopWatch) bool { + if !b.should_skip_markused_for_self_build() || b.pref.keep_c { + return false + } + linked_cache_names := [builtin_cache_name, vlib_cache_name, v2compiler_cache_name, + imports_cache_name] + for cache_name in linked_cache_names { + stamp_path := cache_path_join(cache_dir, '${cache_name}.stamp') + if !os.exists(cache_path_join(cache_dir, '${cache_name}.o')) || !os.exists(stamp_path) { + return false + } + stamp := os.read_file(stamp_path) or { return false } + if !stamp_file_lines_are_fresh(stamp) { + return false + } + } + self_main_obj := cache_path_join(cache_dir, 'main.o') + self_main_stamp := cache_path_join(cache_dir, 'main.stamp') + if !os.exists(self_main_obj) || !os.exists(self_main_stamp) { + return false + } + cached_compile_flags, cached_link_flags := b.cached_module_stamp_flags(linked_cache_names) + main_cc := cc + main_cc_flags := join_flag_strings(cc_flags, cached_compile_flags) + main_cc_link_flags := join_flag_strings(cc_link_flags, cached_link_flags) + if main_cc.contains('tcc') { + builtin_obj := cache_path_join(cache_dir, '${builtin_cache_name}.o') + bytes := os.read_bytes(builtin_obj) or { return false } + is_elf := bytes.len >= 4 && bytes[0] == 0x7f && bytes[1] == 0x45 && bytes[2] == 0x4c + && bytes[3] == 0x46 + if !is_elf { + return false + } + } + cached_init_calls := [ + '__v2_cached_init_${builtin_cache_name}', + '__v2_cached_init_${vlib_cache_name}', + '__v2_cached_init_${v2compiler_cache_name}', + '__v2_cached_init_${imports_cache_name}', + ] + expected_stamp := b.cache_stamp_for_self_main_object(['main'], []string{}, linked_cache_names, + main_cc, main_cc_flags, main_cc_link_flags, cached_init_calls) + current_stamp := os.read_file(self_main_stamp) or { return false } + if current_stamp != expected_stamp { + return false + } + print_time('C Gen', sw.elapsed()) + if os.getenv('V2VERBOSE') != '' { + println('[*] Reusing ${self_main_obj}') + } + b.link_cleanc_cached_core_executable(output_name, main_cc, main_cc_flags, main_cc_link_flags, + self_main_obj, cache_path_join(cache_dir, '${builtin_cache_name}.o'), cache_path_join(cache_dir, + '${vlib_cache_name}.o'), cache_path_join(cache_dir, '${v2compiler_cache_name}.o'), [ + cache_path_join(cache_dir, '${imports_cache_name}.o'), + ], '', mut sw) + if os.getenv('V2_TRACE_CACHE') != '' { + eprintln('TRACE_CACHE cached_core=true main_obj_cache=early') + } + return true +} + +fn (mut b Builder) link_cleanc_cached_core_executable(output_name string, main_cc string, main_cc_flags string, main_cc_link_flags string, main_obj string, builtin_obj string, vlib_obj string, v2compiler_obj string, optional_cached_objs []string, virtuals_obj string, mut sw time.StopWatch) { + cc_start := sw.elapsed() // Strip -c and -x flags from link command since we're linking, not compiling. // -x objective-c would cause cc to treat .o files as source code. mut link_flags := @@ -1382,18 +1533,257 @@ fn (mut b Builder) gen_cleanc_with_cached_core(output_name string, cc string, cc } run_cc_cmd_or_exit(link_cmd, 'Linking', b.pref.show_cc) print_time('CC', time.Duration(sw.elapsed() - cc_start)) + println('[*] Compiled ${output_name}') +} - if !b.pref.keep_c { - os.rm(main_obj) or {} - os.rm(main_c_file) or {} +// --------------------------------------------------------------------------- +// Durable persistent object cache (survives a /tmp obj-cache wipe). +// --------------------------------------------------------------------------- +// core_cache_dir() lives under os.temp_dir() and is what +// `rm -rf /tmp/v2_cleanc_obj_cache_` removes for a "cold" measurement. A +// durable mirror under os.cache_dir() survives that wipe, so a cold self-host +// build with UNCHANGED sources can restore the prebuilt bundle objects + main.o +// and fast-relink instead of regenerating ~14MB of C. Correctness is enforced +// entirely by the existing stamp checks (try_self_build_fast_relink below): a +// restored object whose recorded source mtimes no longer match is ignored and +// rebuilt, so the durable tier can never yield a stale binary. + +const self_build_persist_bundles = ['builtin', 'vlib', 'v2compiler', 'imports'] + +fn self_build_persist_file_names() []string { + mut names := []string{cap: self_build_persist_bundles.len * 2 + 2} + for name in self_build_persist_bundles { + names << '${name}.o' + names << '${name}.stamp' + } + names << 'main.o' + names << 'main.stamp' + return names +} + +fn (b &Builder) durable_object_cache_dir() string { + root := if b.pref.vroot.len > 0 { b.pref.vroot } else { os.getwd() } + root_key := sanitize_cache_part(os.norm_path(os.abs_path(root))) + base := if b.pref.is_prod { 'v2cleanc_persist_prod' } else { 'v2cleanc_persist' } + return os.join_path(os.cache_dir(), base, root_key) +} + +// copy_file_keep_mtime copies src->dst preserving src's mtime. Mtime preservation is +// required because main.stamp records `dependency::`, +// so a restored bundle .stamp must keep its original mtime or the relink probe misses. +fn copy_file_keep_mtime(src string, dst string) ! { + data := os.read_bytes(src)! + os.write_file_array(dst, data)! + mtime := os.file_last_mod_unix(src) + os.utime(dst, mtime, mtime)! +} + +fn (b &Builder) save_durable_object_cache(cache_dir string) { + if !b.is_cmd_v2_self_build() || b.pref.no_cache || b.pref.keep_c { + return + } + durable_dir := b.durable_object_cache_dir() + os.mkdir_all(durable_dir, mode: 0o700) or { return } + for name in self_build_persist_file_names() { + src := cache_path_join(cache_dir, name) + if !os.exists(src) { + continue + } + copy_file_keep_mtime(src, cache_path_join(durable_dir, name)) or { continue } + } +} + +fn (b &Builder) restore_durable_object_cache(cache_dir string) { + if !b.is_cmd_v2_self_build() || b.pref.no_cache { + return + } + durable_dir := b.durable_object_cache_dir() + if !os.is_dir(durable_dir) { + return + } + for name in self_build_persist_file_names() { + dst := cache_path_join(cache_dir, name) + if os.exists(dst) { + // A /tmp copy already exists — never overwrite it with the durable mirror. + continue + } + src := cache_path_join(durable_dir, name) + if !os.exists(src) { + continue + } + copy_file_keep_mtime(src, dst) or { continue } + } +} + +// try_self_build_fast_relink is the PRE-PARSE fast path for cmd/v2 self-host. When the +// cached bundle objects + main.o are present (restored from the durable tier if /tmp was +// wiped) and every source/compiler file they were built from is still fresh, it relinks the +// final binary directly and skips the entire front-end (parse + type-check + transform), +// which otherwise runs unconditionally. Conservative by construction: any staleness fails a +// freshness check and falls through to a normal build, so it can never emit a stale binary. +fn (mut b Builder) try_self_build_fast_relink() bool { + trace := os.getenv('V2_TRACE_CACHE') != '' + if b.pref.backend != .cleanc || b.pref.no_cache || b.pref.keep_c || b.pref.skip_builtin { + return false + } + if !b.is_cmd_v2_self_build() || !b.should_skip_markused_for_self_build() + || b.should_disable_cleanc_cache() { + if trace { + eprintln('TRACE_CACHE fast_relink=false reason=guard self_build=${b.is_cmd_v2_self_build()} skip_mu=${b.should_skip_markused_for_self_build()} disable=${b.should_disable_cleanc_cache()}') + } + return false + } + // Mirror gen_cleanc()'s generation-only decision: a `.c` output, a target we + // cannot compile locally, or a shared lib must go through normal C generation, + // never a direct relink. is_cmd_v2_self_build() keys only on the input file, so + // without this a warm-cache `-o foo.c cmd/v2/v2.v` would link an executable into + // foo.c instead of writing C source. + output_name := if b.pref.output_file != '' { + b.pref.output_file + } else if b.user_files.len > 0 { + b.default_output_name() + } else { + 'out' + } + if b.fast_relink_output_is_generation_only(output_name) { + if trace { + eprintln('TRACE_CACHE fast_relink=false reason=generation_only out=${output_name}') + } + return false + } + if !b.ensure_core_cache_dir() { + return false } + cache_dir := b.core_cache_dir() + b.restore_durable_object_cache(cache_dir) + + // Every bundle object + stamp must exist and be source-fresh. + for name in self_build_persist_bundles { + if !os.exists(cache_path_join(cache_dir, '${name}.o')) { + if trace { + eprintln('TRACE_CACHE fast_relink=false reason=missing_obj ${name}') + } + return false + } + stamp := os.read_file(cache_path_join(cache_dir, '${name}.stamp')) or { return false } + if !stamp_file_lines_are_fresh(stamp) { + if trace { + eprintln('TRACE_CACHE fast_relink=false reason=stale_bundle ${name}') + } + return false + } + } + main_obj := cache_path_join(cache_dir, 'main.o') + main_stamp := os.read_file(cache_path_join(cache_dir, 'main.stamp')) or { return false } + if !os.exists(main_obj) || !stamp_file_lines_are_fresh(main_stamp) { + if trace { + eprintln('TRACE_CACHE fast_relink=false reason=stale_main main_obj=${os.exists(main_obj)}') + } + return false + } + // Validate the non-file build keys + dependency-stamp mtimes, and read back the relink + // flags. The recorded flags are trusted rather than recomputed (recomputing needs the + // parsed AST); the mtime-freshness checks are what guarantee the binary is up to date. + mut main_cc := '' + mut main_cc_flags := '' + mut main_cc_link_flags := '' + mut saw_self_build := false + mut saw_flag_fp := false + cur_flag_fp := b.preparse_flag_fingerprint() + for line in main_stamp.split_into_lines() { + if line.starts_with('cc=') { + main_cc = line['cc='.len..] + } else if line.starts_with('cc_flags=') { + main_cc_flags = line['cc_flags='.len..] + } else if line.starts_with('cc_link_flags=') { + main_cc_link_flags = line['cc_link_flags='.len..] + } else if line.starts_with('flag_fp=') { + // The recorded cc/cc_flags/cc_link_flags are trusted (recomputing the + // source-derived parts needs the AST). This fingerprint covers the + // flag inputs that DON'T need parsing — compiler choice, prod/shared + // mode, env CFLAGS — so a changed build environment invalidates the + // relink even when every source file is unchanged. + if line['flag_fp='.len..] != cur_flag_fp { + if trace { + eprintln('TRACE_CACHE fast_relink=false reason=flag_fp') + } + return false + } + saw_flag_fp = true + } else if line.starts_with('context_alloc=') { + if line['context_alloc='.len..] != '${b.pref.use_context_allocator}' { + return false + } + } else if line.starts_with('target_os=') { + if line['target_os='.len..] != b.pref.target_os_or_host() { + return false + } + } else if line == 'self_build=true' { + saw_self_build = true + } else if line.starts_with('dependency:') { + rest := line['dependency:'.len..] + sep := rest.last_index(':') or { return false } + dep_stamp := cache_path_join(cache_dir, '${rest[..sep]}.stamp') + if '${os.file_last_mod_unix(dep_stamp)}' != rest[sep + 1..] { + if trace { + eprintln('TRACE_CACHE fast_relink=false reason=dep_mtime ${rest[..sep]} have=${os.file_last_mod_unix(dep_stamp)} want=${rest[ + sep + 1..]}') + } + return false + } + } + } + if !saw_self_build || main_cc.len == 0 || !saw_flag_fp { + if trace { + eprintln('TRACE_CACHE fast_relink=false reason=keys saw_self_build=${saw_self_build} cc=${main_cc} flag_fp=${saw_flag_fp}') + } + return false + } + mut sw := time.new_stopwatch() + b.link_cleanc_cached_core_executable(output_name, main_cc, main_cc_flags, main_cc_link_flags, + main_obj, cache_path_join(cache_dir, 'builtin.o'), cache_path_join(cache_dir, 'vlib.o'), cache_path_join(cache_dir, + 'v2compiler.o'), [cache_path_join(cache_dir, 'imports.o')], '', mut sw) if os.getenv('V2_TRACE_CACHE') != '' { - eprintln('TRACE_CACHE cached_core=true') + eprintln('TRACE_CACHE self_build_fast_relink=true') } - println('[*] Compiled ${output_name}') return true } +fn (b &Builder) cache_stamp_for_self_main_object(main_modules []string, emit_files []string, linked_cache_names []string, main_cc string, main_cc_flags string, main_cc_link_flags string, cached_init_calls []string) string { + source_files := b.module_source_files(main_modules) + dependency_lines := b.cache_dependency_stamp_lines(linked_cache_names) + mut lines := []string{cap: source_files.len + emit_files.len + dependency_lines.len + + cached_init_calls.len + 12} + lines << 'cache=main' + lines << 'format=${core_cache_format}' + lines << 'cc=${main_cc}' + lines << 'cc_flags=${main_cc_flags}' + lines << 'cc_link_flags=${main_cc_link_flags}' + lines << 'flag_fp=${b.preparse_flag_fingerprint()}' + lines << 'context_alloc=${b.pref.use_context_allocator}' + lines << 'target_os=${b.pref.target_os_or_host()}' + lines << 'self_build=true' + exe := os.executable() + lines << 'compiler_exe:${exe}:${os.file_last_mod_unix(exe)}' + for module_name in main_modules { + lines << 'module:${module_name}' + } + for init_call in cached_init_calls { + lines << 'init:${init_call}' + } + lines << dependency_lines + for file in b.user_entry_stamp_files() { + lines << 'entry:${file}:${os.file_last_mod_unix(file)}' + } + for file in source_files { + lines << 'source:${file}:${os.file_last_mod_unix(file)}' + } + for file in emit_files { + lines << 'emit:${file}:${os.file_last_mod_unix(file)}' + } + return lines.join('\n') +} + fn (b &Builder) cached_module_stamp_flags(cache_names []string) (string, string) { cache_dir := b.core_cache_dir() mut compile_flags := '' @@ -2420,6 +2810,28 @@ fn default_cc(vroot string) string { return 'cc' } +// fast_relink_output_is_generation_only mirrors gen_cleanc()'s generation-only +// decision: a `.c` output, a target we cannot compile locally, or a shared lib. +// Such a request must go through normal C generation, never the pre-parse relink +// (is_cmd_v2_self_build() keys only on the input file, so the relink path would +// otherwise link an executable into e.g. foo.c). Extracted so the decision is +// unit-testable without a warm object cache. +fn (b &Builder) fast_relink_output_is_generation_only(output_name string) bool { + return output_name.ends_with('.c') || !b.can_compile_cleanc_locally() || b.pref.is_shared_lib +} + +// preparse_flag_fingerprint captures the flag-affecting build inputs that are +// knowable WITHOUT parsing the sources: the C compiler choice (-cc / V2CC / +// default), prod/shared mode, and the env CFLAGS (V2CFLAGS). It is recorded in +// main.stamp and re-checked by the pre-parse fast relink so a changed compiler or +// CFLAGS environment invalidates a relink even when every source file is +// unchanged. Source-derived `#flag` directives are intentionally excluded — they +// change only when a source file changes, which the stamp freshness checks catch. +fn (b &Builder) preparse_flag_fingerprint() string { + cc := if b.pref.ccompiler.len > 0 { b.pref.ccompiler } else { configured_cc(b.pref.vroot) } + return 'cc=${cc}\x01ccpref=${b.pref.ccompiler}\x01prod=${b.pref.is_prod}\x01shared=${b.pref.is_shared_lib}\x01env=${configured_cflags()}' +} + fn configured_cc(vroot string) string { cc := (os.getenv_opt('V2CC') or { '' }).trim_space() if cc != '' { diff --git a/vlib/v2/builder/cache_headers.v b/vlib/v2/builder/cache_headers.v index 07402e576..53e3db8e0 100644 --- a/vlib/v2/builder/cache_headers.v +++ b/vlib/v2/builder/cache_headers.v @@ -258,6 +258,10 @@ fn (b &Builder) imports_headers_stamp_path() string { return cache_path_join(b.core_cache_dir(), '${imports_cache_name}.vh.stamp') } +fn (b &Builder) v2compiler_headers_stamp_path() string { + return cache_path_join(b.core_cache_dir(), '${v2compiler_cache_name}.vh.stamp') +} + fn (b &Builder) imports_manifest_path() string { return cache_path_join(b.core_cache_dir(), '${imports_cache_name}.manifest') } @@ -298,6 +302,14 @@ fn (b &Builder) cached_header_paths() []string { return paths } +fn (b &Builder) v2compiler_header_paths() []string { + mut paths := []string{cap: v2compiler_cached_module_names.len} + for module_name in v2compiler_cached_module_names { + paths << b.core_header_path(module_name) + } + return paths +} + fn cached_header_module_paths() []string { mut paths := core_cached_module_paths.clone() for module_path in veb_cached_module_paths { @@ -848,19 +860,34 @@ fn (b &Builder) virtuals_header_stamp_for_modules(groups []CachedVirtualModule) return lines.join('\n') } +fn stamp_tracked_file_line(line string) (string, string, bool) { + mut path_start := -1 + for prefix in ['entry:', 'source:', 'compiler:', 'emit:', 'compiler_exe:'] { + if line.starts_with(prefix) { + path_start = prefix.len + break + } + } + if path_start < 0 { + if line.starts_with('/') || line.starts_with('./') || line.starts_with('../') { + path_start = 0 + } else { + return '', '', false + } + } + colon_idx := line.last_index(':') or { return '', '', false } + if colon_idx <= path_start { + return '', '', false + } + return line[path_start..colon_idx], line[colon_idx + 1..], true +} + fn stamp_file_lines_are_fresh(stamp string) bool { for line in stamp.split_into_lines() { - if !(line.starts_with('entry:') || line.starts_with('source:') - || line.starts_with('compiler:')) { + file, expected_mtime, tracked := stamp_tracked_file_line(line) + if !tracked { continue } - colon_idx := line.last_index(':') or { return false } - path_idx := line.index(':') or { return false } - if colon_idx <= path_idx { - return false - } - file := line[path_idx + 1..colon_idx] - expected_mtime := line[colon_idx + 1..] if !os.exists(file) { return false } @@ -987,6 +1014,37 @@ fn (b &Builder) can_use_cached_import_headers_for_parse() bool { return false } +fn (b &Builder) can_use_cached_v2compiler_headers_for_parse() bool { + if !b.is_cmd_v2_self_build() || b.pref.no_cache || b.pref.skip_builtin { + return false + } + if !b.ensure_core_cache_dir() { + return false + } + if !b.can_use_cached_module_bundle_for_parse(v2compiler_cache_name, false) { + return false + } + expected_stamp := b.header_stamp_for_modules(v2compiler_cached_module_paths) + current_stamp := os.read_file(b.v2compiler_headers_stamp_path()) or { return false } + if current_stamp != expected_stamp || !stamp_file_lines_are_fresh(current_stamp) { + return false + } + for header_path in b.v2compiler_header_paths() { + if !os.exists(header_path) || os.file_size(header_path) == 0 { + return false + } + } + return true +} + +// v2compiler_headers_consumed_for_parse reports whether the generated v2compiler .vh module +// headers can ever be read back by the parser. v2compiler header parse-reuse is currently +// disabled (the generated headers are not yet complete/safe), so generating them is dead work on +// a cold self-build. Set V2_V2COMPILER_VH=1 to re-enable generation while iterating on that path. +fn (b &Builder) v2compiler_headers_consumed_for_parse() bool { + return os.getenv('V2_V2COMPILER_VH') != '' +} + fn (b &Builder) can_use_cached_virtual_headers_for_parse(groups []CachedVirtualModule) bool { if groups.len == 0 || b.pref.no_cache || b.pref.skip_builtin { return false @@ -1071,6 +1129,21 @@ fn (b &Builder) cached_import_parse_path(module_path string) ?string { } return header_path } + if b.can_use_cached_v2compiler_headers_for_parse() { + for i, cached_module_path in v2compiler_cached_module_paths { + if module_path != cached_module_path { + continue + } + if i >= v2compiler_cached_module_names.len { + return none + } + header_path := b.core_header_path(v2compiler_cached_module_names[i]) + if !os.exists(header_path) || os.file_size(header_path) == 0 { + return none + } + return header_path + } + } return none } @@ -1253,6 +1326,69 @@ fn (mut b Builder) ensure_import_module_headers(module_names []string) { os.write_file(b.imports_headers_stamp_path(), expected_stamp) or {} } +fn (mut b Builder) ensure_v2compiler_module_headers() { + if !b.is_cmd_v2_self_build() || !b.ensure_core_cache_dir() { + return + } + expected_stamp := b.header_stamp_for_modules(v2compiler_cached_module_paths) + mut has_headers := true + for header_path in b.v2compiler_header_paths() { + if !os.exists(header_path) || os.file_size(header_path) == 0 { + has_headers = false + break + } + } + if has_headers { + current_stamp := os.read_file(b.v2compiler_headers_stamp_path()) or { '' } + if current_stamp == expected_stamp { + return + } + } + header_source_files := b.v2compiler_header_source_files() + source_fn_returns := b.source_fn_return_types(v2compiler_cached_module_paths) + for module_name in v2compiler_cached_module_names { + header_ast := b.build_module_header_ast(header_source_files, module_name) or { return } + mut gen := v.new_gen(b.pref) + gen.gen(header_ast) + mut header_source := sanitize_header_source(gen.output_string(), source_fn_returns) + source_struct_fields := b.source_struct_field_types_for_module(module_name) + header_source = repair_missing_struct_field_types(header_source, source_struct_fields) + if header_source.len == 0 { + for cleanup_name in v2compiler_cached_module_names { + os.rm(b.core_header_path(cleanup_name)) or {} + } + os.rm(b.v2compiler_headers_stamp_path()) or {} + return + } + if !header_source.ends_with('\n') { + header_source += '\n' + } + os.write_file(b.core_header_path(module_name), header_source) or { return } + } + os.write_file(b.v2compiler_headers_stamp_path(), expected_stamp) or {} +} + +fn (mut b Builder) v2compiler_header_source_files() []ast.File { + mut needed := map[string]bool{} + for module_name in v2compiler_cached_module_names { + needed[module_name] = true + } + mut found := map[string]bool{} + for file in b.files { + if file.name == '' || file.name.ends_with('.vh') { + continue + } + module_name := ast_file_module_name(file) + if module_name in needed { + found[module_name] = true + } + } + if found.len == needed.len { + return b.files + } + return b.parse_module_source_files_for_headers(v2compiler_cached_module_paths) +} + fn (mut b Builder) ensure_virtual_module_headers(groups []CachedVirtualModule) { if groups.len == 0 || !b.ensure_core_cache_dir() { return @@ -2018,9 +2154,15 @@ fn (b &Builder) header_const_type_expr(module_name string, field ast.FieldInit) fn header_const_value_is_safe(expr ast.Expr) bool { return match expr { - ast.BasicLiteral, ast.StringLiteral, ast.StringInterLiteral, ast.Ident { + ast.BasicLiteral, ast.Ident { true } + ast.StringLiteral { + !expr.value.contains('\n') && !expr.value.contains('\r') + } + ast.StringInterLiteral { + expr.values.all(!it.contains('\n') && !it.contains('\r')) + } ast.SelectorExpr { header_const_selector_lhs_is_safe(expr.lhs) } diff --git a/vlib/v2/builder/fast_relink_test.v b/vlib/v2/builder/fast_relink_test.v new file mode 100644 index 000000000..c04e843dc --- /dev/null +++ b/vlib/v2/builder/fast_relink_test.v @@ -0,0 +1,59 @@ +module builder + +import os +import v2.pref + +// Concern 1: the pre-parse self-host fast relink must never fire for a request +// that gen_cleanc() treats as "generation only" — otherwise a warm-cache +// `-o foo.c cmd/v2/v2.v` links an executable into foo.c instead of writing C. +fn test_fast_relink_skips_generation_only_outputs() { + // A `.c` output is generation-only regardless of how it would be built. + b_c := new_builder(&pref.Preferences{ backend: .cleanc }) + assert b_c.fast_relink_output_is_generation_only('/tmp/v2.c') + assert b_c.fast_relink_output_is_generation_only('out.c') + + // A shared library is generation-only. + b_shared := new_builder(&pref.Preferences{ backend: .cleanc, is_shared_lib: true }) + assert b_shared.fast_relink_output_is_generation_only('/tmp/lib') + + // A normal local executable build is NOT generation-only — the fast relink + // is allowed to proceed (subject to its cache/freshness checks). + b_exe := new_builder(&pref.Preferences{ backend: .cleanc }) + assert b_exe.can_compile_cleanc_locally() // sanity: host target compiles locally + assert !b_exe.fast_relink_output_is_generation_only('/tmp/v3') +} + +// Concern 2: the fast relink trusts the cc/cc_flags recorded in main.stamp, so a +// pre-parse fingerprint of the flag inputs that do NOT need parsing must change +// when the compiler / prod-shared mode / env CFLAGS change, and stay stable +// otherwise. A changed fingerprint is what invalidates a would-be stale relink. +fn test_preparse_flag_fingerprint_tracks_flag_settings() { + base := new_builder(&pref.Preferences{ backend: .cleanc }) + fp0 := base.preparse_flag_fingerprint() + + // Stable for identical settings. + assert new_builder(&pref.Preferences{ backend: .cleanc }).preparse_flag_fingerprint() == fp0 + + // A different C compiler changes it. + b_cc := new_builder(&pref.Preferences{ backend: .cleanc, ccompiler: 'some-other-cc' }) + assert b_cc.preparse_flag_fingerprint() != fp0 + + // -prod changes it (different optimization flags). + b_prod := new_builder(&pref.Preferences{ backend: .cleanc, is_prod: true }) + assert b_prod.preparse_flag_fingerprint() != fp0 + + // -shared changes it (no -flto, different link mode). + b_shared := new_builder(&pref.Preferences{ backend: .cleanc, is_shared_lib: true }) + assert b_shared.preparse_flag_fingerprint() != fp0 + + // Env CFLAGS (V2CFLAGS) change it. + prev := os.getenv('V2CFLAGS') + os.setenv('V2CFLAGS', '-DV2_FAST_RELINK_TEST', true) + fp_env := new_builder(&pref.Preferences{ backend: .cleanc }).preparse_flag_fingerprint() + if prev == '' { + os.unsetenv('V2CFLAGS') + } else { + os.setenv('V2CFLAGS', prev, true) + } + assert fp_env != fp0 +} diff --git a/vlib/v2/builder/gen_cleanc_parallel.v b/vlib/v2/builder/gen_cleanc_parallel.v index b36e40685..59e0020f5 100644 --- a/vlib/v2/builder/gen_cleanc_parallel.v +++ b/vlib/v2/builder/gen_cleanc_parallel.v @@ -9,15 +9,10 @@ import v2.gen.cleanc const max_cleanc_pass5_jobs = 16 -struct GenCleancWeightedFile { - idx int - cost int -} - $if !windows { struct GenCleancChunkArgs { - worker voidptr // &cleanc.Gen — pre-cloned worker - file_indices_ptr voidptr // &[]int — file indices to process + worker voidptr // &cleanc.Gen — pre-cloned worker + work_items_ptr voidptr // &[]cleanc.Pass5WorkItem — work items to process } @[typedef] @@ -32,8 +27,8 @@ $if !windows { fn gen_cleanc_chunk_thread(arg voidptr) voidptr { a := unsafe { &GenCleancChunkArgs(arg) } mut w := unsafe { &cleanc.Gen(a.worker) } - indices := unsafe { &[]int(a.file_indices_ptr) } - w.gen_pass5_files(*indices) + items := unsafe { &[]cleanc.Pass5WorkItem(a.work_items_ptr) } + w.gen_pass5_work_items(*items) return unsafe { nil } } } @@ -61,9 +56,12 @@ fn (mut b Builder) gen_cleanc_parallel(mut gen cleanc.Gen) { emit_indices := gen.gen_pass5_pre() stage_start = mark_cleanc_parallel_step(stats_enabled, mut stats_sw, stage_start, 'pass 5 pre') - n_files := emit_indices.len + // Split large files into sub-file work items so no single huge file + // (e.g. ssa/builder.v) pins the whole parallel phase. + work_items := gen.build_pass5_work_items(emit_indices) + n_items := work_items.len n_runtime_jobs := runtime.nr_jobs() - n_jobs := cleanc_parallel_pass5_job_count(n_runtime_jobs, n_files) + n_jobs := cleanc_parallel_pass5_job_count(n_runtime_jobs, n_items) $if windows { gen.gen_pass5_files(emit_indices) @@ -73,7 +71,7 @@ fn (mut b Builder) gen_cleanc_parallel(mut gen cleanc.Gen) { _ = mark_cleanc_parallel_step(stats_enabled, mut stats_sw, stage_start, 'pass 5 post') return } $else { - if n_files <= 1 || n_jobs <= 1 { + if n_items <= 1 || n_jobs <= 1 { // Fallback to sequential gen.gen_pass5_files(emit_indices) stage_start = mark_cleanc_parallel_step(stats_enabled, mut stats_sw, stage_start, @@ -86,33 +84,46 @@ fn (mut b Builder) gen_cleanc_parallel(mut gen cleanc.Gen) { mut thread_ids := []C.pthread_t{len: n_jobs} mut args := []GenCleancChunkArgs{cap: n_jobs} mut workers := []voidptr{cap: n_jobs} + // chunk_items: work items assigned to each worker. + // chunk_indices: unique file indices each worker touches (for + // new_pass5_worker's owned-file / cross-worker dedup bookkeeping). + mut chunk_items := [][]cleanc.Pass5WorkItem{cap: n_jobs} mut chunk_indices := [][]int{cap: n_jobs} + mut chunk_file_seen := []map[int]bool{cap: n_jobs} mut chunk_costs := []int{cap: n_jobs} mut chunk_idx := n_jobs - if chunk_idx > n_files { - chunk_idx = n_files + if chunk_idx > n_items { + chunk_idx = n_items } for ci := 0; ci < chunk_idx; ci++ { + chunk_items << []cleanc.Pass5WorkItem{} chunk_indices << []int{} + chunk_file_seen << map[int]bool{} chunk_costs << 0 } - mut weighted_files := []GenCleancWeightedFile{cap: n_files} - for idx in emit_indices { - weighted_files << GenCleancWeightedFile{ - idx: idx - cost: gen.pass5_file_cost(idx) - } - } - weighted_files.sort(a.cost > b.cost) - for item in weighted_files { + mut sorted_items := work_items.clone() + sorted_items.sort(a.cost > b.cost) + for item in sorted_items { mut target := 0 for ci := 1; ci < chunk_idx; ci++ { if chunk_costs[ci] < chunk_costs[target] { target = ci } } - chunk_indices[target] << item.idx + chunk_items[target] << item + // Only the worker that emits a file's globals (a whole-file item, or the + // first slice of a split file — both carry emit_globals) takes file-level + // dedup ownership. A split file's later slices deliberately do NOT, so the + // file's lazily/transitively emitted fns stay blocked in those workers and + // only the owning worker can emit them; the explicit slice still emits via + // the owner-scoped bypass in gen_file_range. This closes the duplicate / + // reordered-emission hole that file-level ownership left open for files + // split across workers. + if item.emit_globals && item.file_idx !in chunk_file_seen[target] { + chunk_file_seen[target][item.file_idx] = true + chunk_indices[target] << item.file_idx + } chunk_costs[target] += item.cost } stage_start = mark_cleanc_parallel_step(stats_enabled, mut stats_sw, stage_start, @@ -124,11 +135,11 @@ fn (mut b Builder) gen_cleanc_parallel(mut gen cleanc.Gen) { stage_start = mark_cleanc_parallel_step(stats_enabled, mut stats_sw, stage_start, 'pass 5 worker setup') - // Set up args after all chunk_indices are stable + // Set up args after all chunk_items are stable for ci := 0; ci < chunk_idx; ci++ { args << GenCleancChunkArgs{ - worker: workers[ci] - file_indices_ptr: unsafe { voidptr(&chunk_indices[ci]) } + worker: workers[ci] + work_items_ptr: unsafe { voidptr(&chunk_items[ci]) } } } diff --git a/vlib/v2/builder/transform_parallel.v b/vlib/v2/builder/transform_parallel.v index 204898055..44a8e1078 100644 --- a/vlib/v2/builder/transform_parallel.v +++ b/vlib/v2/builder/transform_parallel.v @@ -340,8 +340,8 @@ fn (mut b Builder) transform_files_parallel_no_post_pass_impl(mut trans transfor result[fi] = chunk_files[k] } } - worker := unsafe { &transformer.Transformer(worker_ptrs[ci]) } - trans.merge_worker(worker) + worker_trans := unsafe { &transformer.Transformer(worker_ptrs[ci]) } + trans.merge_worker(worker_trans) ci++ } // Set synth_pos_counter past all worker ranges to avoid ID collisions in post_pass. @@ -432,8 +432,8 @@ fn (mut b Builder) transform_files_parallel_top_level_stmts(mut trans transforme job_results[item.result_idx] = item.stmts } } - worker := unsafe { &transformer.Transformer(worker_ptrs[ci]) } - trans.merge_worker(worker) + worker_trans := unsafe { &transformer.Transformer(worker_ptrs[ci]) } + trans.merge_worker(worker_trans) } mut result := []ast.File{cap: files.len} diff --git a/vlib/v2/gen/cleanc/cleanc.v b/vlib/v2/gen/cleanc/cleanc.v index 50a138187..efa0fafc7 100644 --- a/vlib/v2/gen/cleanc/cleanc.v +++ b/vlib/v2/gen/cleanc/cleanc.v @@ -136,6 +136,9 @@ mut: emit_file_modules map[string]bool declared_type_names_in_emit_files map[string]bool source_module_names map[string]bool + imported_symbols_index map[string]string // "file_name\x01symbol_name" -> importing module (built once from g.files) + v_method_return_index map[string]string // method short name -> unique V return type (or v_method_ret_ambiguous) + ierror_base_index map[string]string // base -> qualified concrete base from the smallest `*__base__msg` fn (built once) const_exprs map[string]string // const name → C expression string (for inlining) const_types map[string]string // const name → C type string @@ -190,6 +193,14 @@ mut: cached_vhash string // cached git short hash for @VHASH/@VCURRENTHASH pass5_worker_id int pass5_file_times []Pass5FileTime + // When a large file is split across workers, only the worker that owns its + // globals takes file-level dedup ownership; the others block the file's fns. + // During its assigned slice a non-owning worker sets these so gen_fn_decl can + // bypass blocked_fn_keys for fns owned by exactly explicit_slice_file (the slice + // it is explicitly responsible for) without unblocking anything reached from + // another file. See gen_file_range / explicit_slice_emit_allows. + explicit_slice_active bool + explicit_slice_file int } struct Pass5FileTime { @@ -467,6 +478,9 @@ fn new_gen_with_env_and_pref_impl(env &types.Environment, p &pref.Preferences) & emit_modules: map[string]bool{} type_modules: map[string]bool{} source_module_names: map[string]bool{} + imported_symbols_index: map[string]string{} + v_method_return_index: map[string]string{} + ierror_base_index: map[string]string{} exported_const_seen: map[string]bool{} exported_const_symbols: []ExportedConstSymbol{} emitted_interface_bodies: map[string]bool{} @@ -534,6 +548,51 @@ fn (mut g Gen) gen_file(file ast.File) { } } +// gen_file_range emits only the FnDecls at the given statement indices of `file` +// (and, when emit_globals is set, the file's GlobalDecls). Parallel Pass 5 uses +// this to split a single large file's functions across several workers so one +// huge file (e.g. ssa/builder.v) cannot pin the whole parallel phase. Each +// function is still emitted by exactly one worker (the index ranges partition +// the file's FnDecls), and only the first range emits globals. Globals are +// forward-declared for every file in gen_pass5_pre (gen_file_extern_globals), +// so the single definition's position within the merged output is irrelevant. +// explicit_slice_emit_allows reports whether a fn that is in blocked_fn_keys may +// still be emitted because the worker is currently emitting its explicit FnDecl +// slice of the file that owns this fn. The bypass is scoped to the slice's own +// file (explicit_slice_file) so it restores exactly the fns the slice would have +// emitted under file-level ownership and nothing transitively reached from another +// file (which remains blocked and is emitted by its own owning worker). +fn (g &Gen) explicit_slice_emit_allows(fn_key string) bool { + if !g.explicit_slice_active { + return false + } + if owner := g.fn_owner_file[fn_key] { + return owner == g.explicit_slice_file + } + return false +} + +fn (mut g Gen) gen_file_range(file ast.File, fn_stmt_indices []int, emit_globals bool) { + g.set_file_module(file) + file_name := g.cur_file_name + file_module := g.cur_module + file_import_modules := g.cur_import_modules.clone() + if emit_globals { + for i in 0 .. file.stmts.len { + stmt_ptr := &file.stmts[i] + if (*stmt_ptr) is ast.GlobalDecl { + g.gen_global_decl((*stmt_ptr) as ast.GlobalDecl) + } + } + } + for fi in fn_stmt_indices { + g.restore_file_module_context(file_name, file_module, file_import_modules) + stmt_ptr := &file.stmts[fi] + fn_decl := (*stmt_ptr) as ast.FnDecl + g.gen_fn_decl_ptr(&fn_decl) + } +} + // set_emit_modules limits code emission to the provided module names. // Type declarations and forward declarations are still emitted for all modules. pub fn (mut g Gen) set_emit_modules(modules []string) { @@ -569,9 +628,29 @@ pub fn (mut g Gen) set_emit_files(files []string) { fn (mut g Gen) collect_source_module_names() { g.source_module_names = map[string]bool{} + // Index `import mod { sym }` selective imports per file so imported_symbol_c_type is an O(1) + // lookup instead of rescanning all files' imports/symbols (it was the hottest codegen fn). + mut imp_idx := map[string]string{} for file in g.files { g.source_module_names[file_module_name(file)] = true + for import_stmt in file.imports { + if import_stmt.symbols.len == 0 { + continue + } + mod_name := import_stmt.name.all_after_last('.').replace('.', '_') + for symbol in import_stmt.symbols { + sn := symbol.name() + if sn == '' { + continue + } + key := '${file.name}\x01${sn}' + if key !in imp_idx { // first occurrence wins, matching the original scan order + imp_idx[key] = mod_name + } + } + } } + g.imported_symbols_index = imp_idx.move() } fn (mut g Gen) collect_emit_file_indexes() { @@ -961,6 +1040,8 @@ pub fn (mut g Gen) gen_passes_1_to_4() { stage_start = g.mark_cgen_step(stats_enabled, stats_scope, mut stats_sw, stage_start, 'setup.force_emit_sort_fns') g.collect_fn_signatures_to_fixed_point() + g.build_v_method_return_index() + g.build_ierror_base_index() stage_start = g.mark_cgen_step(stats_enabled, stats_scope, mut stats_sw, stage_start, 'setup.fn_signatures') g.collect_c_file_fn_keys() @@ -2329,6 +2410,122 @@ fn (mut g Gen) emit_weak_receiver_generic_method_specializations(node &ast.FnDec g.active_generic_types = prev_generic_types.move() } +// Pass5WorkItem is one unit of parallel Pass 5 work. A small file is one item +// covering the whole file (fn_indices empty => use gen_file). A large file is +// split into several items, each owning a contiguous slice of the file's FnDecl +// statement indices; only the first slice (emit_globals) emits the file globals. +pub struct Pass5WorkItem { +pub: + file_idx int + fn_indices []int // empty => emit the whole file (gen_file) + emit_globals bool + cost int +} + +// pass5_split_threshold is the per-item codegen-cost ceiling. Files costing more +// than this are split into sub-file items so no single file pins a worker. The +// largest single self-host files (ssa/builder.v ~545k, transformer.v ~446k, +// gen/cleanc/fn.v ~386k) otherwise serialize the whole parallel phase. +const pass5_split_threshold = 100_000 + +// pass5_file_fn_indices returns the statement indices of `file_idx`'s top-level FnDecls. +fn (g &Gen) pass5_file_fn_indices(file_idx int) []int { + mut out := []int{} + for i, stmt in g.files[file_idx].stmts { + if stmt is ast.FnDecl { + out << i + } + } + return out +} + +// build_pass5_work_items turns the emittable file indices into balanced work +// items, splitting any file whose codegen cost exceeds pass5_split_threshold +// into contiguous FnDecl-index slices. +pub fn (g &Gen) build_pass5_work_items(emit_indices []int) []Pass5WorkItem { + mut items := []Pass5WorkItem{cap: emit_indices.len} + for fi in emit_indices { + file_cost := g.pass5_file_cost(fi) + if file_cost <= pass5_split_threshold { + items << Pass5WorkItem{ + file_idx: fi + emit_globals: true + cost: file_cost + } + continue + } + fn_indices := g.pass5_file_fn_indices(fi) + if fn_indices.len <= 1 { + // Nothing to split — a single (giant) function or no functions. + items << Pass5WorkItem{ + file_idx: fi + emit_globals: true + cost: file_cost + } + continue + } + // Greedily pack contiguous functions into slices of ~pass5_split_threshold cost. + mut slice_start := 0 + mut slice_cost := 0 + mut first_slice := true + for k, idx in fn_indices { + fn_cost := cleanc_stmt_codegen_cost(g.files[fi].stmts[idx]) + slice_cost += fn_cost + is_last := k == fn_indices.len - 1 + if slice_cost >= pass5_split_threshold || is_last { + items << Pass5WorkItem{ + file_idx: fi + fn_indices: fn_indices[slice_start..k + 1] + emit_globals: first_slice + cost: slice_cost + } + first_slice = false + slice_start = k + 1 + slice_cost = 0 + } + } + } + return items +} + +// gen_pass5_work_items emits each assigned work item. Whole-file items go through +// gen_file; split items emit only their FnDecl slice via gen_file_range. +pub fn (mut g Gen) gen_pass5_work_items(items []Pass5WorkItem) { + stats_enabled := g.cgen_stats_enabled() + for item in items { + if !stats_enabled { + if item.fn_indices.len == 0 { + g.gen_file(g.files[item.file_idx]) + } else { + g.explicit_slice_active = true + g.explicit_slice_file = item.file_idx + g.gen_file_range(g.files[item.file_idx], item.fn_indices, item.emit_globals) + g.explicit_slice_active = false + } + continue + } + mut sw := time.new_stopwatch() + if item.fn_indices.len == 0 { + g.gen_file(g.files[item.file_idx]) + } else { + g.explicit_slice_active = true + g.explicit_slice_file = item.file_idx + g.gen_file_range(g.files[item.file_idx], item.fn_indices, item.emit_globals) + g.explicit_slice_active = false + } + elapsed_ms := sw.elapsed().milliseconds() + if elapsed_ms > 0 { + suffix := if item.fn_indices.len == 0 { '' } else { ' [${item.fn_indices.len}fns]' } + g.pass5_file_times << Pass5FileTime{ + file: g.files[item.file_idx].name + suffix + ms: elapsed_ms + cost: item.cost + worker_id: g.pass5_worker_id + } + } + } +} + // gen_pass5_files generates function bodies for a range of file indices. // Used by parallel dispatch — each worker calls this with its assigned chunk. pub fn (mut g Gen) gen_pass5_files(file_indices []int) { @@ -2592,10 +2789,14 @@ pub fn (g &Gen) new_pass5_worker(file_indices []int, worker_id int) &Gen { } } return &Gen{ - files: g.files - env: unsafe { g.env } - pref: unsafe { g.pref } - sb: strings.new_builder(64_000) + files: g.files + env: unsafe { g.env } + pref: unsafe { g.pref } + imported_symbols_index: g.imported_symbols_index.clone() + v_method_return_index: g.v_method_return_index.clone() + ierror_base_index: g.ierror_base_index.clone() + fn_owner_file: g.fn_owner_file.clone() + sb: strings.new_builder(64_000) // Read-only lookup maps — clone to avoid COW data races fn_param_is_ptr: g.fn_param_is_ptr.clone() fn_param_types: g.fn_param_types.clone() diff --git a/vlib/v2/gen/cleanc/expr.v b/vlib/v2/gen/cleanc/expr.v index 129798450..ecb63824d 100644 --- a/vlib/v2/gen/cleanc/expr.v +++ b/vlib/v2/gen/cleanc/expr.v @@ -474,6 +474,34 @@ fn (mut g Gen) ierror_concrete_base_for_expr(value_expr ast.Expr) string { value_expr).trim_right('*')) } +// build_ierror_base_index indexes fn_return_types by the `base` of every `X__base__msg` function, +// storing the smallest matching `X__base` per base (the original scan sorted the keys and took the +// first). fn_return_types is final after collect_fn_signatures_to_fixed_point, so this builds once. +fn (mut g Gen) build_ierror_base_index() { + mut idx := map[string]string{} + for fn_name, _ in g.fn_return_types { + if !fn_name.ends_with('__msg') { + continue + } + stripped := fn_name[..fn_name.len - '__msg'.len] // X__base + if !stripped.contains('__') { + continue // need a real `__base` suffix, matching ends_with('__${base}__msg') + } + base := stripped.all_after_last('__') + if base == '' { + continue + } + if existing := idx[base] { + if stripped < existing { + idx[base] = stripped + } + } else { + idx[base] = stripped + } + } + g.ierror_base_index = idx.move() +} + fn (g &Gen) qualify_ierror_concrete_base(base string) string { normalized_base := g.normalize_builtin_qualified_c_type(base) if normalized_base != base { @@ -488,13 +516,11 @@ fn (g &Gen) qualify_ierror_concrete_base(base string) string { return qualified } } - suffix := '__${base}__msg' - mut fn_names := g.fn_return_types.keys() - fn_names.sort() - for fn_name in fn_names { - if fn_name.ends_with(suffix) { - return fn_name[..fn_name.len - '__msg'.len] - } + // Index lookup replaces a per-call `fn_return_types.keys().sort()` + scan. The index stores, + // per base, the smallest `X__base` (the original returned the first sorted `*__base__msg` + // minus the `__msg` suffix). + if qualified := g.ierror_base_index[base] { + return qualified } body_suffix := '__${base}' mut body_keys := g.emitted_types.keys() diff --git a/vlib/v2/gen/cleanc/fn.v b/vlib/v2/gen/cleanc/fn.v index 435a54234..89ce46742 100644 --- a/vlib/v2/gen/cleanc/fn.v +++ b/vlib/v2/gen/cleanc/fn.v @@ -729,7 +729,8 @@ fn (mut g Gen) register_fn_signature(node ast.FnDecl, fn_name string) { 'void' } ret_type = normalize_signature_type_name(ret_type, 'void') - if ret_type == 'int' { + if ret_type == 'int' && node.typ.return_type !is ast.EmptyExpr + && expr_has_generic_placeholder(node.typ.return_type) { inferred := g.infer_vector_return_type_from_stmts(node.stmts) if inferred != '' { ret_type = inferred @@ -5787,7 +5788,7 @@ fn (mut g Gen) gen_fn_decl_with_name_ptr(node &ast.FnDecl, fn_name string) { return } fn_key := 'fn_${fn_name}' - if fn_key in g.blocked_fn_keys { + if fn_key in g.blocked_fn_keys && !g.explicit_slice_emit_allows(fn_key) { return } if g.should_skip_plain_v_fallback_fn(fn_key) { @@ -9169,10 +9170,47 @@ fn (g &Gen) knows_fn_signature(fn_name string) bool { || g.has_specialized_fn_base(fn_name) } +// v_method_ret_ambiguous marks an index entry whose method short-name maps to >1 distinct V +// return type (the original scan returned none for that case). A control char can't be a real type. +const v_method_ret_ambiguous = '\x02' + +// build_v_method_return_index indexes v_fn_return_types by method short-name (all_after_last('__')), +// replacing the per-call scan in unique_v_method_return_type. v_fn_return_types is final after +// collect_fn_signatures_to_fixed_point, so this is built once there. +fn (mut g Gen) build_v_method_return_index() { + mut idx := map[string]string{} + for fn_name, ret in g.v_fn_return_types { + m := fn_name.all_after_last('__') + if m == '' { + continue + } + if existing := idx[m] { + if existing != ret && existing != v_method_ret_ambiguous { + idx[m] = v_method_ret_ambiguous + } + } else { + idx[m] = ret + } + } + g.v_method_return_index = idx.move() +} + fn (g &Gen) unique_v_method_return_type(method_name string) ?string { if method_name == '' { return none } + // Fast path: index lookup. The scan matched fn_name == method_name OR + // fn_name.ends_with('__${method_name}') (ends_with('___...') is subsumed), which for a + // method without '__' is exactly all_after_last('__') == method_name. + if !method_name.contains('__') { + if ret := g.v_method_return_index[method_name] { + if ret != '' && ret != v_method_ret_ambiguous { + return ret + } + } + return none + } + // Rare fallback for method names containing '__'. mut found_name := '' mut found_ret := '' for fn_name, ret in g.v_fn_return_types { @@ -9266,13 +9304,26 @@ fn (mut g Gen) resolve_specialized_receiver_method(receiver_type string, method_ if g.specialized_fn_bases.len > 0 && receiver_type !in g.specialized_fn_bases { return none } - key := specialized_receiver_method_key(receiver_type, method_name) - if key in g.specialized_receiver_method_ambiguous || key in g.specialized_receiver_method_miss { + // Fast path: consult the incremental (base|method -> specialized fn) index that + // remember_specialized_fn_base maintains on every registration. It replaces a per-key + // linear scan over the whole fn_return_types + fn_param_is_ptr tables and, unlike a + // once-built snapshot, stays correct when a second specialization is registered after the + // first lookup (which turns a previously unambiguous (base, method) ambiguous). The scan it + // replaces matched `candidate.starts_with('${receiver_type}_T_') && + // candidate.ends_with('__${method_name}')`, which for a base receiver_type (no `_T_`) and a + // simple method (no `__`) is exactly (all_before('_T_') == base) && (all_after_last('__') == + // method) — i.e. the same keys remember_specialized_receiver_method records. + if !receiver_type.contains('_T_') && !method_name.contains('__') { + key := specialized_receiver_method_key(receiver_type, method_name) + if key in g.specialized_receiver_method_ambiguous { + return none + } + if found := g.specialized_receiver_methods[key] { + return found + } return none } - if found := g.specialized_receiver_methods[key] { - return found - } + // Rare fallback: receiver_type/method contains the `_T_`/`__` separator the index keys on. prefix := '${receiver_type}_T_' suffix := '__${method_name}' mut found := '' @@ -9293,10 +9344,8 @@ fn (mut g Gen) resolve_specialized_receiver_method(receiver_type string, method_ } } if found != '' { - g.specialized_receiver_methods[key] = found return found } - g.specialized_receiver_method_miss[key] = true return none } diff --git a/vlib/v2/gen/cleanc/pass5_worker_test.v b/vlib/v2/gen/cleanc/pass5_worker_test.v index 874cdc6d5..29d80af95 100644 --- a/vlib/v2/gen/cleanc/pass5_worker_test.v +++ b/vlib/v2/gen/cleanc/pass5_worker_test.v @@ -24,3 +24,31 @@ fn test_specialized_receiver_method_index_tracks_ambiguity() { assert true } } + +// When a file is split across pass-5 workers, a non-owning worker blocks the +// file's fns but must still emit its own assigned slice. The owner-scoped bypass +// allows exactly the fns owned by the slice's file (explicit_slice_file) and +// nothing transitively reached from another file. +fn test_explicit_slice_emit_allows_is_scoped_to_owning_file() { + mut g := Gen.new([]) + g.fn_owner_file['fn_foo'] = 3 // foo is owned by file 3 + g.fn_owner_file['fn_bar'] = 5 // bar is owned by a different file + g.blocked_fn_keys['fn_foo'] = true + g.blocked_fn_keys['fn_bar'] = true + + // Not in an explicit slice: nothing is unblocked. + assert !g.explicit_slice_emit_allows('fn_foo') + assert !g.explicit_slice_emit_allows('fn_bar') + + // Emitting file 3's slice: only file 3's fns are unblocked. + g.explicit_slice_active = true + g.explicit_slice_file = 3 + assert g.explicit_slice_emit_allows('fn_foo') + assert !g.explicit_slice_emit_allows('fn_bar') // owned by file 5 -> stays blocked + assert !g.explicit_slice_emit_allows('fn_unknown') // no recorded owner -> blocked + + // Switching to file 5's slice flips which fn is allowed. + g.explicit_slice_file = 5 + assert !g.explicit_slice_emit_allows('fn_foo') + assert g.explicit_slice_emit_allows('fn_bar') +} diff --git a/vlib/v2/gen/cleanc/types.v b/vlib/v2/gen/cleanc/types.v index 7df5dd452..b8abeec51 100644 --- a/vlib/v2/gen/cleanc/types.v +++ b/vlib/v2/gen/cleanc/types.v @@ -5164,22 +5164,11 @@ fn (g &Gen) imported_symbol_c_type(name string) ?string { } symbol_name = name[prefix.len..] } - for file in g.files { - if file.name != g.cur_file_name { - continue - } - for import_stmt in file.imports { - for symbol in import_stmt.symbols { - if symbol.name() != symbol_name { - continue - } - mod_name := import_stmt.name.all_after_last('.').replace('.', '_') - if mod_name == '' || mod_name == g.cur_module { - return none - } - return '${mod_name}__${symbol_name}' - } + if mod_name := g.imported_symbols_index['${g.cur_file_name}\x01${symbol_name}'] { + if mod_name == '' || mod_name == g.cur_module { + return none } + return '${mod_name}__${symbol_name}' } return none } diff --git a/vlib/v2/markused/markused.v b/vlib/v2/markused/markused.v index 1978ad2c9..2808a6813 100644 --- a/vlib/v2/markused/markused.v +++ b/vlib/v2/markused/markused.v @@ -106,19 +106,20 @@ mut: used_keys map[string]bool - module_names map[string]bool - module_alias_to_real map[string]string - type_names map[string]bool - interface_type_names map[string]bool - interface_method_names map[string][]string - interface_embedded_names map[string][]string - methods_by_receiver map[string][]int - struct_field_receivers map[string][]string - struct_embedded_receivers map[string][]string - struct_fn_fields map[string]bool - global_interface_names map[string]string - const_fn_value_aliases map[string]string - selective_import_fn_targets map[string]string + module_names map[string]bool + module_alias_to_real map[string]string + type_names map[string]bool + interface_type_names map[string]bool + interface_method_names map[string][]string + interface_embedded_names map[string][]string + methods_by_receiver map[string][]int + struct_field_receivers map[string][]string + struct_embedded_receivers map[string][]string + struct_fn_fields map[string]bool + global_interface_names map[string]string + const_fn_value_aliases map[string]string + selective_import_fn_targets map[string]string + called_fn_name_candidate_cache map[string][]string lookup map[string][]int @@ -156,24 +157,25 @@ pub fn decl_key(module_name string, decl ast.FnDecl, env &types.Environment) str fn new_walker(files []ast.File, env &types.Environment, opts MarkUsedOptions) Walker { return Walker{ - files: files - env: unsafe { env } - opts: opts - used_keys: map[string]bool{} - queued_fn_indices: map[int]bool{} - module_names: map[string]bool{} - type_names: map[string]bool{} - interface_type_names: map[string]bool{} - interface_method_names: map[string][]string{} - interface_embedded_names: map[string][]string{} - methods_by_receiver: map[string][]int{} - struct_field_receivers: map[string][]string{} - struct_embedded_receivers: map[string][]string{} - struct_fn_fields: map[string]bool{} - global_interface_names: map[string]string{} - const_fn_value_aliases: map[string]string{} - selective_import_fn_targets: map[string]string{} - lookup: map[string][]int{} + files: files + env: unsafe { env } + opts: opts + used_keys: map[string]bool{} + queued_fn_indices: map[int]bool{} + module_names: map[string]bool{} + type_names: map[string]bool{} + interface_type_names: map[string]bool{} + interface_method_names: map[string][]string{} + interface_embedded_names: map[string][]string{} + methods_by_receiver: map[string][]int{} + struct_field_receivers: map[string][]string{} + struct_embedded_receivers: map[string][]string{} + struct_fn_fields: map[string]bool{} + global_interface_names: map[string]string{} + const_fn_value_aliases: map[string]string{} + selective_import_fn_targets: map[string]string{} + called_fn_name_candidate_cache: map[string][]string{} + lookup: map[string][]int{} } } @@ -2170,8 +2172,8 @@ fn (w &Walker) lookup_count(key string) int { if !string_ok(key) { return 0 } - if key in w.lookup { - return w.lookup[key].len + if values := w.lookup[key] { + return values.len } return 0 } @@ -2180,8 +2182,8 @@ fn (mut w Walker) mark_lookup(key string) { if !string_ok(key) { return } - if key in w.lookup { - for idx in w.lookup[key] { + if values := w.lookup[key] { + for idx in values { w.mark_fn(idx) } } @@ -2223,6 +2225,15 @@ fn called_fn_name_candidates(name string) []string { return out } +fn (mut w Walker) cached_called_fn_name_candidates(name string) []string { + if candidates := w.called_fn_name_candidate_cache[name] { + return candidates + } + candidates := called_fn_name_candidates(name) + w.called_fn_name_candidate_cache[name] = candidates + return candidates +} + fn strip_generic_specialization_suffix(name string) string { if idx := name.index('_T_') { if idx > 0 { @@ -2242,7 +2253,7 @@ fn should_mark_ident_as_fn(name string) bool { || name.starts_with('_result_') } -fn (w &Walker) ident_resolves_to_fn_value(name string, mod_name string) bool { +fn (mut w Walker) ident_resolves_to_fn_value(name string, mod_name string) bool { if name == '' || !string_ok(name) || name == 'C' || name in w.module_names || w.is_cast_type_name(name) { return false @@ -2252,7 +2263,7 @@ fn (w &Walker) ident_resolves_to_fn_value(name string, mod_name string) bool { return obj is types.Fn } } - candidates := called_fn_name_candidates(name) + candidates := w.cached_called_fn_name_candidates(name) if _ := w.exact_mangled_fn_lookup_key(name) { return true } @@ -2318,7 +2329,7 @@ fn (mut w Walker) mark_fn_name(name string, mod_name string) { if w.opts.minimal_runtime_roots && name in ['array__eq', 'builtin__array__eq'] { w.mark_fn_name('map_map_eq', 'builtin') } - candidates := called_fn_name_candidates(name) + candidates := w.cached_called_fn_name_candidates(name) if key := w.exact_mangled_fn_lookup_key(name) { w.mark_lookup(key) return @@ -2354,12 +2365,7 @@ fn (mut w Walker) mark_method_name(name string, receivers []string) { if !string_ok(receiver) { continue } - for candidate in receiver_lookup_candidates(receiver) { - w.mark_lookup('meth:${candidate}:${name}') - if normalized != name { - w.mark_lookup('meth:${candidate}:${normalized}') - } - } + w.mark_method_receiver_candidate(receiver, name, normalized) } } @@ -2373,38 +2379,87 @@ fn (w &Walker) method_lookup_count(name string, receivers []string) int { if !string_ok(receiver) { continue } - for candidate in receiver_lookup_candidates(receiver) { - count += w.lookup_count('meth:${candidate}:${name}') - if normalized != name { - count += w.lookup_count('meth:${candidate}:${normalized}') - } - } + count += w.method_receiver_candidate_count(receiver, name, normalized) } return count } -fn receiver_lookup_candidates(receiver string) []string { - mut out := []string{} - if !string_ok(receiver) { - return out +fn (mut w Walker) mark_method_lookup_candidate(candidate string, name string, normalized string) { + w.mark_lookup('meth:${candidate}:${name}') + if normalized != name { + w.mark_lookup('meth:${candidate}:${normalized}') } - add_unique_string(mut out, receiver) +} + +fn (mut w Walker) mark_method_receiver_candidate(receiver string, name string, normalized string) { + w.mark_method_lookup_candidate(receiver, name, normalized) + mut short_name := '' if receiver.contains('__') { - add_unique_string(mut out, receiver.all_after_last('__')) + short_name = receiver.all_after_last('__') + if short_name != '' && short_name != receiver { + w.mark_method_lookup_candidate(short_name, name, normalized) + } } if receiver.starts_with('Array_') || receiver.starts_with('Array_fixed_') { - add_unique_string(mut out, 'array') + if receiver != 'array' && short_name != 'array' { + w.mark_method_lookup_candidate('array', name, normalized) + } } if receiver.starts_with('Map_') { - add_unique_string(mut out, 'map') + if receiver != 'map' && short_name != 'map' { + w.mark_method_lookup_candidate('map', name, normalized) + } } if receiver.starts_with('_option_') { - add_unique_string(mut out, '_option') + if receiver != '_option' && short_name != '_option' { + w.mark_method_lookup_candidate('_option', name, normalized) + } } if receiver.starts_with('_result_') { - add_unique_string(mut out, '_result') + if receiver != '_result' && short_name != '_result' { + w.mark_method_lookup_candidate('_result', name, normalized) + } } - return out +} + +fn (w &Walker) method_lookup_candidate_count(candidate string, name string, normalized string) int { + mut count := w.lookup_count('meth:${candidate}:${name}') + if normalized != name { + count += w.lookup_count('meth:${candidate}:${normalized}') + } + return count +} + +fn (w &Walker) method_receiver_candidate_count(receiver string, name string, normalized string) int { + mut count := w.method_lookup_candidate_count(receiver, name, normalized) + mut short_name := '' + if receiver.contains('__') { + short_name = receiver.all_after_last('__') + if short_name != '' && short_name != receiver { + count += w.method_lookup_candidate_count(short_name, name, normalized) + } + } + if receiver.starts_with('Array_') || receiver.starts_with('Array_fixed_') { + if receiver != 'array' && short_name != 'array' { + count += w.method_lookup_candidate_count('array', name, normalized) + } + } + if receiver.starts_with('Map_') { + if receiver != 'map' && short_name != 'map' { + count += w.method_lookup_candidate_count('map', name, normalized) + } + } + if receiver.starts_with('_option_') { + if receiver != '_option' && short_name != '_option' { + count += w.method_lookup_candidate_count('_option', name, normalized) + } + } + if receiver.starts_with('_result_') { + if receiver != '_result' && short_name != '_result' { + count += w.method_lookup_candidate_count('_result', name, normalized) + } + } + return count } fn (mut w Walker) mark_method_name_fallback(name string) { @@ -2905,18 +2960,18 @@ fn (w &Walker) add_lookup_indices(key string, mut out []int) { if !string_ok(key) { return } - if key in w.lookup { - for idx in w.lookup[key] { + if values := w.lookup[key] { + for idx in values { add_unique_int(mut out, idx) } } } -fn (w &Walker) add_fn_name_indices(name string, mod_name string, mut out []int) { +fn (mut w Walker) add_fn_name_indices(name string, mod_name string, mut out []int) { if name == '' || !string_ok(name) || !string_ok(mod_name) { return } - candidates := called_fn_name_candidates(name) + candidates := w.cached_called_fn_name_candidates(name) if key := w.exact_mangled_fn_lookup_key(name) { w.add_lookup_indices(key, mut out) return @@ -2952,16 +3007,49 @@ fn (w &Walker) add_method_name_indices(name string, receivers []string, mut out if !string_ok(receiver) { continue } - for candidate in receiver_lookup_candidates(receiver) { - w.add_lookup_indices('meth:${candidate}:${name}', mut out) - if normalized != name { - w.add_lookup_indices('meth:${candidate}:${normalized}', mut out) - } + w.add_method_receiver_candidate_indices(receiver, name, normalized, mut out) + } +} + +fn (w &Walker) add_method_candidate_indices(candidate string, name string, normalized string, mut out []int) { + w.add_lookup_indices('meth:${candidate}:${name}', mut out) + if normalized != name { + w.add_lookup_indices('meth:${candidate}:${normalized}', mut out) + } +} + +fn (w &Walker) add_method_receiver_candidate_indices(receiver string, name string, normalized string, mut out []int) { + w.add_method_candidate_indices(receiver, name, normalized, mut out) + mut short_name := '' + if receiver.contains('__') { + short_name = receiver.all_after_last('__') + if short_name != '' && short_name != receiver { + w.add_method_candidate_indices(short_name, name, normalized, mut out) + } + } + if receiver.starts_with('Array_') || receiver.starts_with('Array_fixed_') { + if receiver != 'array' && short_name != 'array' { + w.add_method_candidate_indices('array', name, normalized, mut out) + } + } + if receiver.starts_with('Map_') { + if receiver != 'map' && short_name != 'map' { + w.add_method_candidate_indices('map', name, normalized, mut out) + } + } + if receiver.starts_with('_option_') { + if receiver != '_option' && short_name != '_option' { + w.add_method_candidate_indices('_option', name, normalized, mut out) + } + } + if receiver.starts_with('_result_') { + if receiver != '_result' && short_name != '_result' { + w.add_method_candidate_indices('_result', name, normalized, mut out) } } } -fn (w &Walker) call_lhs_decl_indices(lhs ast.Expr, mod_name string) []int { +fn (mut w Walker) call_lhs_decl_indices(lhs ast.Expr, mod_name string) []int { mut out := []int{} if !expr_ok(lhs) { return out @@ -3140,7 +3228,7 @@ fn (w &Walker) call_lhs_fn_type_cursor(c ast.Cursor, mod_name string) ?types.FnT // input. Same shape: Ident / GenericArgs / GenericArgOrIndexExpr / Selector // — the only ast-decoded subtree is lhs.lhs for SelectorExpr's // receiver_candidates_for_expr fallback. -fn (w &Walker) call_lhs_decl_indices_cursor(c ast.Cursor, mod_name string) []int { +fn (mut w Walker) call_lhs_decl_indices_cursor(c ast.Cursor, mod_name string) []int { mut out := []int{} if !c.is_valid() { return out diff --git a/vlib/v2/transformer/fn.v b/vlib/v2/transformer/fn.v index 7b1335808..76eda6ab1 100644 --- a/vlib/v2/transformer/fn.v +++ b/vlib/v2/transformer/fn.v @@ -1174,23 +1174,36 @@ fn (t &Transformer) generic_call_concrete_return_type(expr ast.Expr) ?types.Type } fn (t &Transformer) append_method_lookup_type_name(mut names []string, raw_name string) { - if raw_name.len == 0 || !transformer_string_has_valid_data(raw_name) { + normalized := normalized_method_lookup_type_name(raw_name) + if normalized == '' { return } - mut normalized := raw_name.replace('.', '__') + names << normalized + dunder := last_double_underscore(normalized) + if dunder >= 0 { + short_name := normalized[dunder + 2..] + if short_name != '' && short_name != normalized { + names << short_name + } + } +} + +fn normalized_method_lookup_type_name(raw_name string) string { + if raw_name.len == 0 || !transformer_string_has_valid_data(raw_name) { + return '' + } + mut normalized := if raw_name.index_u8(`.`) >= 0 { + raw_name.replace('.', '__') + } else { + raw_name + } if normalized.starts_with('&') { normalized = normalized[1..] } if normalized.ends_with('*') { normalized = normalized[..normalized.len - 1] } - if normalized == '' { - return - } - names << normalized - if normalized.contains('__') { - names << normalized.all_after_last('__') - } + return normalized } fn transformer_string_has_valid_data(s string) bool { @@ -1417,13 +1430,26 @@ fn (t &Transformer) lookup_method_exists(type_names []string, method_name string } fn (t &Transformer) type_has_cached_method(typ types.Type, method_name string) bool { - mut lookup_names := []string{} - t.append_method_lookup_type_name(mut lookup_names, typ.name()) + if t.type_name_has_cached_method(typ.name(), method_name) { + return true + } base_type := t.unwrap_alias_and_pointer_type(typ) - t.append_method_lookup_type_name(mut lookup_names, base_type.name()) - for name in lookup_names { - if _ := t.lookup_method_cached(name, method_name) { - return true + return t.type_name_has_cached_method(base_type.name(), method_name) +} + +fn (t &Transformer) type_name_has_cached_method(raw_name string, method_name string) bool { + normalized := normalized_method_lookup_type_name(raw_name) + if normalized == '' { + return false + } + if _ := t.lookup_method_cached(normalized, method_name) { + return true + } + dunder := last_double_underscore(normalized) + if dunder >= 0 { + short_name := normalized[dunder + 2..] + if short_name != '' && short_name != normalized { + return t.lookup_method_cached(short_name, method_name) != none } } return false @@ -1433,15 +1459,24 @@ fn (t &Transformer) receiver_has_cached_method(receiver ast.Expr, method_name st if typ := t.get_expr_type(receiver) { return t.type_has_cached_method(typ, method_name) } - mut lookup_names := []string{} if receiver is ast.SelectorExpr { selector_type_name := t.get_selector_type_name(receiver) + if t.type_name_has_cached_method(selector_type_name, method_name) { + return true + } + mut lookup_names := []string{} t.append_method_lookup_type_name(mut lookup_names, selector_type_name) + return t.lookup_method_exists(lookup_names, method_name) } else if receiver is ast.Ident { var_type_name := t.get_var_type_name(receiver.name) + if t.type_name_has_cached_method(var_type_name, method_name) { + return true + } + mut lookup_names := []string{} t.append_method_lookup_type_name(mut lookup_names, var_type_name) + return t.lookup_method_exists(lookup_names, method_name) } - return t.lookup_method_exists(lookup_names, method_name) + return false } fn (t &Transformer) smartcast_source_has_cached_method(ctx SmartcastContext, method_name string) bool { @@ -1451,6 +1486,9 @@ fn (t &Transformer) smartcast_source_has_cached_method(ctx SmartcastContext, met if typ := t.c_name_to_type(ctx.sumtype) { return t.type_has_cached_method(typ, method_name) } + if t.type_name_has_cached_method(ctx.sumtype, method_name) { + return true + } mut lookup_names := []string{} t.append_method_lookup_type_name(mut lookup_names, ctx.sumtype) return t.lookup_method_exists(lookup_names, method_name) @@ -4343,8 +4381,9 @@ fn (t &Transformer) resolve_explicit_cast_method_name(receiver_type_name string, if t.lookup_method_cached(receiver_type_name, method_name) != none { return '${receiver_type_name}__${method_name}' } - if receiver_type_name.contains('__') { - short_name := receiver_type_name.all_after_last('__') + dunder := last_double_underscore(receiver_type_name) + if dunder >= 0 { + short_name := receiver_type_name[dunder + 2..] if t.lookup_method_cached(short_name, method_name) != none { return '${short_name}__${method_name}' } @@ -4368,22 +4407,21 @@ fn (t &Transformer) exact_method_owner_name(raw_name string, method_name string) if raw_name == '' { return none } - mut normalized := raw_name.replace('.', '__') - if normalized.starts_with('&') { - normalized = normalized[1..] - } - if normalized.ends_with('*') { - normalized = normalized[..normalized.len - 1] - } + normalized := normalized_method_lookup_type_name(raw_name) if normalized == '' { return none } if t.lookup_method_cached(normalized, method_name) != none { return normalized } - if !normalized.contains('__') && t.cur_module != '' && t.cur_module != 'main' + if last_double_underscore(normalized) < 0 && t.cur_module != '' && t.cur_module != 'main' && t.cur_module != 'builtin' { - qualified := '${t.cur_module.replace('.', '__')}__${normalized}' + qualified_mod := if t.cur_module.index_u8(`.`) >= 0 { + t.cur_module.replace('.', '__') + } else { + t.cur_module + } + qualified := '${qualified_mod}__${normalized}' if t.lookup_method_cached(qualified, method_name) != none { return qualified } @@ -4534,13 +4572,11 @@ fn (t &Transformer) resolve_method_call_name(receiver ast.Expr, method_name stri } else if c_prefix == 'string' && type_name != 'string' && 'string' !in lookup_names { lookup_names << 'string' } - if method_name.contains('_T_') || method_name.ends_with('_T') { - base_method_name := generic_base_name_without_specialization(method_name) - if base_method_name != method_name { - for name in lookup_names { - if t.lookup_method_cached(name, base_method_name) != none { - return '${c_prefix}__${method_name}' - } + base_method_name := generic_base_name_without_specialization(method_name) + if base_method_name != method_name { + for name in lookup_names { + if t.lookup_method_cached(name, base_method_name) != none { + return '${c_prefix}__${method_name}' } } } @@ -4691,10 +4727,12 @@ fn (t &Transformer) specific_array_method_c_name(receiver ast.Expr, method_name if c_name == '' { return none } - for lookup_name in [base_type.name(), c_name] { - if t.lookup_method_cached(lookup_name, method_name) != none { - return '${c_name}__${method_name}' - } + base_name := base_type.name() + if t.lookup_method_cached(base_name, method_name) != none { + return '${c_name}__${method_name}' + } + if c_name != base_name && t.lookup_method_cached(c_name, method_name) != none { + return '${c_name}__${method_name}' } } else {} diff --git a/vlib/v2/transformer/struct.v b/vlib/v2/transformer/struct.v index 86b04ec3d..515f9f91e 100644 --- a/vlib/v2/transformer/struct.v +++ b/vlib/v2/transformer/struct.v @@ -84,6 +84,36 @@ fn (t &Transformer) lookup_struct_field_type(struct_name string, field_name stri if dunder >= 0 { mod = struct_name[..dunder] sname = struct_name[dunder + 2..] + if field_type := t.cached_struct_field_types[struct_field_lookup_cache_key(mod, sname, + field_name)] + { + return field_type + } + if field_type := t.cached_struct_field_types[struct_field_generic_decl_key(struct_name, + field_name)] + { + return field_type + } + } else if dot := struct_name.last_index('.') { + mod = struct_name[..dot] + sname = struct_name[dot + 1..] + if field_type := t.cached_struct_field_types[struct_field_lookup_cache_key(mod, sname, + field_name)] + { + return field_type + } + dot_name := struct_name.replace('.', '__') + if field_type := t.cached_struct_field_types[struct_field_generic_decl_key(dot_name, + field_name)] + { + return field_type + } + } else if t.cur_module != '' { + if field_type := t.cached_struct_field_types[struct_field_lookup_cache_key(t.cur_module, + sname, field_name)] + { + return field_type + } } // Try module scope first if qualified if mod != '' { @@ -115,8 +145,7 @@ fn (t &Transformer) lookup_struct_field_type(struct_name string, field_name stri } } // Fallback: scan all scopes - scope_keys := t.cached_scopes.keys() - for sk in scope_keys { + for sk in t.cached_scope_keys { scope := t.cached_scopes[sk] or { continue } if obj := scope.objects[struct_name] { if typ := transformer_object_type(obj) { @@ -146,30 +175,84 @@ fn (t &Transformer) lookup_struct_field_generic_decl_type(struct_name string, fi if struct_name == '' || field_name == '' { return none } - for candidate in struct_field_lookup_candidates(struct_name) { - if field_type := t.struct_field_generic_decl_types[struct_field_generic_decl_key(candidate, - field_name)] - { - return field_type + if field_type := t.struct_field_generic_decl_type_for_candidate(struct_name, field_name) { + return field_type + } + if struct_name.contains('__') { + short_name := struct_name.all_after_last('__') + if short_name != '' && short_name != struct_name { + if field_type := t.struct_field_generic_decl_type_for_candidate(short_name, field_name) { + return field_type + } + } + } + if struct_name.index_u8(`.`) >= 0 { + dot_name := struct_name.replace('.', '__') + if dot_name != struct_name { + if field_type := t.struct_field_generic_decl_type_for_candidate(dot_name, field_name) { + return field_type + } + } + short_name := struct_name.all_after_last('.') + if short_name != '' && short_name != struct_name && short_name != dot_name { + if field_type := t.struct_field_generic_decl_type_for_candidate(short_name, field_name) { + return field_type + } } } return none } +fn (t &Transformer) struct_field_generic_decl_type_for_candidate(struct_name string, field_name string) ?types.Type { + key := struct_field_generic_decl_key(struct_name, field_name) + field_type := t.struct_field_generic_decl_types[key] or { return none } + return field_type +} + fn (t &Transformer) lookup_struct_field_generic_decl_bindings(struct_name string, field_name string) ?map[string]types.Type { if struct_name == '' || field_name == '' { return none } - for candidate in struct_field_lookup_candidates(struct_name) { - if bindings := t.struct_field_generic_decl_bindings[struct_field_generic_decl_key(candidate, - field_name)] - { - return bindings.clone() + if bindings := t.struct_field_generic_decl_bindings_for_candidate(struct_name, field_name) { + return bindings + } + if struct_name.contains('__') { + short_name := struct_name.all_after_last('__') + if short_name != '' && short_name != struct_name { + if bindings := t.struct_field_generic_decl_bindings_for_candidate(short_name, + field_name) + { + return bindings + } + } + } + if struct_name.index_u8(`.`) >= 0 { + dot_name := struct_name.replace('.', '__') + if dot_name != struct_name { + if bindings := t.struct_field_generic_decl_bindings_for_candidate(dot_name, field_name) { + return bindings + } + } + short_name := struct_name.all_after_last('.') + if short_name != '' && short_name != struct_name && short_name != dot_name { + if bindings := t.struct_field_generic_decl_bindings_for_candidate(short_name, + field_name) + { + return bindings + } } } return none } +fn (t &Transformer) struct_field_generic_decl_bindings_for_candidate(struct_name string, field_name string) ?map[string]types.Type { + key := struct_field_generic_decl_key(struct_name, field_name) + if bindings := t.struct_field_generic_decl_bindings[key] { + return bindings.clone() + } + return none +} + fn struct_field_lookup_candidates(struct_name string) []string { mut candidates := []string{} if struct_name != '' { diff --git a/vlib/v2/transformer/transformer.v b/vlib/v2/transformer/transformer.v index 95ce4564e..7c5fd20bb 100644 --- a/vlib/v2/transformer/transformer.v +++ b/vlib/v2/transformer/transformer.v @@ -121,12 +121,15 @@ mut: live_source_file string // Cached scope/method/fn_scope snapshots for lock-free parallel access. // Populated once in pre_pass from the shared Environment fields. - cached_scopes map[string]&types.Scope - cached_methods map[string][]&types.Fn - cached_method_keys []string - cached_fn_scopes map[string]&types.Scope - cached_fn_type_index map[string]types.Type - cached_fn_return_type_index map[string]types.Type + cached_scopes map[string]&types.Scope + cached_scope_keys []string + cached_methods map[string][]&types.Fn + cached_method_keys []string + cached_fn_scopes map[string]&types.Scope + cached_fn_type_index map[string]types.Type + cached_fn_return_type_index map[string]types.Type + cached_imported_module_scopes map[string][]&types.Scope + cached_struct_field_types map[string]types.Type // cached_method_base_index maps type_name -> generic-base method name -> // FnType, precomputed once from cached_methods. lookup_method_cached used to // linearly scan every method of a type and recompute its base name (via @@ -394,12 +397,15 @@ pub fn (t &Transformer) new_worker_clone(worker_idx int) &Transformer { comptime_vmodroot: t.comptime_vmodroot file_set: unsafe { t.file_set } cached_scopes: t.cached_scopes.clone() - cached_methods: t.cached_methods.clone() - cached_fn_type_index: t.cached_fn_type_index.clone() - cached_fn_return_type_index: t.cached_fn_return_type_index.clone() - cached_method_base_index: t.cached_method_base_index.clone() - cached_method_keys_by_short: t.cached_method_keys_by_short.clone() - cached_method_keys: t.cached_method_keys.clone() + cached_scope_keys: t.cached_scope_keys + cached_methods: t.cached_methods + cached_fn_type_index: t.cached_fn_type_index + cached_fn_return_type_index: t.cached_fn_return_type_index + cached_imported_module_scopes: t.cached_imported_module_scopes + cached_struct_field_types: t.cached_struct_field_types + cached_method_base_index: t.cached_method_base_index + cached_method_keys_by_short: t.cached_method_keys_by_short + cached_method_keys: t.cached_method_keys cached_fn_scopes: t.cached_fn_scopes.clone() synth_types: t.synth_types.clone() synth_pos_counter: t.synth_pos_counter - (worker_idx * 100_000) @@ -1584,9 +1590,12 @@ pub fn (mut t Transformer) pre_pass_from_flat(flat &ast.FlatAst) { // for lock-free access during parallel file transformation. fn (mut t Transformer) cache_env_maps() { t.cached_scopes = t.env.snapshot_scopes() + t.cached_scope_keys = t.cached_scopes.keys() t.cached_methods = t.env.snapshot_methods() t.cached_method_keys = t.cached_methods.keys() t.cached_fn_scopes = t.env.snapshot_fn_scopes() + t.build_cached_imported_module_scopes() + t.build_cached_struct_field_type_index() t.build_cached_fn_type_index() t.build_cached_method_base_index() mut by_short := map[string][]string{} @@ -1596,10 +1605,95 @@ fn (mut t Transformer) cache_env_maps() { t.cached_method_keys_by_short = by_short.move() } +fn (mut t Transformer) build_cached_imported_module_scopes() { + mut by_module := map[string][]&types.Scope{} + for module_name in t.cached_scope_keys { + scope := t.cached_scopes[module_name] or { continue } + mut imported_scopes := []&types.Scope{} + for key, obj in scope.objects { + if obj !is types.Module { + continue + } + module_obj := obj as types.Module + import_name := if module_obj.name != '' { module_obj.name } else { key } + if import_name == '' || import_name == module_name || import_name == 'C' { + continue + } + mut module_scope := module_obj.scope + if module_scope == unsafe { nil } { + module_scope = t.cached_scopes[import_name] or { continue } + } + imported_scopes << module_scope + } + by_module[module_name] = imported_scopes + } + t.cached_imported_module_scopes = by_module.move() +} + +fn (mut t Transformer) build_cached_struct_field_type_index() { + mut index := map[string]types.Type{} + for module_name in t.cached_scope_keys { + scope := t.cached_scopes[module_name] or { continue } + for obj_name, obj in scope.objects { + typ := transformer_object_type(obj) or { continue } + if typ is types.Struct { + add_struct_field_type_index_entries(mut index, module_name, obj_name, typ) + } + } + } + t.cached_struct_field_types = index.move() +} + +fn add_struct_field_type_index_entries(mut index map[string]types.Type, module_name string, obj_name string, st types.Struct) { + mut names := []string{cap: 4} + if obj_name != '' { + names << obj_name + } + if st.name != '' && st.name !in names { + names << st.name + } + if module_name != '' && obj_name != '' && !obj_name.contains('__') { + qualified := '${module_name}__${obj_name}' + if qualified !in names { + names << qualified + } + } + for field in st.fields { + if field.name == '' || !transformer_string_has_valid_data(field.name) { + continue + } + for name in names { + key := struct_field_generic_decl_key(name, field.name) + if key !in index { + index[key] = field.typ + } + } + if module_name != '' && obj_name != '' { + module_key := struct_field_lookup_cache_key(module_name, obj_name, field.name) + if module_key !in index { + index[module_key] = field.typ + } + } + if module_name != '' && st.name.contains('__') { + short_name := st.name.all_after_last('__') + if short_name != '' { + module_key := struct_field_lookup_cache_key(module_name, short_name, field.name) + if module_key !in index { + index[module_key] = field.typ + } + } + } + } +} + +fn struct_field_lookup_cache_key(module_name string, struct_name string, field_name string) string { + return '${module_name}#${struct_name}.${field_name}' +} + fn (mut t Transformer) build_cached_fn_type_index() { mut fn_index := map[string]types.Type{} mut ret_index := map[string]types.Type{} - for module_name in t.cached_scopes.keys() { + for module_name in t.cached_scope_keys { scope := t.cached_scopes[module_name] or { continue } for fn_name, obj in scope.objects { if obj is types.Fn { diff --git a/vlib/v2/transformer/types.v b/vlib/v2/transformer/types.v index 65ac8e8f8..390f11878 100644 --- a/vlib/v2/transformer/types.v +++ b/vlib/v2/transformer/types.v @@ -28,7 +28,16 @@ fn (t &Transformer) get_synth_type(pos token.Pos) ?types.Type { fn (t &Transformer) lookup_method_cached(type_name string, method_name string) ?types.FnType { // O(1) via the precomputed base-name index (built in build_cached_method_base_index). // Equivalent to the old linear scan: match by generic base name, first FnType wins. + if typ := t.cached_method_base_index['${type_name}#${method_name}'] { + if typ is types.FnType { + return typ + } + return none + } base_method_name := generic_base_name_without_specialization(method_name) + if base_method_name == method_name { + return none + } typ := t.cached_method_base_index['${type_name}#${base_method_name}'] or { return none } if typ is types.FnType { return typ @@ -251,25 +260,10 @@ fn (t &Transformer) lookup_imported_var_type(name string) ?types.Type { if name == '' || name.contains('__') || t.cur_module == '' { return none } - current_scope := t.get_module_scope(t.cur_module) or { return none } - mut keys := current_scope.objects.keys() - keys.sort() + imported_scopes := t.cached_imported_module_scopes[t.cur_module] or { return none } mut found_type := types.Type(types.void_) mut found := false - for key in keys { - obj := current_scope.objects[key] or { continue } - if obj !is types.Module { - continue - } - module_obj := obj as types.Module - module_name := if module_obj.name != '' { module_obj.name } else { key } - if module_name == '' || module_name == t.cur_module || module_name == 'C' { - continue - } - mut module_scope := module_obj.scope - if module_scope == unsafe { nil } { - module_scope = t.get_module_scope(module_name) or { continue } - } + for module_scope in imported_scopes { typ := module_scope.lookup_var_type(name) or { continue } if found { return none -- 2.39.5