// Copyright (c) 2019-2024 Alexander Medvednikov. All rights reserved. // Use of this source code is governed by an MIT license // that can be found in the LICENSE file. module parser import os import v.ast import v.token import v.errors import v.util import v.vmod const supported_comptime_calls = ['html', 'tmpl', 'env', 'embed_file', 'pkgconfig', 'compile_error', 'compile_warn', 'd', 'res', 'zero', 'new'] const supported_comptime_for_kinds = ['methods', 'fields', 'values', 'variants', 'attributes', 'params'] const comptime_types = ['map', 'array', 'array_dynamic', 'array_fixed', 'int', 'float', 'struct', 'interface', 'enum', 'sumtype', 'alias', 'function', 'option', 'shared', 'string', 'pointer', 'voidptr'] fn find_veb_template_relative_to_vmod(compiled_vfile_path string, relative_path string) ?string { mut mcache := vmod.get_cache() vmod_file_location := mcache.get_by_file(compiled_vfile_path) if vmod_file_location.vmod_file == '' { return none } vmod_template_path := os.join_path(vmod_file_location.vmod_folder, relative_path) if !os.exists(vmod_template_path) { return none } return os.real_path(vmod_template_path) } fn find_existing_veb_template_in_vmod(compiled_vfile_path string, relative_paths []string) ?string { for relative_path in relative_paths { if path := find_veb_template_relative_to_vmod(compiled_vfile_path, relative_path) { return path } } return none } @[inline] fn is_supported_comptime_for_kind(name string) bool { return name in supported_comptime_for_kinds } fn (mut p Parser) parse_comptime_type() ast.ComptimeType { pos := p.tok.pos() p.check(.dollar) name := p.check_name() if name !in comptime_types { p.error('unsupported compile-time type `${name}`: only ${comptime_types} are supported') } mut kind := ast.ComptimeTypeKind.unknown kind = match name { 'map' { .map } 'struct' { .struct } 'interface' { .iface } 'int' { .int } 'float' { .float } 'alias' { .alias } 'function' { .function } 'array' { .array } 'array_fixed' { .array_fixed } 'array_dynamic' { .array_dynamic } 'enum' { .enum } 'sumtype' { .sum_type } 'option' { .option } 'shared' { .shared } 'string' { .string } 'pointer' { .pointer } 'voidptr' { .voidptr } else { .unknown } } return ast.ComptimeType{ kind: kind pos: pos } } // // #include, #flag, #v fn (mut p Parser) hash() ast.HashStmt { pos := p.tok.pos() val := p.tok.lit kind := val.all_before(' ') attrs := p.attrs p.next() mut main_str := '' mut msg := '' mut ct_low_level_cond := '' content := val.all_after('${kind} ').all_before('//') if content.contains(' #') { main_str = content.all_before(' #').trim_space() msg = content.all_after(' #').trim_space() } else { main_str = content.trim_space() msg = '' } // Detect OS-specific conditions like `#include linux ` or `#include darwin "util.h"`. // The condition is the first word before the actual include path. if kind in ['include', 'preinclude', 'postinclude', 'insert'] { first_space := main_str.index_u8(` `) if first_space > 0 { maybe_cond := main_str[..first_space] if maybe_cond in ast.valid_comptime_not_user_defined { ct_low_level_cond = maybe_cond main_str = main_str[first_space + 1..].trim_space() } } } mut is_use_once := false for fna in attrs { match fna.name { 'use_once' { is_use_once = true } else {} } } return ast.HashStmt{ mod: p.mod source_file: p.file_path val: val kind: kind main: main_str msg: msg pos: pos attrs: attrs is_use_once: is_use_once ct_low_level_cond: ct_low_level_cond } } const error_msg = 'only `\$tmpl()`, `\$env()`, `\$embed_file()`, `\$pkgconfig()`, `\$veb.html()`, `\$compile_error()`, `\$compile_warn()`, `\$d()`, `\$res()`, `\$zero()` and `\$new()` comptime functions are supported right now' fn (p &Parser) resolve_tmpl_path_expr(expr ast.Expr) ?string { return p.resolve_tmpl_path_expr_with_depth(expr, 0) } fn (p &Parser) resolve_tmpl_path_expr_with_depth(expr ast.Expr, depth int) ?string { if depth > 50 { return none } match expr { ast.AtExpr { return expr.name } ast.CallExpr { match expr.name { 'os.join_path' { if expr.args.len == 0 { return '' } mut path := p.resolve_tmpl_path_expr_with_depth(expr.args[0].expr, depth + 1)? for arg in expr.args[1..] { path = os.join_path_single(path, p.resolve_tmpl_path_expr_with_depth(arg.expr, depth + 1)?) } return path } 'os.join_path_single' { if expr.args.len != 2 { return none } return os.join_path_single(p.resolve_tmpl_path_expr_with_depth(expr.args[0].expr, depth + 1)?, p.resolve_tmpl_path_expr_with_depth(expr.args[1].expr, depth + 1)?) } else { return none } } } ast.Ident { if var := p.scope.find_var(expr.name) { return p.resolve_tmpl_path_expr_with_depth(var.expr, depth + 1) } if expr.name == 'os.path_separator' { return os.path_separator } if imported_name := p.imported_symbols[expr.name] { if const_field := p.table.global_scope.find_const(imported_name) { return p.resolve_tmpl_path_expr_with_depth(const_field.expr, depth + 1) } } if const_field := p.table.global_scope.find_const(expr.name) { return p.resolve_tmpl_path_expr_with_depth(const_field.expr, depth + 1) } if const_field := p.table.global_scope.find_const('${expr.mod}.${expr.name}') { return p.resolve_tmpl_path_expr_with_depth(const_field.expr, depth + 1) } if const_field := p.table.global_scope.find_const('${p.mod}.${expr.name}') { return p.resolve_tmpl_path_expr_with_depth(const_field.expr, depth + 1) } return none } ast.InfixExpr { if expr.op != .plus { return none } return p.resolve_tmpl_path_expr_with_depth(expr.left, depth + 1)? + p.resolve_tmpl_path_expr_with_depth(expr.right, depth + 1)? } ast.ParExpr { return p.resolve_tmpl_path_expr_with_depth(expr.expr, depth + 1) } ast.SelectorExpr { module_name := p.resolve_tmpl_path_module_name(expr.expr) or { return none } if module_name == 'os' && expr.field_name == 'path_separator' { return os.path_separator } if const_field := p.table.global_scope.find_const('${module_name}.${expr.field_name}') { return p.resolve_tmpl_path_expr_with_depth(const_field.expr, depth + 1) } return none } ast.StringLiteral { return expr.val } else { return none } } } fn (p &Parser) resolve_tmpl_path_module_name(expr ast.Expr) ?string { match expr { ast.Ident { if module_name := p.imports[expr.name] { return module_name } return expr.name } ast.SelectorExpr { return '${p.resolve_tmpl_path_module_name(expr.expr)?}.${expr.field_name}' } else { return none } } } fn (mut p Parser) resolve_tmpl_pseudo_variables(path string, pos token.Pos) ?string { mut resolved := path source_path := os.real_path(p.scanner.file_path) if resolved.contains('@VEXEROOT') { resolved = resolved.replace('@VEXEROOT', p.pref.vroot) } if resolved.contains('@VMODROOT') { resolved = util.resolve_vmodroot(resolved, source_path) or { p.error_with_pos(err.msg(), pos) return none } } if resolved.contains('@DIR') { resolved = resolved.replace('@DIR', os.dir(source_path)) } return resolved } fn (p &Parser) is_comptime_type_selector_at(offset int) bool { if p.peek_token(offset).kind == .key_typeof { return true } if p.peek_token(offset).kind != .name { return false } mut n := offset + 1 for p.peek_token(n).kind == .dot && p.peek_token(n + 1).kind == .name { if is_array_init_type_expr_field(p.peek_token(n + 1).lit) { return true } n += 2 } return false } fn (p &Parser) is_comptime_type_expr_arg() bool { if p.tok.kind in [.key_typeof, .dollar] { return true } if p.tok.kind == .lsbr && p.peek_tok.kind == .rsbr { return p.is_comptime_type_selector_at(2) } return p.is_comptime_type_selector_at(0) } fn (mut p Parser) comptime_call_type_arg() ast.Expr { arg_pos := p.tok.pos() if p.is_comptime_type_expr_arg() { old_inside_array_init_type_expr := p.inside_array_init_type_expr p.inside_array_init_type_expr = true expr := p.expr(0) p.inside_array_init_type_expr = old_inside_array_init_type_expr return expr } typ := p.parse_type() return ast.TypeNode{ typ: typ pos: arg_pos.extend(p.prev_tok.pos()) } } fn (mut p Parser) comptime_call() ast.ComptimeCall { err_node := ast.ComptimeCall{ scope: unsafe { nil } kind: .unknown } start_pos := p.tok.pos() p.check(.dollar) mut is_veb := false if p.peek_tok.kind == .dot { name := p.check_name() if name != 'veb' { p.error(error_msg) return err_node } import_mods := p.ast_imports.map(it.mod) if 'veb' !in import_mods { p.error_with_pos('`\$veb` cannot be used without importing veb', start_pos.extend(p.prev_tok.pos())) return err_node } p.register_used_import('veb') is_veb = true p.check(.dot) } method_name := p.check_name() if method_name !in supported_comptime_calls { p.error(error_msg) return err_node } is_embed_file := method_name == 'embed_file' is_html := method_name == 'html' p.check(.lpar) arg_pos := p.tok.pos() if method_name in ['env', 'pkgconfig'] { s := p.tok.lit p.check(.string) p.check(.rpar) is_env := method_name == 'env' return ast.ComptimeCall{ scope: unsafe { nil } method_name: method_name kind: if is_env { .env } else { .pkgconfig } args_var: s env_pos: start_pos pos: start_pos.extend(p.prev_tok.pos()) } } else if method_name in ['compile_error', 'compile_warn'] { mut s := '' mut args := []ast.CallArg{} if p.tok.kind == .string && p.peek_tok.kind == .rpar { s = p.tok.lit p.check(.string) } else { args << ast.CallArg{ expr: p.string_expr() typ: ast.string_type ct_expr: true } } p.check(.rpar) return ast.ComptimeCall{ scope: unsafe { nil } method_name: method_name kind: if method_name == 'compile_error' { .compile_error } else { .compile_warn } args_var: s env_pos: start_pos pos: start_pos.extend(p.prev_tok.pos()) args: args } } else if method_name == 'res' { mut has_args := false mut type_index := '' if p.tok.kind == .number { has_args = true type_index = p.tok.lit p.check(.number) } p.check(.rpar) if has_args { return ast.ComptimeCall{ scope: unsafe { nil } method_name: method_name kind: .res args_var: type_index pos: start_pos.extend(p.prev_tok.pos()) } } return ast.ComptimeCall{ scope: unsafe { nil } method_name: method_name kind: .res pos: start_pos.extend(p.prev_tok.pos()) } } else if method_name == 'd' { const_string := p.tok.lit // const_name_pos := p.tok.pos() p.check(.string) p.check(.comma) arg_expr := p.expr(0) args := [ ast.CallArg{ expr: arg_expr pos: p.tok.pos() }, ] p.check(.rpar) return ast.ComptimeCall{ scope: unsafe { nil } method_name: method_name kind: .d args_var: const_string args: args pos: start_pos.extend(p.prev_tok.pos()) } } else if method_name in ['zero', 'new'] { arg_expr := p.comptime_call_type_arg() p.check(.rpar) return ast.ComptimeCall{ scope: unsafe { nil } method_name: method_name kind: if method_name == 'zero' { .zero } else { .new } args: [ ast.CallArg{ expr: arg_expr pos: arg_pos }, ] pos: start_pos.extend(p.prev_tok.pos()) } } has_string_arg := p.tok.kind == .string mut literal_string_param := if is_html && !has_string_arg { '' } else { p.tok.lit } mut arg := ast.CallArg{} if is_html && !(has_string_arg || p.tok.kind == .rpar) { p.error('expecting `\$veb.html()` for a default template path or `\$veb.html("/path/to/template.html")`') } if is_html && p.tok.kind != .string { // $veb.html() can have no arguments } else { arg_expr := p.expr(0) if resolved_path := p.resolve_tmpl_path_expr(arg_expr) { literal_string_param = resolved_path } arg = ast.CallArg{ expr: arg_expr } } mut embed_compression_type := 'none' if is_embed_file { if p.tok.kind == .comma { p.next() p.check(.dot) embed_compression_type = p.check_name() } } p.check(.rpar) // $embed_file('/path/to/file') if is_embed_file { p.register_auto_import('v.preludes.embed_file') if embed_compression_type == 'zlib' { p.register_auto_import('v.preludes.embed_file.zlib') } return ast.ComptimeCall{ scope: unsafe { nil } method_name: method_name kind: .embed_file embed_file: ast.EmbeddedFile{ compression_type: embed_compression_type } args: [arg] pos: start_pos.extend(p.prev_tok.pos()) } } // Compile veb html template to V code, parse that V code and embed the resulting V function // that returns an html string. fn_path := p.cur_fn_name.split('_') fn_path_joined := fn_path.join(os.path_separator) fn_name_html := '${p.cur_fn_name}.html' compiled_vfile_path := os.real_path(p.scanner.file_path.replace('/', os.path_separator)) tmpl_path := if is_html && !has_string_arg { '${fn_path.last()}.html' } else { mut resolved_path := literal_string_param.replace('/', os.path_separator) resolved_path = p.resolve_tmpl_pseudo_variables(resolved_path, arg_pos) or { return err_node } resolved_path } // Looking next to the veb program dir := os.dir(compiled_vfile_path) mut path := os.join_path_single(dir, fn_path_joined) path += '.html' if !is_html || has_string_arg { if os.is_abs_path(tmpl_path) { path = tmpl_path } else { path = os.join_path_single(dir, tmpl_path) } } if !os.exists(path) { if is_html { if has_string_arg && !os.is_abs_path(tmpl_path) { path = find_veb_template_relative_to_vmod(compiled_vfile_path, tmpl_path) or { path } } else { flat_path := os.join_path_single(dir, fn_name_html) if os.exists(flat_path) { path = flat_path } if !os.exists(path) { // can be in `templates/` path = os.join_path(dir, 'templates', fn_path_joined) path += '.html' } if !os.exists(path) { flat_template_path := os.join_path(dir, 'templates', fn_name_html) if os.exists(flat_template_path) { path = flat_template_path } } if !os.exists(path) { // Same-module subdirs and `base_url` can place route handlers below the module // root while keeping `templates/` next to `v.mod`. vmod_relative_paths := [ os.join_path('templates', fn_path_joined) + '.html', os.join_path('templates', fn_name_html), ] path = find_existing_veb_template_in_vmod(compiled_vfile_path, vmod_relative_paths) or { path } } } } if !os.exists(path) { if p.pref.is_fmt { return ast.ComptimeCall{ scope: unsafe { nil } is_template: true is_veb: is_veb method_name: method_name kind: if is_html { .html } else { .tmpl } args_var: literal_string_param args: [arg] pos: start_pos.extend(p.prev_tok.pos()) } } if is_html { p.error_with_pos('veb HTML template "${tmpl_path}" not found', arg_pos) } else { p.error_with_pos('template file "${tmpl_path}" not found', arg_pos) } return err_node } // println('path is now "$path"') } tmp_fn_name := p.cur_fn_name.replace('.', '__').to_lower() + start_pos.pos.str() $if trace_comptime ? { println('>>> compiling comptime template file "${path}" for ${tmp_fn_name}') } v_code := p.compile_template_file(path, tmp_fn_name) $if print_veb_template_expansions ? { lines := v_code.split('\n') for i, line in lines { println('${path}:${i + 1}: ${line}') } } $if trace_comptime ? { println('') println('>>> template for ${path}:') println(v_code) println('>>> end of template END') println('') } // the tmpl inherits all parent scopes. previous functionality was just to // inherit the scope from which the comptime call was made and no parents. // this is much simpler and allows access to globals. can be changed if needed. p.open_scope() defer { p.close_scope() } mut file := parse_comptime(tmpl_path, v_code, mut p.table, p.pref, mut p.scope) file.path = tmpl_path template_call_stack := [ errors.CallStackItem{ file_path: p.file_path pos: start_pos }, ] // Store call stack info for template errors file.call_stack = template_call_stack // Transfer template paths and line mapping from parser to file for error reporting file.template_paths = p.template_paths file.template_line_map = p.template_line_map for i, err in file.errors { mut file_path := err.file_path mut line_nr := err.pos.line_nr if err.pos.line_nr >= 0 && err.pos.line_nr < file.template_line_map.len { line_info := file.template_line_map[err.pos.line_nr] file_path = line_info.tmpl_path line_nr = line_info.tmpl_line } file.errors[i] = errors.Error{ message: err.message details: err.details file_path: file_path pos: token.Pos{ ...err.pos line_nr: line_nr } reporter: err.reporter call_stack: if err.call_stack.len > 0 { err.call_stack } else { template_call_stack } } } for i, warn in file.warnings { mut file_path := warn.file_path mut line_nr := warn.pos.line_nr if warn.pos.line_nr >= 0 && warn.pos.line_nr < file.template_line_map.len { line_info := file.template_line_map[warn.pos.line_nr] file_path = line_info.tmpl_path line_nr = line_info.tmpl_line } file.warnings[i] = errors.Warning{ message: warn.message details: warn.details file_path: file_path pos: token.Pos{ ...warn.pos line_nr: line_nr } reporter: warn.reporter call_stack: if warn.call_stack.len > 0 { warn.call_stack } else { template_call_stack } } } for i, notice in file.notices { mut file_path := notice.file_path mut line_nr := notice.pos.line_nr if notice.pos.line_nr >= 0 && notice.pos.line_nr < file.template_line_map.len { line_info := file.template_line_map[notice.pos.line_nr] file_path = line_info.tmpl_path line_nr = line_info.tmpl_line } file.notices[i] = errors.Notice{ message: notice.message details: notice.details file_path: file_path pos: token.Pos{ ...notice.pos line_nr: line_nr } reporter: notice.reporter call_stack: if notice.call_stack.len > 0 { notice.call_stack } else { template_call_stack } } } return ast.ComptimeCall{ scope: unsafe { nil } is_template: true is_veb: is_veb veb_tmpl: file method_name: method_name kind: if is_html { .html } else { .tmpl } args_var: literal_string_param args: [arg] pos: start_pos.extend(p.prev_tok.pos()) } } fn (mut p Parser) comptime_for() ast.ComptimeFor { // p.comptime_for() handles these special forms: // `$for method in App.methods {` // `$for val in App.values {` // `$for field in App.fields {` // `$for attr in App.attributes {` // `$for variant in App.variants {` // `$for variant in field.typ.variants {` p.next() p.check(.key_for) var_pos := p.tok.pos() val_var := p.check_name() p.check(.key_in) mut expr := ast.empty_expr mut typ_pos := p.tok.pos() lang := p.parse_language() mut typ := ast.void_type if p.tok.lit.len == 0 { p.error('invalid expr, use `${p.peek_tok.lit}` instead') return ast.ComptimeFor{} } if p.tok.lit[0].is_capital() || p.tok.lit in p.imports { typ = p.parse_any_type(lang, false, true, false) } else { mut selector_expr := ast.Expr(p.ident(lang)) p.scope.mark_var_as_used((selector_expr as ast.Ident).name) for p.tok.kind == .dot && p.peek_tok.kind == .name && !is_supported_comptime_for_kind(p.peek_tok.lit) { selector_expr = p.dot_expr(selector_expr) if p.name_error { return ast.ComptimeFor{} } if selector_expr !is ast.SelectorExpr { p.error_with_pos('invalid expr, use a selector like `field.typ`', selector_expr.pos()) return ast.ComptimeFor{} } } expr = selector_expr } typ_pos = typ_pos.extend(p.prev_tok.pos()) p.check(.dot) for_val := p.check_name() mut kind := ast.ComptimeForKind.methods p.open_scope() defer { p.close_scope() } match for_val { 'params' { p.scope.register(ast.Var{ name: val_var typ: p.table.find_type('FunctionParam') pos: var_pos }) kind = .params } 'methods' { p.scope.register(ast.Var{ name: val_var typ: p.table.find_type('FunctionData') pos: var_pos }) } 'values' { p.scope.register(ast.Var{ name: val_var typ: p.table.find_type('EnumData') pos: var_pos }) kind = .values } 'fields' { p.scope.register(ast.Var{ name: val_var typ: p.table.find_type('FieldData') pos: var_pos }) kind = .fields } 'variants' { p.scope.register(ast.Var{ name: val_var typ: p.table.find_type('VariantData') pos: var_pos }) kind = .variants } 'attributes' { p.scope.register(ast.Var{ name: val_var typ: p.table.find_type('VAttribute') pos: var_pos }) kind = .attributes } else { p.error_with_pos('unknown kind `${for_val}`, available are: `methods`, `fields`, `values`, `variants`, `attributes` or `params`', p.prev_tok.pos()) return ast.ComptimeFor{} } } spos := p.tok.pos() stmts := p.parse_block() return ast.ComptimeFor{ val_var: val_var stmts: stmts kind: kind typ: typ expr: expr typ_pos: typ_pos scope: p.scope pos: spos.extend(p.tok.pos()) } } // @FN, @STRUCT, @MOD etc. See full list in token.valid_at_tokens fn (mut p Parser) at() ast.AtExpr { name := p.tok.lit kind := match name { '@FN' { token.AtKind.fn_name } '@METHOD' { token.AtKind.method_name } '@MOD' { token.AtKind.mod_name } '@STRUCT' { token.AtKind.struct_name } '@FILE' { token.AtKind.file_path } '@DIR' { token.AtKind.file_dir } '@LINE' { token.AtKind.line_nr } '@FILE_LINE' { token.AtKind.file_path_line_nr } '@LOCATION' { token.AtKind.location } '@COLUMN' { token.AtKind.column_nr } '@VCURRENTHASH' { token.AtKind.v_current_hash } '@VHASH' { token.AtKind.vhash } '@VMOD_FILE' { token.AtKind.vmod_file } '@VEXE' { token.AtKind.vexe_path } '@VEXEROOT' { token.AtKind.vexeroot_path } '@VMODROOT' { token.AtKind.vmodroot_path } '@VMODHASH' { token.AtKind.vmod_hash } '@VROOT' { token.AtKind.vroot_path } // deprecated, use @VEXEROOT or @VMODROOT '@BUILD_DATE' { token.AtKind.build_date } '@BUILD_TIME' { token.AtKind.build_time } '@BUILD_TIMESTAMP' { token.AtKind.build_timestamp } '@OS' { token.AtKind.os } '@CCOMPILER' { token.AtKind.ccompiler } '@BACKEND' { token.AtKind.backend } '@PLATFORM' { token.AtKind.platform } else { token.AtKind.unknown } } expr := ast.AtExpr{ name: name pos: p.tok.pos() kind: kind } p.next() return expr } fn (mut p Parser) comptime_selector(left ast.Expr) ast.Expr { p.check(.dollar) start_pos := p.prev_tok.pos() if p.peek_tok.kind == .lpar { method_pos := p.tok.pos() method_name := p.check_name() p.scope.mark_var_as_used(method_name) // `app.$action()` (`action` is a string) p.check(.lpar) args := p.call_args() p.check(.rpar) mut or_kind := ast.OrKind.absent mut or_pos := p.tok.pos() mut or_stmts := []ast.Stmt{} mut or_scope := ast.empty_scope if p.tok.kind == .key_orelse { // `$method() or {}`` or_kind = .block or_stmts, or_pos, or_scope = p.or_block(.with_err_var) } return ast.ComptimeCall{ left: left method_name: method_name kind: .method method_pos: method_pos scope: p.scope args_var: '' args: args pos: start_pos.extend(p.prev_tok.pos()) or_block: ast.OrExpr{ stmts: or_stmts kind: or_kind pos: or_pos scope: or_scope } } } mut has_parens := false if p.tok.kind == .lpar { p.next() has_parens = true } else { p.warn_with_pos('use brackets instead e.g. `s.$(field.name)` - run vfmt', p.tok.pos()) } expr := p.expr(0) if has_parens { p.check(.rpar) } return ast.ComptimeSelector{ has_parens: has_parens left: left field_expr: expr pos: start_pos.extend(p.prev_tok.pos()) or_block: ast.OrExpr{ stmts: []ast.Stmt{} kind: if p.tok.kind == .question { .propagate_option } else { .absent } scope: p.scope pos: p.tok.pos() } } }