module main import os import net.urllib import encoding.html import strings import markdown import v.scanner import v.ast import v.token import document as doc import v.pref import v.util { tabs } const css_js_assets = ['doc.css', 'normalize.css', 'doc.js', 'dark-mode.js'] const default_theme = os.resource_abs_path('theme') const link_svg = '' const single_quote = "'" const double_quote = '"' const quote_escape_seq = [single_quote, '', double_quote, ''] enum HighlightTokenTyp { unone boolean builtin char comment function keyword name number operator punctuation string // For string interpolation opening_string string_interp partial_string closing_string symbol none module_ prefix } struct SearchModuleResult { description string link string } struct SearchResult { prefix string badge string description string link string } fn (vd &VDoc) render_search_index(out Output) { mut js_search_index := strings.new_builder(200) mut js_search_data := strings.new_builder(200) js_search_index.write_string('var searchModuleIndex = [\n') js_search_data.write_string('var searchModuleData = [\n') for i, title in vd.search_module_index { data := vd.search_module_data[i] js_search_index.write_string('"${title}",\n') description := data.description.replace('\n', '').replace('\r', '') // fix multiline js string bug js_search_data.write_string('["${description}","${data.link}"],\n') } js_search_index.writeln('];\n') js_search_index.write_string('var searchIndex = [\n') js_search_data.writeln('];\n') js_search_data.write_string('var searchData = [\n') for i, title in vd.search_index { data := vd.search_data[i] js_search_index.write_string('"${title}",\n') // array instead of object to reduce file size js_search_data.write_string('["${data.badge}","${data.description}","${data.link}","${data.prefix}"],\n') } js_search_index.writeln('];\n') js_search_data.writeln('];\n') final := js_search_index.str() + js_search_data.str() out_file_path := os.join_path(out.path, 'search_index.js') println('Generating search_index.js of ${final.len:8} bytes in `${out_file_path} ...') os.write_file(out_file_path, final) or { panic(err) } } fn (mut vd VDoc) render_static_html(out Output) { vd.assets = { 'doc_css': vd.get_resource(css_js_assets[0], out) 'normalize_css': vd.get_resource(css_js_assets[1], out) 'doc_js': vd.get_resource(css_js_assets[2], out) 'dark_mode_js': vd.get_resource(css_js_assets[3], out) 'light_icon': vd.get_resource('light.svg', out) 'dark_icon': vd.get_resource('dark.svg', out) 'menu_icon': vd.get_resource('menu.svg', out) 'arrow_icon': vd.get_resource('arrow.svg', out) } } fn (vd &VDoc) get_resource(name string, out Output) string { cfg := vd.cfg path := os.join_path(cfg.theme_dir, name) mut res := os.read_file(path) or { panic('vdoc: could not read ${path}') } /* if minify { if name.ends_with('.js') { res = js_compress(res) } else { res = res.split_into_lines().map(it.trim_space()).join('') } } */ // TODO: Make SVG inline for now if cfg.inline_assets || path.ends_with('.svg') { return res } else { output_path := os.join_path(out.path, name) if !os.exists(output_path) { println('Copying ${res.len:8} bytes from `${path}` to `${output_path}` ...') os.write_file(output_path, res) or { panic(err) } } return name } } fn (mut vd VDoc) collect_search_index(out Output) { cfg := vd.cfg for doc in vd.docs { mod := doc.head.name vd.search_module_index << mod comments := if cfg.include_examples { doc.head.merge_comments() } else { doc.head.merge_comments_without_examples() } vd.search_module_data << SearchModuleResult{ description: trim_doc_node_description(mod, comments) link: vd.get_file_name(mod, out) } for _, dn in doc.contents { vd.create_search_results(mod, dn, out) } } } fn (mut vd VDoc) create_search_results(mod string, dn doc.DocNode, out Output) { cfg := vd.cfg if dn.kind == .const_group { return } comments := if cfg.include_examples { dn.merge_comments() } else { dn.merge_comments_without_examples() } dn_description := trim_doc_node_description(dn.name, comments) vd.search_index << dn.name vd.search_data << SearchResult{ prefix: if dn.parent_name != '' { '${dn.kind} (${dn.parent_name})' } else { '${dn.kind} ' } description: dn_description badge: mod link: vd.get_file_name(mod, out) + '#' + get_node_id(dn) } for child in dn.children { vd.create_search_results(mod, child, out) } } fn (vd &VDoc) get_repo_file_path_for_links(file_path string) string { if file_path == '' { return '' } cfg := vd.cfg if !cfg.is_multi { return os.file_name(file_path).replace('\\', '/') } base_dir := os.dir(os.real_path(cfg.input_path)) prefix := base_dir + os.path_separator if file_path.starts_with(prefix) { return file_path[prefix.len..].replace('\\', '/') } return file_path.replace('\\', '/') } fn (vd &VDoc) write_content(cn &doc.DocNode, d &doc.Doc, mut hw strings.Builder) { cfg := vd.cfg file_path_name := vd.get_repo_file_path_for_links(cn.file_path) src_link := get_src_link(vd.manifest.repo_url, vd.manifest.repo_branch, file_path_name, cn.pos.line_nr + 1) md_link_base := get_src_dir_link(vd.manifest.repo_url, vd.manifest.repo_branch, file_path_name) if cn.content.len != 0 || cn.name == 'Constants' { hw.write_string(vd.doc_node_html(cn, src_link, md_link_base, false, cfg.include_examples, d.table)) hw.write_string('\n') } for child in cn.children { child_file_path_name := vd.get_repo_file_path_for_links(child.file_path) child_src_link := get_src_link(vd.manifest.repo_url, vd.manifest.repo_branch, child_file_path_name, child.pos.line_nr + 1) child_md_link_base := get_src_dir_link(vd.manifest.repo_url, vd.manifest.repo_branch, child_file_path_name) hw.write_string(vd.doc_node_html(child, child_src_link, child_md_link_base, false, cfg.include_examples, d.table)) hw.write_string('\n') } } fn (vd &VDoc) gen_html(d doc.Doc) string { cfg := vd.cfg mut symbols_toc := strings.new_builder(200) mut contents := strings.new_builder(200) dcs_contents := d.contents.arr() // generate toc first head_md_link_base := if is_module_readme(d.head) { readme_file_path := vd.get_repo_file_path_for_links(d.head.file_path) get_src_dir_link(vd.manifest.repo_url, vd.manifest.repo_branch, readme_file_path) } else { '' } contents.writeln(vd.doc_node_html(d.head, '', head_md_link_base, true, cfg.include_examples, d.table)) if is_module_readme(d.head) { write_toc(d.head, mut symbols_toc) } for cn in dcs_contents { vd.write_content(&cn, &d, mut contents) write_toc(cn, mut symbols_toc) // write head } if cfg.html_only_contents { // no need for theming, styling etc, useful for testing and for external documentation generators return contents.str() } // write css header_name := if cfg.is_multi && vd.docs.len > 1 { os.file_name(os.real_path(cfg.input_path)) } else { d.head.name } modules_toc_str := if cfg.is_multi || vd.docs.len > 1 { vd.gen_modules_toc(d.head.name) } else { '' } symbols_toc_str := symbols_toc.str() mut result := os.read_file(os.join_path(cfg.theme_dir, 'index.html')) or { panic(err) } if cfg.html_no_vhash { result = result.replace('{{ version }}', 'latest') } else { mut version := if vd.manifest.version.len != 0 { vd.manifest.version } else { '' } version = [version, @VCURRENTHASH].join(' ') result = result.replace('{{ version }}', version) } result = result.replace('{{ title }}', d.head.name) result = result.replace('{{ head_name }}', header_name) result = result.replace('{{ light_icon }}', vd.assets['light_icon']) result = result.replace('{{ dark_icon }}', vd.assets['dark_icon']) result = result.replace('{{ menu_icon }}', vd.assets['menu_icon']) if cfg.html_no_assets { result = result.replace('{{ head_assets }}', '') } else { result = result.replace('{{ head_assets }}', if cfg.inline_assets { ' ${tabs(2)} ${tabs(2)}' } else { ' ${tabs(2)} ${tabs(2)}' }) } if cfg.html_no_toc_urls { result = result.replace('{{ toc_links }}', '') } else { result = result.replace('{{ toc_links }}', if cfg.is_multi || vd.docs.len > 1 { modules_toc_str } else { symbols_toc_str }) } result = result.replace('{{ contents }}', contents.str()) if cfg.html_no_right { result = result.replace('{{ right_content }}', '') } else { result = result.replace('{{ right_content }}', if cfg.is_multi && d.head.name != 'README' { '
' } else { '' }) } if cfg.html_no_footer { result = result.replace('{{ footer_content }}', '') } else { result = result.replace('{{ footer_content }}', gen_footer_text(d, !cfg.no_timestamp)) } if cfg.html_no_assets { result = result.replace('{{ footer_assets }}', '') } else { result = result.replace('{{ footer_assets }}', if cfg.inline_assets { '' } else { '' }) } return result } fn (vd &VDoc) gen_modules_toc(active_doc string) string { mut modules_toc := strings.new_builder(200) mut used_submod_prefixes := map[string]bool{} doc_names := vd.docs.map(it.head.name) for dc in vd.docs { mut submod_prefix := dc.head.name.all_before('.') if index := dc.head.frontmatter['index'] { if dc.head.name == 'index' { submod_prefix = index } } if used_submod_prefixes[submod_prefix] { continue } used_submod_prefixes[submod_prefix] = true mut href_name := '' if dc.head.name in ['README', 'index'] { href_name = './index.html' } else if submod_prefix in doc_names { href_name = './${submod_prefix}.html' } submodules := vd.docs.filter(it.head.name.starts_with(submod_prefix + '.')) dropdown := if submodules.len > 0 { vd.assets['arrow_icon'] } else { '' } active_class := if dc.head.name == active_doc { ' active' } else { '' } menu_item := if href_name != '' { '${submod_prefix}' } else { '${submod_prefix}' } modules_toc.write_string('
  • \n\n') for j, cdoc in submodules { if j == 0 { modules_toc.write_string('\n') } } modules_toc.write_string('
  • \n') } return modules_toc.str() } fn get_repo_file_link(repo_url string, repo_branch string, file_name string) string { mut url := urllib.parse(repo_url) or { return '' } if url.path.len <= 1 || file_name == '' { return '' } url.path = url.path.trim_right('/') + match url.host { 'github.com' { '/blob/${repo_branch}/${file_name}' } 'gitlab.com' { '/-/blob/${repo_branch}/${file_name}' } 'git.sir.ht' { '/tree/${repo_branch}/${file_name}' } else { '' } } if url.path == '/' { return '' } return url.str() } fn get_src_dir_link(repo_url string, repo_branch string, file_name string) string { file_url := get_repo_file_link(repo_url, repo_branch, file_name) if file_url == '' { return '' } mut parsed_file_url := urllib.parse(file_url) or { return '' } mut dir_path := parsed_file_url.path.all_before_last('/') if dir_path == '' { dir_path = '/' } parsed_file_url.path = if dir_path == '/' { '/' } else { dir_path + '/' } parsed_file_url.raw_query = '' parsed_file_url.fragment = '' return parsed_file_url.str() } fn get_src_link(repo_url string, repo_branch string, file_name string, line_nr int) string { file_url := get_repo_file_link(repo_url, repo_branch, file_name) if file_url == '' { return '' } mut parsed_file_url := urllib.parse(file_url) or { return '' } parsed_file_url.fragment = 'L${line_nr}' return parsed_file_url.str() } fn normalize_url_path(path string) string { if path == '' { return '' } is_absolute := path.starts_with('/') mut parts := []string{} for part in path.split('/') { match part { '', '.' {} '..' { if parts.len > 0 { parts.delete_last() } } else { parts << part } } } mut normalized := parts.join('/') if is_absolute { normalized = '/' + normalized } return if normalized == '' && is_absolute { '/' } else { normalized } } fn is_relative_markdown_link(link string) bool { value := link.trim_space() if value == '' || value.starts_with('#') || value.starts_with('//') { return false } if value.starts_with('/') { return false } if url := urllib.parse(value) { return url.scheme == '' && url.host == '' } return true } fn resolve_relative_markdown_link(base_url string, link string) string { if base_url == '' || !is_relative_markdown_link(link) { return link } mut parsed_base := urllib.parse(base_url) or { return link } mut relative_path := link mut fragment := '' if hash_idx := relative_path.index('#') { fragment = relative_path[hash_idx + 1..] relative_path = relative_path[..hash_idx] } mut query := '' if query_idx := relative_path.index('?') { query = relative_path[query_idx + 1..] relative_path = relative_path[..query_idx] } base_path := if parsed_base.path.ends_with('/') { parsed_base.path } else { parsed_base.path.all_before_last('/') + '/' } parsed_base.path = normalize_url_path(base_path + relative_path) parsed_base.raw_query = query parsed_base.fragment = fragment return parsed_base.str() } fn write_token(tok token.Token, typ HighlightTokenTyp, mut buf strings.Builder) { mut token_content := '' match typ { .unone, .operator, .punctuation { token_content = tok.kind.str() } .string_interp { // tok.kind.str() for this returns $2 instead of $ token_content = '$' } .opening_string { token_content = "'${tok.lit}" } .closing_string { // A string as the next token of the expression // inside the string interpolation indicates that // this is the closing of string interpolation token_content = "${tok.lit}'" } .string { token_content = "'${tok.lit}'" } .char { token_content = '`${tok.lit}`' } .comment { if tok.lit != '' && tok.lit[0] == 1 { token_content = '//${tok.lit[1..]}' } else { token_content = '//${tok.lit}' } } else { token_content = tok.lit } } buf.write_string(html.escape(token_content)) } fn html_highlight(code string, tb &ast.Table) string { mut s := scanner.new_scanner(code, .parse_comments, &pref.Preferences{ output_mode: .silent }) mut tok := s.scan() mut prev_tok := tok mut next_tok := s.scan() mut buf := strings.new_builder(200) mut i := 0 mut inside_string_interp := false for i < code.len { if i != tok.pos { // All characters not detected by the scanner // (mostly whitespaces) go here. ch := code[i] if ch == `<` { buf.write_string('<') } else if ch == `>` { buf.write_string('>') } else if ch == `&` { buf.write_string('&') } else { buf.write_u8(ch) } i++ continue } mut tok_typ := HighlightTokenTyp.unone match tok.kind { .name { if tok.lit in highlight_builtin_types || tb.known_type(tok.lit) { tok_typ = .builtin } else if next_tok.kind == .lcbr { tok_typ = .symbol } else if next_tok.kind == .lpar || (!tok.lit[0].is_capital() && next_tok.kind == .lt && next_tok.pos == tok.pos + tok.lit.len) { tok_typ = .function } else { tok_typ = .name } } .comment { tok_typ = .comment } .chartoken { tok_typ = .char } .str_dollar { tok_typ = .string_interp inside_string_interp = true } .string { if inside_string_interp { if next_tok.kind == .str_dollar { // the " hello " in "${a} hello ${b} world" tok_typ = .partial_string } else { // the " world" in "${a} hello ${b} world" tok_typ = .closing_string } // NOTE: Do not switch inside_string_interp yet! // It will be handy later when we do some special // handling in generating code (see code below) } else if next_tok.kind == .str_dollar { tok_typ = .opening_string } else { tok_typ = .string } } .number { tok_typ = .number } .key_true, .key_false { tok_typ = .boolean } .lpar, .lcbr, .rpar, .rcbr, .lsbr, .rsbr, .semicolon, .colon, .comma, .dot, .dotdot, .ellipsis { tok_typ = .punctuation } else { if token.is_key(tok.lit) || token.is_decl(tok.kind) { tok_typ = .keyword } else if tok.kind.is_assign() || tok.is_unary() || tok.kind.is_relational() || tok.kind.is_infix() || tok.kind.is_postfix() { tok_typ = .operator } } } if tok_typ in [.unone, .name] { write_token(tok, tok_typ, mut buf) } else { // Special handling for "complex" string literals if tok_typ in [.partial_string, .closing_string] && inside_string_interp { // rcbr is not rendered when the string on the right // side of the expr/string interpolation is not empty. // e.g. "${a}.${b}${c}" // expectation: "${a}.${b}${c}" // reality: "${a.${b}${c}" if tok.lit.len != 0 { write_token(token.Token{ kind: .rcbr }, .unone, mut buf) } inside_string_interp = false } // Properly treat and highlight the "string"-related types // as if they are "string" type. final_tok_typ := match tok_typ { .opening_string, .partial_string, .closing_string { HighlightTokenTyp.string } else { tok_typ } } buf.write_string('') if tok_typ == .string { // Make sure to escape html in strings. Otherwise it will be rendered in the // html documentation outputs / its style rules will affect the readme. buf.write_string("'${html.escape(tok.lit.str())}'") } else { if final_tok_typ == .string && prev_tok.lit == 'return' { buf.write_string(' ') } write_token(tok, tok_typ, mut buf) } buf.write_string('') } if next_tok.kind == .eof { break } i = tok.pos + tok.len // This is to avoid issues that skips any "unused" tokens // For example: Call expr with complex string literals as arg if i - 1 == next_tok.pos { i-- } prev_tok = tok tok = next_tok next_tok = s.scan() } return buf.str() } fn (vd &VDoc) doc_node_html(dn doc.DocNode, link string, md_link_base string, head bool, include_examples bool, tb &ast.Table) string { mut dnw := strings.new_builder(200) head_tag := if head { 'h1' } else { 'h2' } mut renderer := markdown.HtmlRenderer{ transformer: &MdHtmlCodeHighlighter{ table: tb relative_link_base: md_link_base } } only_comments_text := dn.merge_comments_without_examples() md_content := markdown.render(prepare_markdown_for_html(only_comments_text), mut renderer) or { '' } highlighted_code := html_highlight(dn.content, tb) node_class := if dn.kind == .const_group { ' const' } else { '' } sym_name := get_sym_name(dn) mut deprecated_tags := dn.tags.filter(it.starts_with('deprecated')) if doc.should_sort { deprecated_tags.sort() } mut tags := dn.tags.filter(!it.starts_with('deprecated')) if doc.should_sort { tags.sort() } mut node_id := get_node_id(dn) mut hash_link := if !head { ' #' } else { '' } if head && is_module_readme(dn) { node_id = 'readme_${node_id}' hash_link = ' #' } dnw.writeln('${tabs(2)}
    ') if dn.name != '' { if dn.kind == .const_group { dnw.write_string('${tabs(3)}
    <${head_tag}>${sym_name}${hash_link}') } else { dnw.write_string('${tabs(3)}
    <${head_tag}>${dn.kind} ${sym_name}${hash_link}') } if link != '' { dnw.write_string('${link_svg}') } dnw.write_string('
    \n') } if deprecated_tags.len > 0 { attributes := deprecated_tags.map('
    ${it.replace_each(quote_escape_seq)}
    \n').join('') dnw.writeln('
    ${attributes}
    \n') } if tags.len > 0 { attributes := tags.map('
    ${it}
    ').join('') dnw.writeln('
    ${attributes}
    ') } if !head && dn.content.len > 0 { dnw.writeln('
    \n${highlighted_code}
    ') } // do not mess with md_content further, its formatting is important, just output it 1:1 ! dnw.writeln('${md_content}\n') // Write examples if any found examples := dn.examples() if include_examples && examples.len > 0 { example_title := if examples.len > 1 { 'Examples' } else { 'Example' } dnw.writeln('

    ${example_title}

    ') for example in examples { hl_example := html_highlight(example, tb) dnw.writeln('
    \n${hl_example}
    ') } dnw.writeln('
    ') } dnw.writeln('
    ') dnw_str := dnw.str() return dnw_str } fn prepare_markdown_for_html(text string) string { if !text.contains('\n') { return text } lines := text.split_into_lines() mut prepared := []string{cap: lines.len} mut is_codeblock := false mut prev_line := '' for i, line in lines { trimmed := line.trim_space() if trimmed.starts_with('```') { prepared << line is_codeblock = !is_codeblock prev_line = line continue } if is_codeblock { prepared << line prev_line = line continue } if line_continues_previous_block(prev_line, line) && prepared.len > 0 { prepared[prepared.len - 1] += ' ' + trimmed prev_line = line continue } next_line := if i + 1 < lines.len { lines[i + 1] } else { '' } if blockquote_line_needs_hard_break(line, next_line) { prepared << line + ' ' } else { prepared << line } prev_line = line } return prepared.join('\n') } fn line_continues_previous_block(prev_line string, line string) bool { prev_trimmed := prev_line.trim_space() trimmed := line.trim_space() if prev_trimmed == '' || trimmed == '' { return false } if prev_trimmed.starts_with('>') || trimmed.starts_with('>') { return false } if prev_trimmed.starts_with('```') || trimmed.starts_with('```') { return false } if markdown_line_starts_new_block(line) { return false } if prev_trimmed.starts_with('#') || prev_trimmed.starts_with('|') || markdown_line_is_horizontal_rule(prev_trimmed) { return false } return true } fn markdown_line_starts_new_block(line string) bool { trimmed := line.trim_space() if trimmed == '' { return false } if trimmed.starts_with('#') || trimmed.starts_with('>') || trimmed.starts_with('|') || trimmed.starts_with('```') || markdown_line_is_horizontal_rule(trimmed) { return true } if markdown_indent_width(line) >= 4 { return true } return markdown_line_is_list_item(trimmed) } fn markdown_line_is_list_item(line string) bool { if line.len > 1 && line[1] == ` ` && line[0] in [`-`, `*`, `+`] { return true } return line.len > 2 && line[2] == ` ` && line[1] == `.` && line[0].is_digit() } fn markdown_line_is_horizontal_rule(line string) bool { line_no_spaces := line.replace(' ', '') if line_no_spaces.len < 3 { return false } for ch in ['-', '=', '*', '_', '~'] { if line_no_spaces.starts_with(ch.repeat(3)) && line_no_spaces.count(ch) == line_no_spaces.len { return true } } return false } fn markdown_indent_width(line string) int { mut width := 0 for ch in line { if ch == ` ` { width++ continue } if ch == `\t` { width += 4 continue } break } return width } fn blockquote_line_needs_hard_break(line string, next_line string) bool { if line.ends_with(' ') { return false } payload := blockquote_payload(line) or { return false } if payload == '' { return false } next_payload := blockquote_payload(next_line) or { return false } if next_payload == '' { return false } return !blockquote_payload_starts_new_block(next_payload) } fn blockquote_payload(line string) ?string { trimmed := line.trim_space() if !trimmed.starts_with('>') { return none } return trimmed[1..].trim_space() } fn blockquote_payload_starts_new_block(payload string) bool { if payload == '' || payload.starts_with('```') || payload.starts_with('>') || payload.starts_with('|') { return true } if payload.len > 1 && payload[1] == ` ` && payload[0] in [`-`, `*`, `+`] { return true } if payload.len > 2 && payload[2] == ` ` && payload[1] == `.` && payload[0].is_digit() { return true } if !payload.starts_with('#') { return false } line_before_spaces := payload.before(' ') return line_before_spaces.count('#') == line_before_spaces.len } fn write_toc(dn doc.DocNode, mut toc strings.Builder) { mut toc_slug := if dn.name == '' || dn.content.len == 0 { '' } else { slug(dn.name) } if toc_slug == '' && dn.children.len > 0 { if dn.children[0].name == '' { toc_slug = slug(dn.name) } else { toc_slug = slug(dn.name + '.' + dn.children[0].name) } } if is_module_readme(dn) { if dn.comments.len == 0 || (dn.comments.len > 0 && dn.comments[0].text.len == 0) { return } toc.write_string('
  • README') } else if dn.name != 'Constants' { toc.write_string('
  • ${dn.kind} ${dn.name}') toc.writeln(' ') } else { toc.write_string('
  • ${dn.name}') } toc.writeln('
  • ') } struct MdHtmlCodeHighlighter { mut: language string table &ast.Table relative_link_base string } fn (f &MdHtmlCodeHighlighter) transform_attribute(p markdown.ParentType, name string, value string) string { mut transformed := value if p is markdown.MD_SPANTYPE && p in [.md_span_a, .md_span_img] && name in ['href', 'src'] { transformed = resolve_relative_markdown_link(f.relative_link_base, value) } return markdown.default_html_transformer.transform_attribute(p, name, transformed) } fn (f &MdHtmlCodeHighlighter) transform_content(parent markdown.ParentType, text string) string { if parent is markdown.MD_BLOCKTYPE && parent == .md_block_code { if f.language == '' { return html.escape(text) } output := html_highlight(text, f.table) // Reset the language, so that it will not persist between blocks, // and will not be accidentally re-used for the next block, that may be lacking ```language : unsafe { f.language = '' } return output } return text } fn (mut f MdHtmlCodeHighlighter) config_set(key string, val string) { if key == 'code_language' { f.language = val } }