| 1 | import os |
| 2 | import flag |
| 3 | import term |
| 4 | import time |
| 5 | import v.parser |
| 6 | import v.ast |
| 7 | import v.pref |
| 8 | |
| 9 | const support_color = term.can_show_color_on_stderr() && term.can_show_color_on_stdout() |
| 10 | const ecode_timeout = 101 |
| 11 | const ecode_details = { |
| 12 | -1: 'worker executable not found' |
| 13 | 101: 'too slow' |
| 14 | 102: 'too memory hungry' |
| 15 | } |
| 16 | |
| 17 | struct Context { |
| 18 | mut: |
| 19 | is_help bool |
| 20 | is_worker bool |
| 21 | is_verbose bool |
| 22 | is_silent bool // do not print any status/progress during processing, just failures. |
| 23 | is_linear bool // print linear progress log, without trying to do term cursor up + \r msg. Easier to use in a CI job |
| 24 | show_src bool // show the partial source, that cause the parser to panic/fault, when it happens. |
| 25 | timeout_ms int |
| 26 | myself string // path to this executable, so the supervisor can launch worker processes |
| 27 | all_paths []string // all files given to the supervisor process |
| 28 | path string // the current path, given to a worker process |
| 29 | cut_index int // the cut position in the source from context.path |
| 30 | max_index int // the maximum index (equivalent to the file content length) |
| 31 | // parser context in the worker processes: |
| 32 | table ast.Table |
| 33 | pref &pref.Preferences = unsafe { nil } |
| 34 | period_ms int // print periodic progress |
| 35 | stop_print bool // stop printing the periodic progress |
| 36 | } |
| 37 | |
| 38 | fn main() { |
| 39 | mut context := process_cli_args() |
| 40 | if context.is_worker { |
| 41 | pid := os.getpid() |
| 42 | context.log('> worker ${pid:5} starts parsing at cut_index: ${context.cut_index:5} | ${context.path}') |
| 43 | // A worker's process job is to try to parse a single given file in context.path. |
| 44 | // It can crash/panic freely. |
| 45 | context.table = ast.new_table() |
| 46 | context.pref = &pref.Preferences{ |
| 47 | output_mode: .silent |
| 48 | } |
| 49 | mut source := os.read_file(context.path)! |
| 50 | source = source[..context.cut_index] |
| 51 | |
| 52 | spawn fn (ms int) { |
| 53 | time.sleep(ms * time.millisecond) |
| 54 | exit(ecode_timeout) |
| 55 | }(context.timeout_ms) |
| 56 | _ := parser.parse_text(source, context.path, mut context.table, .skip_comments, |
| 57 | context.pref) |
| 58 | context.log('> worker ${pid:5} finished parsing ${context.path}') |
| 59 | exit(0) |
| 60 | } else { |
| 61 | // The process supervisor should NOT crash/panic, unlike the workers. |
| 62 | // Its job, is to: |
| 63 | // 1) start workers |
| 64 | // 2) accumulate results |
| 65 | // 3) produce a summary at the end |
| 66 | context.expand_all_paths() |
| 67 | mut fails := 0 |
| 68 | mut panics := 0 |
| 69 | sw := time.new_stopwatch() |
| 70 | for path in context.all_paths { |
| 71 | filesw := time.new_stopwatch() |
| 72 | context.start_printing() |
| 73 | new_fails, new_panics := context.process_whole_file_in_worker(path) |
| 74 | fails += new_fails |
| 75 | panics += new_panics |
| 76 | context.stop_printing() |
| 77 | context.info('File: ${path:-30} | new_fails: ${new_fails:5} | new_panics: ${new_panics:5} | Elapsed time: ${filesw.elapsed().milliseconds()}ms') |
| 78 | } |
| 79 | non_panics := fails - panics |
| 80 | context.info('Total files processed: ${context.all_paths.len:5} | Errors found: ${fails:5} | Panics: ${panics:5} | Non panics: ${non_panics:5} | Elapsed time: ${sw.elapsed().milliseconds()}ms') |
| 81 | if fails > 0 { |
| 82 | exit(1) |
| 83 | } |
| 84 | exit(0) |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | fn process_cli_args() &Context { |
| 89 | mut context := &Context{ |
| 90 | pref: pref.new_preferences() |
| 91 | } |
| 92 | context.myself = os.executable() |
| 93 | mut fp := flag.new_flag_parser(os.args_after('test-parser')) |
| 94 | fp.application(os.file_name(context.myself)) |
| 95 | fp.version('0.0.1') |
| 96 | fp.description('Test the V parser, by parsing each .v file in each PATH,\n' + |
| 97 | 'as if it was typed character by character by the user.\n' + |
| 98 | 'A PATH can be either a folder, or a specific .v file.\n' + |
| 99 | 'Note: you *have to quote* the PATH, if it contains spaces/punctuation.') |
| 100 | fp.arguments_description('PATH1 PATH2 ...') |
| 101 | fp.skip_executable() |
| 102 | context.is_help = fp.bool('help', `h`, false, 'Show help/usage screen.') |
| 103 | context.is_verbose = fp.bool('verbose', `v`, false, 'Be more verbose.') |
| 104 | context.is_silent = fp.bool('silent', `S`, false, 'Do not print progress at all.') |
| 105 | context.is_linear = fp.bool('linear', `L`, false, 'Print linear progress log. Suitable for CI.') |
| 106 | context.show_src = fp.bool('show_source', `E`, false, |
| 107 | 'Print the partial source code that caused a fault/panic in the parser.') |
| 108 | context.period_ms = fp.int('progress_ms', `s`, 500, |
| 109 | 'print a status report periodically, the period is given in milliseconds.') |
| 110 | context.is_worker = fp.bool('worker', `w`, false, |
| 111 | 'worker specific flag - is this a worker process, that can crash/panic.') |
| 112 | context.cut_index = fp.int('cut_index', `c`, 1, |
| 113 | 'worker specific flag - cut index in the source file, everything before that will be parsed, the rest - ignored.') |
| 114 | context.timeout_ms = fp.int('timeout_ms', `t`, 250, |
| 115 | 'worker specific flag - timeout in ms; a worker taking longer, will self terminate.') |
| 116 | context.path = fp.string('path', `p`, '', |
| 117 | 'worker specific flag - path to the current source file, which will be parsed.') |
| 118 | |
| 119 | if context.is_help { |
| 120 | println(fp.usage()) |
| 121 | exit(0) |
| 122 | } |
| 123 | context.all_paths = fp.finalize() or { |
| 124 | context.error(err.msg()) |
| 125 | exit(1) |
| 126 | } |
| 127 | if !context.is_worker && context.all_paths.len == 0 { |
| 128 | println(fp.usage()) |
| 129 | exit(0) |
| 130 | } |
| 131 | return context |
| 132 | } |
| 133 | |
| 134 | // //////////////// |
| 135 | fn bold(msg string) string { |
| 136 | if !support_color { |
| 137 | return msg |
| 138 | } |
| 139 | return term.bold(msg) |
| 140 | } |
| 141 | |
| 142 | fn red(msg string) string { |
| 143 | if !support_color { |
| 144 | return msg |
| 145 | } |
| 146 | return term.red(msg) |
| 147 | } |
| 148 | |
| 149 | fn yellow(msg string) string { |
| 150 | if !support_color { |
| 151 | return msg |
| 152 | } |
| 153 | return term.yellow(msg) |
| 154 | } |
| 155 | |
| 156 | fn (mut context Context) info(msg string) { |
| 157 | println(msg) |
| 158 | } |
| 159 | |
| 160 | fn (mut context Context) log(msg string) { |
| 161 | if context.is_verbose { |
| 162 | label := yellow('info') |
| 163 | ts := time.now().format_ss_micro() |
| 164 | eprintln('${label}: ${ts} | ${msg}') |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | fn (mut context Context) error(msg string) { |
| 169 | label := red('error') |
| 170 | eprintln('${label}: ${msg}') |
| 171 | } |
| 172 | |
| 173 | fn (mut context Context) expand_all_paths() { |
| 174 | context.log('> context.all_paths before: ${context.all_paths}') |
| 175 | mut files := []string{} |
| 176 | for path in context.all_paths { |
| 177 | if os.is_dir(path) { |
| 178 | files << os.walk_ext(path, '.v') |
| 179 | files << os.walk_ext(path, '.vsh') |
| 180 | continue |
| 181 | } |
| 182 | if !path.ends_with('.v') && !path.ends_with('.vv') && !path.ends_with('.vsh') { |
| 183 | context.error('`v test-parser` can only be used on .v/.vv/.vsh files.\nOffending file: "${path}".') |
| 184 | continue |
| 185 | } |
| 186 | if !os.exists(path) { |
| 187 | context.error('"${path}" does not exist.') |
| 188 | continue |
| 189 | } |
| 190 | files << path |
| 191 | } |
| 192 | context.all_paths = files |
| 193 | context.log('> context.all_paths after: ${context.all_paths}') |
| 194 | } |
| 195 | |
| 196 | fn (mut context Context) process_whole_file_in_worker(path string) (int, int) { |
| 197 | context.path = path // needed for the progress bar |
| 198 | context.log('> context.process_whole_file_in_worker path: ${path}') |
| 199 | if !(os.is_file(path) && os.is_readable(path)) { |
| 200 | context.error('${path} is not readable') |
| 201 | return 1, 0 |
| 202 | } |
| 203 | source := os.read_file(path) or { '' } |
| 204 | if source == '' { |
| 205 | // an empty file is a valid .v file |
| 206 | return 0, 0 |
| 207 | } |
| 208 | len := source.len - 1 |
| 209 | mut fails := 0 |
| 210 | mut panics := 0 |
| 211 | context.max_index = len |
| 212 | for i in 0 .. len { |
| 213 | verbosity := if context.is_verbose { '-v' } else { '' } |
| 214 | context.cut_index = i // needed for the progress bar |
| 215 | cmd := '${os.quoted_path(context.myself)} ${verbosity} --worker --timeout_ms ${context.timeout_ms:5} --cut_index ${i:5} --path ${os.quoted_path(path)} ' |
| 216 | context.log(cmd) |
| 217 | mut res := os.execute(cmd) |
| 218 | context.log('worker exit_code: ${res.exit_code} | worker output:\n${res.output}') |
| 219 | if res.exit_code != 0 { |
| 220 | fails++ |
| 221 | mut is_panic := false |
| 222 | if res.output.contains('V panic:') { |
| 223 | is_panic = true |
| 224 | panics++ |
| 225 | } |
| 226 | part := source[..i] |
| 227 | line := part.count('\n') + 1 |
| 228 | last_line := part.all_after_last('\n') |
| 229 | col := last_line.len |
| 230 | err := if is_panic { |
| 231 | red('parser failure: panic') |
| 232 | } else { |
| 233 | red('parser failure: crash, ${ecode_details[res.exit_code]}') |
| 234 | } |
| 235 | path_to_line := bold('${path}:${line}:${col}:') |
| 236 | err_line := last_line.trim_left('\t') |
| 237 | println('${path_to_line} ${err}') |
| 238 | println('\t${line} | ${err_line}') |
| 239 | println('') |
| 240 | eprintln(res.output) |
| 241 | eprintln('>>> failed command: ${cmd}') |
| 242 | if context.show_src { |
| 243 | eprintln('>>> source so far:') |
| 244 | eprintln('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>') |
| 245 | partial_source := source[..context.cut_index] |
| 246 | eprintln(partial_source) |
| 247 | eprintln('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>') |
| 248 | } |
| 249 | } |
| 250 | } |
| 251 | return fails, panics |
| 252 | } |
| 253 | |
| 254 | fn (mut context Context) start_printing() { |
| 255 | context.stop_print = false |
| 256 | if !context.is_linear && !context.is_silent { |
| 257 | println('\n') |
| 258 | } |
| 259 | spawn context.print_periodic_status() |
| 260 | } |
| 261 | |
| 262 | fn (mut context Context) stop_printing() { |
| 263 | context.stop_print = true |
| 264 | time.sleep(time.millisecond * context.period_ms / 5) |
| 265 | } |
| 266 | |
| 267 | fn (mut context Context) print_status() { |
| 268 | if context.is_silent { |
| 269 | return |
| 270 | } |
| 271 | if context.cut_index == 1 && context.max_index == 0 { |
| 272 | return |
| 273 | } |
| 274 | msg := '> ${context.path:-30} | index: ${context.cut_index:5}/${context.max_index - 1:5}' |
| 275 | if context.is_linear { |
| 276 | eprintln(msg) |
| 277 | return |
| 278 | } |
| 279 | term.cursor_up(1) |
| 280 | eprint('\r ${msg}\n') |
| 281 | } |
| 282 | |
| 283 | fn (mut context Context) print_periodic_status() { |
| 284 | context.print_status() |
| 285 | mut printed_at_least_once := false |
| 286 | for !context.stop_print { |
| 287 | context.print_status() |
| 288 | for i := 0; i < 10 && !context.stop_print; i++ { |
| 289 | time.sleep(time.millisecond * context.period_ms / 10) |
| 290 | if context.cut_index > 50 && !printed_at_least_once { |
| 291 | context.print_status() |
| 292 | printed_at_least_once = true |
| 293 | } |
| 294 | } |
| 295 | } |
| 296 | context.print_status() |
| 297 | } |
| 298 | |