v2 / cmd / tools / vtest-parser.v
297 lines · 279 sloc · 9.05 KB · dd1ad2b6abfa62a9695e4e04d4827bff137ff94e
Raw
1import os
2import flag
3import term
4import time
5import v.parser
6import v.ast
7import v.pref
8
9const support_color = term.can_show_color_on_stderr() && term.can_show_color_on_stdout()
10const ecode_timeout = 101
11const ecode_details = {
12 -1: 'worker executable not found'
13 101: 'too slow'
14 102: 'too memory hungry'
15}
16
17struct Context {
18mut:
19 is_help bool
20 is_worker bool
21 is_verbose bool
22 is_silent bool // do not print any status/progress during processing, just failures.
23 is_linear bool // print linear progress log, without trying to do term cursor up + \r msg. Easier to use in a CI job
24 show_src bool // show the partial source, that cause the parser to panic/fault, when it happens.
25 timeout_ms int
26 myself string // path to this executable, so the supervisor can launch worker processes
27 all_paths []string // all files given to the supervisor process
28 path string // the current path, given to a worker process
29 cut_index int // the cut position in the source from context.path
30 max_index int // the maximum index (equivalent to the file content length)
31 // parser context in the worker processes:
32 table ast.Table
33 pref &pref.Preferences = unsafe { nil }
34 period_ms int // print periodic progress
35 stop_print bool // stop printing the periodic progress
36}
37
38fn main() {
39 mut context := process_cli_args()
40 if context.is_worker {
41 pid := os.getpid()
42 context.log('> worker ${pid:5} starts parsing at cut_index: ${context.cut_index:5} | ${context.path}')
43 // A worker's process job is to try to parse a single given file in context.path.
44 // It can crash/panic freely.
45 context.table = ast.new_table()
46 context.pref = &pref.Preferences{
47 output_mode: .silent
48 }
49 mut source := os.read_file(context.path)!
50 source = source[..context.cut_index]
51
52 spawn fn (ms int) {
53 time.sleep(ms * time.millisecond)
54 exit(ecode_timeout)
55 }(context.timeout_ms)
56 _ := parser.parse_text(source, context.path, mut context.table, .skip_comments,
57 context.pref)
58 context.log('> worker ${pid:5} finished parsing ${context.path}')
59 exit(0)
60 } else {
61 // The process supervisor should NOT crash/panic, unlike the workers.
62 // Its job, is to:
63 // 1) start workers
64 // 2) accumulate results
65 // 3) produce a summary at the end
66 context.expand_all_paths()
67 mut fails := 0
68 mut panics := 0
69 sw := time.new_stopwatch()
70 for path in context.all_paths {
71 filesw := time.new_stopwatch()
72 context.start_printing()
73 new_fails, new_panics := context.process_whole_file_in_worker(path)
74 fails += new_fails
75 panics += new_panics
76 context.stop_printing()
77 context.info('File: ${path:-30} | new_fails: ${new_fails:5} | new_panics: ${new_panics:5} | Elapsed time: ${filesw.elapsed().milliseconds()}ms')
78 }
79 non_panics := fails - panics
80 context.info('Total files processed: ${context.all_paths.len:5} | Errors found: ${fails:5} | Panics: ${panics:5} | Non panics: ${non_panics:5} | Elapsed time: ${sw.elapsed().milliseconds()}ms')
81 if fails > 0 {
82 exit(1)
83 }
84 exit(0)
85 }
86}
87
88fn process_cli_args() &Context {
89 mut context := &Context{
90 pref: pref.new_preferences()
91 }
92 context.myself = os.executable()
93 mut fp := flag.new_flag_parser(os.args_after('test-parser'))
94 fp.application(os.file_name(context.myself))
95 fp.version('0.0.1')
96 fp.description('Test the V parser, by parsing each .v file in each PATH,\n' +
97 'as if it was typed character by character by the user.\n' +
98 'A PATH can be either a folder, or a specific .v file.\n' +
99 'Note: you *have to quote* the PATH, if it contains spaces/punctuation.')
100 fp.arguments_description('PATH1 PATH2 ...')
101 fp.skip_executable()
102 context.is_help = fp.bool('help', `h`, false, 'Show help/usage screen.')
103 context.is_verbose = fp.bool('verbose', `v`, false, 'Be more verbose.')
104 context.is_silent = fp.bool('silent', `S`, false, 'Do not print progress at all.')
105 context.is_linear = fp.bool('linear', `L`, false, 'Print linear progress log. Suitable for CI.')
106 context.show_src = fp.bool('show_source', `E`, false,
107 'Print the partial source code that caused a fault/panic in the parser.')
108 context.period_ms = fp.int('progress_ms', `s`, 500,
109 'print a status report periodically, the period is given in milliseconds.')
110 context.is_worker = fp.bool('worker', `w`, false,
111 'worker specific flag - is this a worker process, that can crash/panic.')
112 context.cut_index = fp.int('cut_index', `c`, 1,
113 'worker specific flag - cut index in the source file, everything before that will be parsed, the rest - ignored.')
114 context.timeout_ms = fp.int('timeout_ms', `t`, 250,
115 'worker specific flag - timeout in ms; a worker taking longer, will self terminate.')
116 context.path = fp.string('path', `p`, '',
117 'worker specific flag - path to the current source file, which will be parsed.')
118
119 if context.is_help {
120 println(fp.usage())
121 exit(0)
122 }
123 context.all_paths = fp.finalize() or {
124 context.error(err.msg())
125 exit(1)
126 }
127 if !context.is_worker && context.all_paths.len == 0 {
128 println(fp.usage())
129 exit(0)
130 }
131 return context
132}
133
134// ////////////////
135fn bold(msg string) string {
136 if !support_color {
137 return msg
138 }
139 return term.bold(msg)
140}
141
142fn red(msg string) string {
143 if !support_color {
144 return msg
145 }
146 return term.red(msg)
147}
148
149fn yellow(msg string) string {
150 if !support_color {
151 return msg
152 }
153 return term.yellow(msg)
154}
155
156fn (mut context Context) info(msg string) {
157 println(msg)
158}
159
160fn (mut context Context) log(msg string) {
161 if context.is_verbose {
162 label := yellow('info')
163 ts := time.now().format_ss_micro()
164 eprintln('${label}: ${ts} | ${msg}')
165 }
166}
167
168fn (mut context Context) error(msg string) {
169 label := red('error')
170 eprintln('${label}: ${msg}')
171}
172
173fn (mut context Context) expand_all_paths() {
174 context.log('> context.all_paths before: ${context.all_paths}')
175 mut files := []string{}
176 for path in context.all_paths {
177 if os.is_dir(path) {
178 files << os.walk_ext(path, '.v')
179 files << os.walk_ext(path, '.vsh')
180 continue
181 }
182 if !path.ends_with('.v') && !path.ends_with('.vv') && !path.ends_with('.vsh') {
183 context.error('`v test-parser` can only be used on .v/.vv/.vsh files.\nOffending file: "${path}".')
184 continue
185 }
186 if !os.exists(path) {
187 context.error('"${path}" does not exist.')
188 continue
189 }
190 files << path
191 }
192 context.all_paths = files
193 context.log('> context.all_paths after: ${context.all_paths}')
194}
195
196fn (mut context Context) process_whole_file_in_worker(path string) (int, int) {
197 context.path = path // needed for the progress bar
198 context.log('> context.process_whole_file_in_worker path: ${path}')
199 if !(os.is_file(path) && os.is_readable(path)) {
200 context.error('${path} is not readable')
201 return 1, 0
202 }
203 source := os.read_file(path) or { '' }
204 if source == '' {
205 // an empty file is a valid .v file
206 return 0, 0
207 }
208 len := source.len - 1
209 mut fails := 0
210 mut panics := 0
211 context.max_index = len
212 for i in 0 .. len {
213 verbosity := if context.is_verbose { '-v' } else { '' }
214 context.cut_index = i // needed for the progress bar
215 cmd := '${os.quoted_path(context.myself)} ${verbosity} --worker --timeout_ms ${context.timeout_ms:5} --cut_index ${i:5} --path ${os.quoted_path(path)} '
216 context.log(cmd)
217 mut res := os.execute(cmd)
218 context.log('worker exit_code: ${res.exit_code} | worker output:\n${res.output}')
219 if res.exit_code != 0 {
220 fails++
221 mut is_panic := false
222 if res.output.contains('V panic:') {
223 is_panic = true
224 panics++
225 }
226 part := source[..i]
227 line := part.count('\n') + 1
228 last_line := part.all_after_last('\n')
229 col := last_line.len
230 err := if is_panic {
231 red('parser failure: panic')
232 } else {
233 red('parser failure: crash, ${ecode_details[res.exit_code]}')
234 }
235 path_to_line := bold('${path}:${line}:${col}:')
236 err_line := last_line.trim_left('\t')
237 println('${path_to_line} ${err}')
238 println('\t${line} | ${err_line}')
239 println('')
240 eprintln(res.output)
241 eprintln('>>> failed command: ${cmd}')
242 if context.show_src {
243 eprintln('>>> source so far:')
244 eprintln('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
245 partial_source := source[..context.cut_index]
246 eprintln(partial_source)
247 eprintln('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
248 }
249 }
250 }
251 return fails, panics
252}
253
254fn (mut context Context) start_printing() {
255 context.stop_print = false
256 if !context.is_linear && !context.is_silent {
257 println('\n')
258 }
259 spawn context.print_periodic_status()
260}
261
262fn (mut context Context) stop_printing() {
263 context.stop_print = true
264 time.sleep(time.millisecond * context.period_ms / 5)
265}
266
267fn (mut context Context) print_status() {
268 if context.is_silent {
269 return
270 }
271 if context.cut_index == 1 && context.max_index == 0 {
272 return
273 }
274 msg := '> ${context.path:-30} | index: ${context.cut_index:5}/${context.max_index - 1:5}'
275 if context.is_linear {
276 eprintln(msg)
277 return
278 }
279 term.cursor_up(1)
280 eprint('\r ${msg}\n')
281}
282
283fn (mut context Context) print_periodic_status() {
284 context.print_status()
285 mut printed_at_least_once := false
286 for !context.stop_print {
287 context.print_status()
288 for i := 0; i < 10 && !context.stop_print; i++ {
289 time.sleep(time.millisecond * context.period_ms / 10)
290 if context.cut_index > 50 && !printed_at_least_once {
291 context.print_status()
292 printed_at_least_once = true
293 }
294 }
295 }
296 context.print_status()
297}
298