v2 / vlib / v / live / live_test.v
255 lines · 230 sloc · 9.63 KB · c62446b2abebb30193aedccb22a613e45cc32b4a
Raw
1// vtest build: !sanitized_job?
2// vtest retry: 4
3import os
4import log
5import time
6
7/*
8The goal of this test, is to simulate a developer, that has run a program, compiled with -live flag.
9
10It does so by writing a new generated program containing a @[live] fn pmessage() string {...} function,
11(that program is in `vlib/v/live/live_test_template.vv`)
12then runs the generated program at the start *in the background*,
13waits some time, so that the program could run a few iterations, then modifies its source
14(simulates a developer that has saved a new version of the program source),
15then it waits some more, modifies it again and saves it once more.
16
17On each modification, the running program, should detect that its source code has changed,
18and recompile a shared library, which it then it should load, and thus modify its own
19behavior at runtime (the pmessage function).
20
21If everything works fine, the output of the generated program would have changed at least 1-2 times,
22which then is detected by the test program (the histogram checks).
23
24Since this test program is sensitive to coordination (or lack of) of several processes,
25it tries to sidestep the coordination issue by polling the file system for the existence
26of files, ORIGINAL.txt ... STOP.txt , which are appended to by the generated program.
27
28Note: That approach of monitoring the state of the running generated program, is clearly not ideal,
29but sidesteps the issue of coordinating processes through IPC or stdin/stdout in hopefully
30not very flaky way.
31
32TODO: Cleanup this when/if v has better process control/communication primitives.
33*/
34const vexe = os.getenv('VEXE')
35const vtmp_folder = os.join_path(os.vtmp_dir(), 'live_tests')
36const main_source_file = os.join_path(vtmp_folder, 'main.v')
37const tmp_file = os.join_path(vtmp_folder, 'mymodule', 'generated_live_module.tmp')
38const source_file = os.join_path(vtmp_folder, 'mymodule', 'mymodule.v')
39const genexe_file = os.join_path(vtmp_folder, 'generated_live_program.exe')
40const output_file = os.join_path(vtmp_folder, 'generated_live_program.output.txt')
41const res_original_file = os.join_path(vtmp_folder, 'ORIGINAL.txt')
42const res_changed_file = os.join_path(vtmp_folder, 'CHANGED.txt')
43const res_another_file = os.join_path(vtmp_folder, 'ANOTHER.txt')
44const res_stop_file = os.join_path(vtmp_folder, 'STOP.txt')
45const live_program_source = get_source_template()
46
47fn get_source_template() string {
48 src := os.read_file(os.join_path(os.dir(@FILE), 'live_test_template.vv')) or { panic(err) }
49 return src.replace('#OUTPUT_FILE#', output_file.replace('\\', '\\\\'))
50}
51
52fn atomic_write_source(source string) {
53 // Note: here wrtiting is done in 2 steps, since os.write_file can take some time,
54 // during which the file will be modified, but it will still be not completely written.
55 // The os.mv after that, guarantees that the reloader will see a complete valid V program.
56 os.write_file(tmp_file, source) or { panic(err) }
57 os.mv(tmp_file, source_file) or { panic(err) }
58}
59
60//
61fn testsuite_begin() {
62 os.rmdir_all(vtmp_folder) or {}
63 os.mkdir_all(vtmp_folder) or {}
64 os.mkdir_all(os.join_path(vtmp_folder, 'mymodule'))!
65 os.write_file(os.join_path(vtmp_folder, 'v.mod'), '')!
66 os.cp(os.join_path(os.dir(@FILE), 'live_test_template_main.vv'), os.join_path(vtmp_folder,
67 'main.v'))!
68 if os.user_os() !in ['linux', 'solaris'] && os.getenv('FORCE_LIVE_TEST').len == 0 {
69 eprintln('Testing the runtime behaviour of -live mode,')
70 eprintln('is reliable only on Linux/macOS for now.')
71 eprintln('You can still do it by setting FORCE_LIVE_TEST=1 .')
72 exit(0)
73 }
74 atomic_write_source(live_program_source)
75 // os.system('tree ${vtmp_folder}') exit(1)
76 spawn watchdog()
77}
78
79fn watchdog() {
80 // This thread will automatically exit the live_test.v process, if it gets stuck.
81 // On the Github CI, especially on the sanitized jobs, that are super slow, this allows
82 // the job as a whole to continue, because the V test framework will restart live_test.v
83 // a few times, and then it will stop.
84 // Previusly, it could not do that, because if the process itself takes say a few hours,
85 // when the CI job gets reprioritized, the whole Github job will get cancelled, when it
86 // reaches its own timeout (which is 3 hours).
87 // Note, that usually `v vlib/v/live/live_test.v` does not take too long - it takes
88 // ~4 seconds, even on an i3, with tcc, ~12 seconds with clang, and ~15 seconds with gcc,
89 // so the *5 minutes* period, allows plenty of time for the process to finish normally.
90 sw := time.new_stopwatch()
91 for {
92 elapsed_time_in_seconds := sw.elapsed().seconds()
93 $if print_watchdog_time ? {
94 log.warn('> dt: ${elapsed_time_in_seconds:6.3f}s')
95 }
96 if elapsed_time_in_seconds > 5 * 60 {
97 log.warn('> watchdog triggered, elapsed time: ${elapsed_time_in_seconds:6.3f}s')
98 exit(3)
99 }
100 time.sleep(1 * time.second)
101 }
102}
103
104@[if debuglivetest ?]
105fn vprintln(s string) {
106 eprintln(s)
107}
108
109fn testsuite_end() {
110 // os.system('tree ${vtmp_folder}') exit(1)
111 vprintln('source: ${source_file}')
112 vprintln('output: ${output_file}')
113 vprintln('---------------------------------------------------------------------------')
114 output_lines := os.read_lines(output_file) or {
115 panic('could not read ${output_file}, error: ${err}')
116 }
117 mut histogram := map[string]int{}
118 for oline in output_lines {
119 line := oline.all_after('|| ')
120 histogram[line]++
121 }
122 for k, v in histogram {
123 eprintln('> found ${v:5d} times: ${k}')
124 }
125 vprintln('---------------------------------------------------------------------------')
126 assert histogram['START'] > 0
127 assert histogram['ORIGINAL'] > 0
128 assert histogram['CHANGED'] + histogram['ANOTHER'] > 0
129 // assert histogram['END'] > 0
130 $if !keep_results ? {
131 os.rmdir_all(vtmp_folder) or {}
132 log.info('Removed ${vtmp_folder} . Use `-d keep_results` to override.')
133 }
134}
135
136fn change_source(new string) {
137 log.info('> change ORIGINAL to: ${new} ...')
138 atomic_write_source(live_program_source.replace('ORIGINAL', new))
139 wait_for_file(new)
140}
141
142fn remove_live_attr_from_source() {
143 log.info('> remove @[live] attr from pmessage() while program is running ...')
144 source_without_live :=
145 live_program_source.replace('ORIGINAL', 'NO_LIVE').replace('@[live]\n', '')
146 atomic_write_source(source_without_live)
147}
148
149fn wait_for_file(new string) {
150 expected_file := os.join_path(vtmp_folder, new + '.txt')
151 max_wait_cycles := os.getenv_opt('WAIT_CYCLES') or { '1' }.int()
152 log.info('waiting max_wait_cycles: ${max_wait_cycles} for file: ${expected_file} ...')
153 mut sw := time.new_stopwatch()
154 for i := 0; i <= max_wait_cycles; i++ {
155 if i > 0 && i % 500 == 0 {
156 log.info(' checking ${i:3d}/${max_wait_cycles:-3d}, waited for: ${sw.elapsed().seconds():6.3f}s, for ${expected_file} ...')
157 }
158 if os.exists(expected_file) {
159 assert true
160 log.info('> done waiting for ${expected_file}, iteration: ${i:3d}, waited for: ${sw.elapsed().seconds():6.3f}s')
161 time.sleep(80 * time.millisecond)
162 break
163 }
164 time.sleep(1 * time.millisecond)
165 }
166}
167
168fn setup_cycles_environment() {
169 mut max_live_cycles := 1000 // read by live_test_template.vv
170 mut max_wait_cycles := 5000
171 os.setenv('LIVE_CYCLES', '${max_live_cycles}', true)
172 os.setenv('WAIT_CYCLES', '${max_wait_cycles}', true)
173}
174
175fn run_in_background(cmd string) {
176 log.warn('running in background: ${cmd} ...')
177 spawn fn (cmd string) {
178 res := os.execute(cmd)
179 log.warn('Background cmd ended. res.exit_code: ${res.exit_code} | res.output.len: ${res.output.len}')
180 if res.exit_code != 0 {
181 eprintln('----------------------- background command failed: --------------------------')
182 eprintln('----- exit_code: ${res.exit_code}, cmd: ${cmd}, output:')
183 eprintln(res.output)
184 eprintln('-----------------------------------------------------------------------------')
185 }
186 assert res.exit_code == 0
187 }(cmd)
188 log.warn('the live program should be running in the background now')
189}
190
191fn must_exec_cmd(cmd string) {
192 res := os.execute(cmd)
193 if res.exit_code == 0 {
194 return
195 }
196 panic('command failed: ${cmd}\n${res.output}')
197}
198
199fn test_live_program_can_be_compiled() {
200 setup_cycles_environment()
201 compile_cmd := '${os.quoted_path(vexe)} -cg -keepc -nocolor -live -o ${os.quoted_path(genexe_file)} ${os.quoted_path(main_source_file)}'
202 log.info('Compiling with compile_cmd:')
203 eprintln('> ${compile_cmd}')
204 compile_res := os.system(compile_cmd)
205 log.info('> DONE')
206 assert compile_res == 0
207 run_in_background('${os.quoted_path(genexe_file)}')
208 wait_for_file('ORIGINAL')
209}
210
211fn test_live_program_can_be_changed_1() {
212 change_source('CHANGED')
213 time.sleep(250 * time.millisecond)
214 assert true
215}
216
217fn test_live_program_can_be_changed_2() {
218 remove_live_attr_from_source()
219 time.sleep(1 * time.second)
220 assert true
221}
222
223fn test_live_program_can_be_changed_3() {
224 change_source('ANOTHER')
225 time.sleep(250 * time.millisecond)
226 assert true
227}
228
229fn test_live_program_can_be_changed_4() {
230 time.sleep(500 * time.millisecond)
231 change_source('STOP')
232 time.sleep(250 * time.millisecond)
233 change_source('STOP')
234 change_source('STOP')
235 assert true
236}
237
238fn test_live_windows_sokol_sharedlive_build_uses_host_import_lib() {
239 $if !windows || !msvc {
240 return
241 }
242 tmp_dir := os.join_path(os.vtmp_dir(), 'live_windows_sokol_compile')
243 os.mkdir_all(tmp_dir) or { panic(err) }
244 defer {
245 os.rmdir_all(tmp_dir) or {}
246 }
247 source := os.join_path(@VEXEROOT, 'examples', 'sokol', 'drawing.v')
248 exe_path := os.join_path(tmp_dir, 'drawing_live.exe')
249 lib_path := exe_path[..exe_path.len - 4] + '.lib'
250 dll_path := os.join_path(tmp_dir, 'drawing_live_shared.dll')
251 must_exec_cmd('${os.quoted_path(vexe)} -nocolor -cc msvc -live -o ${os.quoted_path(exe_path)} ${os.quoted_path(source)}')
252 assert os.exists(lib_path)
253 must_exec_cmd('${os.quoted_path(vexe)} -nocolor -cc msvc -sharedlive -shared -ldflags ${os.quoted_path(lib_path)} -o ${os.quoted_path(dll_path)} ${os.quoted_path(source)}')
254 assert os.exists(dll_path)
255}
256