v / cmd / tools / performance_compare.v
222 lines · 215 sloc · 8.74 KB · e2e5cf8db56f3562c7baa735061690be936bdf3e
Raw
1import os
2import flag
3import scripting
4import vgit
5
6const tool_version = '0.0.6'
7const tool_description = " Compares V executable size and performance,
8| between 2 commits from V's local git history.
9| When only one commit is given, it is compared to master.
10| ".strip_margin()
11
12struct Context {
13 cwd string // current working folder
14mut:
15 vgo vgit.VGitOptions
16 a string // the full path to the 'after' folder inside workdir
17 b string // the full path to the 'before' folder inside workdir
18 vc string // the full path to the vc folder inside workdir. It is used during bootstrapping v from the C source.
19 commit_before string // the git commit for the 'before' state
20 commit_after string // the git commit for the 'after' state
21 warmups int // how many times to execute a command before gathering stats
22 hyperfineopts string // use for additional CLI options that will be given to the hyperfine command
23 vflags string // other v options to pass to compared v commands
24}
25
26fn new_context() Context {
27 return Context{
28 cwd: os.getwd()
29 commit_after: 'master'
30 warmups: 4
31 }
32}
33
34fn (c Context) compare_versions() {
35 // Input is validated at this point...
36 // Cleanup artifacts from previous runs of this tool:
37 scripting.chdir(c.vgo.workdir)
38 scripting.run('rm -rf "${c.a}" "${c.b}" "${c.vc}" ')
39 // clone the VC source *just once per comparison*, and reuse it:
40 scripting.run('git clone --filter=blob:none --quiet "${c.vgo.vc_repo_url}" "${c.vc}" ')
41 println('Comparing V performance of commit ${c.commit_before} (before) vs commit ${c.commit_after} (after) ...')
42 c.prepare_v(c.b, c.commit_before)
43 c.prepare_v(c.a, c.commit_after)
44 scripting.chdir(c.vgo.workdir)
45 if c.vflags.len > 0 {
46 os.setenv('VFLAGS', c.vflags, true)
47 }
48 // The first is the baseline, against which all the others will be compared.
49 // It is the fastest, since hello_world.v has only a single println in it,
50 mut perf_files := []string{}
51 perf_files << c.compare_v_performance('source_hello', [
52 'vprod @DEBUG@ -o source.c examples/hello_world.v',
53 'vprod -o source.c examples/hello_world.v',
54 'v @DEBUG@ -o source.c examples/hello_world.v',
55 'v -o source.c examples/hello_world.v',
56 ])
57 perf_files << c.compare_v_performance('source_v', [
58 'vprod @DEBUG@ -o source.c @COMPILER@',
59 'vprod -o source.c @COMPILER@',
60 'v @DEBUG@ -o source.c @COMPILER@',
61 'v -o source.c @COMPILER@',
62 ])
63 perf_files << c.compare_v_performance('binary_hello', [
64 'vprod -o hello examples/hello_world.v',
65 'v -o hello examples/hello_world.v',
66 ])
67 perf_files << c.compare_v_performance('binary_v', [
68 'vprod -o binary @COMPILER@',
69 'v -o binary @COMPILER@',
70 ])
71 println('All performance files:')
72 for f in perf_files {
73 println(' ${f}')
74 }
75}
76
77fn (c &Context) prepare_v(cdir string, commit string) {
78 mut cc := os.getenv('CC')
79 if cc == '' {
80 cc = 'cc'
81 }
82 mut vgit_context := vgit.VGitContext{
83 cc: cc
84 commit_v: commit
85 path_v: cdir
86 path_vc: c.vc
87 workdir: c.vgo.workdir
88 v_repo_url: c.vgo.v_repo_url
89 vc_repo_url: c.vgo.vc_repo_url
90 }
91 vgit_context.compile_oldv_if_needed()
92 scripting.chdir(cdir)
93 scripting.run('${cdir}/v version')
94 println('Making a v compiler in ${cdir}')
95 scripting.run('./v -cc ${cc} -o v ${vgit_context.vvlocation}')
96 println('Making a vprod compiler in ${cdir}')
97 scripting.run('./v -cc ${cc} -prod -o vprod ${vgit_context.vvlocation}')
98 println('Stripping and compressing cv v and vprod binaries in ${cdir}')
99 scripting.run('cp cv cv_stripped')
100 scripting.run('cp v v_stripped')
101 scripting.run('cp vprod vprod_stripped')
102 scripting.run('strip *_stripped')
103 scripting.run('cp cv_stripped cv_stripped_upxed')
104 scripting.run('cp v_stripped v_stripped_upxed')
105 scripting.run('cp vprod_stripped vprod_stripped_upxed')
106 scripting.run('upx -qqq --lzma cv_stripped_upxed')
107 scripting.run('upx -qqq --lzma v_stripped_upxed')
108 scripting.run('upx -qqq --lzma vprod_stripped_upxed')
109 scripting.show_sizes_of_files(['${cdir}/cv', '${cdir}/cv_stripped', '${cdir}/cv_stripped_upxed'])
110 scripting.show_sizes_of_files(['${cdir}/v', '${cdir}/v_stripped', '${cdir}/v_stripped_upxed'])
111 scripting.show_sizes_of_files(['${cdir}/vprod', '${cdir}/vprod_stripped',
112 '${cdir}/vprod_stripped_upxed'])
113 vversion := scripting.run('${cdir}/v -version')
114 vcommit := scripting.run('git rev-parse --short --verify HEAD')
115 println('V version is: ${vversion} , local source commit: ${vcommit}')
116 if vgit_context.vvlocation == 'cmd/v' {
117 if os.exists('vlib/v/ast/ast.v') {
118 println('Source lines of the compiler: ' +
119 scripting.run('find cmd/v/ vlib/v/ -name "*.v" | grep -v /tests/ | xargs wc | tail -n -1'))
120 } else {
121 println('Source lines of the compiler: ' +
122 scripting.run('wc cmd/v/*.v vlib/compiler/*.v | tail -n -1'))
123 }
124 } else if vgit_context.vvlocation == 'v.v' {
125 println('Source lines of the compiler: ' +
126 scripting.run('wc v.v vlib/compiler/*.v | tail -n -1'))
127 } else {
128 println('Source lines of the compiler: ' + scripting.run('wc compiler/*.v | tail -n -1'))
129 }
130}
131
132fn (c Context) compare_v_performance(label string, commands []string) string {
133 println('---------------------------------------------------------------------------------')
134 println('Compare v performance when doing the following commands (${label}):')
135 mut source_location_a := ''
136 mut source_location_b := ''
137 if os.exists('${c.a}/cmd/v') {
138 source_location_a = 'cmd/v'
139 } else {
140 source_location_a = if os.exists('${c.a}/v.v') { 'v.v ' } else { 'compiler/ ' }
141 }
142 if os.exists('${c.b}/cmd/v') {
143 source_location_b = 'cmd/v'
144 } else {
145 source_location_b = if os.exists('${c.b}/v.v') { 'v.v ' } else { 'compiler/ ' }
146 }
147 timestamp_a, _ :=
148 vgit.line_to_timestamp_and_commit(scripting.run('cd ${c.a}/ ; git rev-list -n1 --timestamp HEAD'))
149 timestamp_b, _ :=
150 vgit.line_to_timestamp_and_commit(scripting.run('cd ${c.b}/ ; git rev-list -n1 --timestamp HEAD'))
151 // 1570877641 is 065ce39 2019-10-12
152 debug_option_a := if timestamp_a > 1570877641 { '-cg ' } else { '-debug ' }
153 debug_option_b := if timestamp_b > 1570877641 { '-cg ' } else { '-debug ' }
154 mut hyperfine_commands_arguments := []string{}
155 for cmd in commands {
156 println(cmd)
157 }
158 for cmd in commands {
159 hyperfine_commands_arguments << ' \'cd ${c.b:-34s} ; ./${cmd} \' '.replace_each([
160 '@COMPILER@',
161 source_location_b,
162 '@DEBUG@',
163 debug_option_b,
164 ])
165 }
166 for cmd in commands {
167 hyperfine_commands_arguments << ' \'cd ${c.a:-34s} ; ./${cmd} \' '.replace_each([
168 '@COMPILER@',
169 source_location_a,
170 '@DEBUG@',
171 debug_option_a,
172 ])
173 }
174 // /////////////////////////////////////////////////////////////////////////////
175 cmd_stats_file :=
176 os.real_path([c.vgo.workdir, 'v_performance_stats_${label}.json'].join(os.path_separator))
177 comparison_cmd := 'hyperfine ${c.hyperfineopts} ' + '--export-json ${cmd_stats_file} ' +
178 '--time-unit millisecond ' + '--style full --warmup ${c.warmups} ' +
179 hyperfine_commands_arguments.join(' ')
180 // /////////////////////////////////////////////////////////////////////////////
181 if c.vgo.verbose {
182 println(comparison_cmd)
183 }
184 os.system(comparison_cmd)
185 println('The detailed performance comparison report was saved to: ${cmd_stats_file} .')
186 println('')
187 return cmd_stats_file
188}
189
190fn main() {
191 // allow for `v run cmd/tools/performance_compare.v`, see oldv.v
192 os.setenv('VEXE', '', true)
193 scripting.used_tools_must_exist(['cp', 'rm', 'strip', 'make', 'git', 'upx', 'cc', 'wc', 'tail',
194 'find', 'xargs', 'hyperfine'])
195 mut context := new_context()
196 mut fp := flag.new_flag_parser(os.args)
197 fp.application(os.file_name(os.executable()))
198 fp.version(tool_version)
199 fp.description(tool_description)
200 fp.arguments_description('COMMIT_BEFORE [COMMIT_AFTER]')
201 fp.skip_executable()
202 fp.limit_free_args(1, 2)!
203 context.vflags = fp.string('vflags', 0, '',
204 'Additional options to pass to the v commands, for example "-cc tcc"')
205 context.hyperfineopts = fp.string('hyperfine_options', 0, '', 'Additional options passed to hyperfine.
206${flag.space}For example on linux, you may want to pass:
207${flag.space}--hyperfine_options "--prepare \'sync; echo 3 | sudo tee /proc/sys/vm/drop_caches\'"
208')
209 commits := vgit.add_common_tool_options(mut context.vgo, mut fp)
210 context.commit_before = commits[0]
211 if commits.len > 1 {
212 context.commit_after = commits[1]
213 }
214 context.b = vgit.normalized_workpath_for_commit(context.vgo.workdir, context.commit_before)
215 context.a = vgit.normalized_workpath_for_commit(context.vgo.workdir, context.commit_after)
216 context.vc = vgit.normalized_workpath_for_commit(context.vgo.workdir, 'vc')
217 if !os.is_dir(context.vgo.workdir) {
218 eprintln('Work folder: `{context.vgo.workdir}` , does not exist. Use `--workdir /some/path` to set it.')
219 exit(2)
220 }
221 context.compare_versions()
222}
223