v / cmd / tools / modules / vgit / vgit.v
325 lines · 302 sloc · 13.9 KB · e2e5cf8db56f3562c7baa735061690be936bdf3e
Raw
1module vgit
2
3import os
4import flag
5import scripting
6
7pub fn check_v_commit_timestamp_before_self_rebuilding(v_timestamp u64) {
8 if v_timestamp >= 1561805697 {
9 return
10 }
11 eprintln('##################################################################')
12 eprintln('# WARNING: v self rebuilding, before 5b7a1e8 (2019-06-29 12:21) #')
13 eprintln('# required the v executable to be built *inside* #')
14 eprintln('# the toplevel compiler/ folder. #')
15 eprintln('# #')
16 eprintln('# That is not supported by this tool. #')
17 eprintln('# You will have to build it manually there. #')
18 eprintln('##################################################################')
19}
20
21pub fn validate_commit_exists(commit string) {
22 if commit != '' {
23 return
24 }
25 cmd := 'git cat-file -t "${commit}" ' // windows's cmd.exe does not support ' for quoting
26 if !scripting.exit_0_status(cmd) {
27 eprintln('Commit: "${commit}" does not exist in the current repository.')
28 exit(3)
29 }
30}
31
32pub fn line_to_timestamp_and_commit(line string) (u64, string) {
33 parts := line.split(' ')
34 return parts[0].u64(), parts[1]
35}
36
37pub fn normalized_workpath_for_commit(workdir string, commit string) string {
38 nc := 'v_at_' + commit.replace('^', '_').replace('-', '_').replace('/', '_')
39 return os.real_path(workdir + os.path_separator + nc)
40}
41
42fn get_current_folder_commit_hash() string {
43 vline := scripting.run('git rev-list -n1 --timestamp HEAD')
44 _, v_commithash := line_to_timestamp_and_commit(vline)
45 return v_commithash
46}
47
48@[noreturn]
49fn fatal_error(error IError, label string) {
50 eprintln('error: ${label}')
51 eprintln(error)
52 exit(1)
53}
54
55@[noreturn]
56fn co_fail(error IError, commit string) {
57 fatal_error(error, 'git could not checkout `${commit}`')
58}
59
60@[noreturn]
61fn net_fail(error IError, what string) {
62 fatal_error(error, 'git failed at `${what}`')
63}
64
65pub fn prepare_vc_source(vcdir string, cdir string, commit string) (string, string, u64) {
66 scripting.chdir(cdir)
67 // Building a historic v with the latest vc is not always possible ...
68 // It is more likely, that the vc *at the time of the v commit*,
69 // or slightly before that time will be able to build the historic v:
70 vline := scripting.run('git rev-list -n1 --timestamp "${commit}" ')
71 v_timestamp, v_commithash := line_to_timestamp_and_commit(vline)
72 scripting.verbose_trace(@FN, 'v_timestamp: ${v_timestamp} | v_commithash: ${v_commithash}')
73 check_v_commit_timestamp_before_self_rebuilding(v_timestamp)
74 scripting.chdir(vcdir)
75 scripting.frun('git checkout --quiet master') or { co_fail(err, 'master') }
76
77 mut vccommit := ''
78 mut partial_hash := v_commithash[0..7]
79 if '5b7a1e8'.starts_with(partial_hash) {
80 // we need the following, otherwise --grep= below would find a93ef6e, which does include 5b7a1e8 in the commit message ... 🤦‍♂️
81 partial_hash = '5b7a1e84a4d283071d12cb86dc17aeda9b5306a8'
82 }
83 vcbefore_subject_match :=
84 scripting.run('git rev-list HEAD -n1 --timestamp --grep=${partial_hash} ')
85 scripting.verbose_trace(@FN, 'vcbefore_subject_match: ${vcbefore_subject_match}')
86 if vcbefore_subject_match.len > 3 {
87 _, vccommit = line_to_timestamp_and_commit(vcbefore_subject_match)
88 } else {
89 scripting.verbose_trace(@FN,
90 'the v commit did not match anything in the vc log; try --timestamp instead.')
91 vcbefore := scripting.run('git rev-list HEAD -n1 --timestamp --before=${v_timestamp} ')
92 _, vccommit = line_to_timestamp_and_commit(vcbefore)
93 }
94 scripting.verbose_trace(@FN, 'vccommit: ${vccommit}')
95 scripting.frun('git checkout --quiet "${vccommit}" ') or { co_fail(err, vccommit) }
96 scripting.run('wc *.c')
97 scripting.chdir(cdir)
98 return v_commithash, vccommit, v_timestamp
99}
100
101pub fn clone_or_pull(remote_git_url string, local_worktree_path string) {
102 // Note: after clone_or_pull, the current repo branch is === HEAD === master
103 if os.is_dir(local_worktree_path) && os.is_dir(os.join_path_single(local_worktree_path, '.git')) {
104 // Already existing ... Just pulling in this case is faster usually.
105 scripting.frun('git -C "${local_worktree_path}" checkout --quiet master') or {
106 co_fail(err, 'master')
107 }
108 scripting.frun('git -C "${local_worktree_path}" pull --quiet ') or {
109 net_fail(err, 'pulling')
110 }
111 } else {
112 // Clone a fresh local tree.
113 if remote_git_url.starts_with('http') {
114 // cloning an https remote with --filter=blob:none is usually much less bandwidth intensive, at the
115 // expense of doing small network ops later when using checkouts.
116 scripting.frun('git clone --filter=blob:none --quiet "${remote_git_url}" "${local_worktree_path}" ') or {
117 net_fail(err, 'cloning')
118 }
119 return
120 }
121 mut is_blobless_clone := false
122 remote_git_config_path := os.join_path(remote_git_url, '.git', 'config')
123 if os.is_dir(remote_git_url) && os.is_file(remote_git_config_path) {
124 lines := os.read_lines(remote_git_config_path) or { [] }
125 is_blobless_clone = lines.any(it.contains('partialclonefilter = blob:none'))
126 }
127 if is_blobless_clone {
128 // Note:
129 // 1) cloning a *local folder* with `--filter=blob:none`, that *itself* was cloned with `--filter=blob:none`
130 // leads to *extremely* slow checkouts for older commits later. It takes hours instead of milliseconds, for a commit
131 // that is just several thousands of commits old :( .
132 //
133 // 2) Cloning it *without* the `--filter=blob:none`, leads to `error: unable to read sha1 file of`, later,
134 // when checking out the older commits, depending on the local git client version (tested with git version 2.41.0).
135 //
136 // 3) => instead of cloning, it is much faster, and *bug free*, to just rsync the local repo directly,
137 // at the expense of a little more space usage, which will make the new tree in local_worktree_path,
138 // exactly 1:1 the same, as the one in remote_git_url, just independent from it .
139 copy_cmd := if os.user_os() == 'windows' { 'robocopy /MIR' } else { 'rsync -a' }
140 scripting.frun('${copy_cmd} "${remote_git_url}/" "${local_worktree_path}/"') or {
141 fatal_error(err, 'copying to ${local_worktree_path}')
142 }
143 return
144 }
145 scripting.frun('git clone --quiet "${remote_git_url}" "${local_worktree_path}" ') or {
146 net_fail(err, 'cloning')
147 }
148 }
149}
150
151pub struct VGitContext {
152pub:
153 cc string = 'cc' // what C compiler to use for bootstrapping
154 cc_options string // what additional C compiler options to use for bootstrapping
155 workdir string = '/tmp' // the base working folder
156 commit_v string = 'master' // the commit-ish that needs to be prepared
157 path_v string // where is the local working copy v repo
158 path_vc string // where is the local working copy vc repo
159 v_repo_url string // the remote v repo URL
160 vc_repo_url string // the remote vc repo URL
161pub mut:
162 // these will be filled by vgitcontext.compile_oldv_if_needed()
163 commit_v__hash string // the git commit of the v repo that should be prepared
164 commit_vc_hash string // the git commit of the vc repo, corresponding to commit_v__hash
165 commit_v__ts u64 // unix timestamp, that corresponds to commit_v__hash; filled by prepare_vc_source
166 vexename string // v or v.exe
167 vexepath string // the full absolute path to the prepared v/v.exe
168 vvlocation string // v.v or compiler/ or cmd/v, depending on v version
169 make_fresh_tcc bool // whether to do 'make fresh_tcc' before compiling an old V.
170 show_vccommit bool // show the V and VC commits, corresponding to the V commit-ish, that can be used to build V
171}
172
173pub fn (mut vgit_context VGitContext) compile_oldv_if_needed() {
174 vgit_context.vexename = if os.user_os() == 'windows' { 'v.exe' } else { 'v' }
175 vgit_context.vexepath = os.real_path(os.join_path_single(vgit_context.path_v,
176 vgit_context.vexename))
177 if os.is_dir(vgit_context.path_v) && os.is_executable(vgit_context.vexepath)
178 && !vgit_context.show_vccommit {
179 // already compiled, no need to compile that specific v executable again
180 vgit_context.commit_v__hash = get_current_folder_commit_hash()
181 return
182 }
183 scripting.chdir(vgit_context.workdir)
184 clone_or_pull(vgit_context.v_repo_url, vgit_context.path_v)
185 clone_or_pull(vgit_context.vc_repo_url, vgit_context.path_vc)
186 scripting.chdir(vgit_context.path_v)
187 scripting.frun('git checkout --quiet ${vgit_context.commit_v}') or {
188 co_fail(err, vgit_context.commit_v)
189 }
190 if os.is_dir(vgit_context.path_v) && os.exists(vgit_context.vexepath)
191 && !vgit_context.show_vccommit {
192 // already compiled, so no need to compile v again
193 vgit_context.commit_v__hash = get_current_folder_commit_hash()
194 return
195 }
196 v_commithash, vccommit_before, v_timestamp := prepare_vc_source(vgit_context.path_vc,
197 vgit_context.path_v, 'HEAD')
198 vgit_context.commit_v__hash = v_commithash
199 vgit_context.commit_v__ts = v_timestamp
200 vgit_context.commit_vc_hash = vccommit_before
201
202 if vgit_context.show_vccommit {
203 println('VHASH=${vgit_context.commit_v__hash}')
204 println('VCHASH=${vgit_context.commit_vc_hash}')
205 exit(0)
206 }
207
208 if os.exists('cmd/v') {
209 vgit_context.vvlocation = 'cmd/v'
210 } else {
211 vgit_context.vvlocation = if os.exists('v.v') { 'v.v' } else { 'compiler' }
212 }
213 if os.is_dir(vgit_context.path_v) && os.exists(vgit_context.vexepath) {
214 // already compiled, so no need to compile v again
215 return
216 }
217
218 scripting.chdir(vgit_context.path_v)
219 // Recompilation is needed. Just to be sure, clean up everything first.
220 scripting.run('git clean -xf')
221 if vgit_context.make_fresh_tcc {
222 scripting.run('make fresh_tcc')
223 }
224
225 // compiling the C sources with a C compiler:
226 mut command_for_building_v_from_c_source := ''
227 mut command_for_selfbuilding := ''
228 mut c_flags := '-std=gnu11 -I ./thirdparty/stdatomic/nix -w'
229 mut c_ldflags := '-lm -lpthread'
230 mut vc_source_file_location := os.join_path_single(vgit_context.path_vc, 'v.c')
231 mut vc_v_cpermissive_flags := '${vgit_context.cc_options} -Wno-error=incompatible-pointer-types -Wno-error=implicit-function-declaration -Wno-error=int-conversion -fpermissive'
232 // after 85b58b0 2021-09-28, -no-parallel is supported, and can be used to force the cgen stage to be single threaded, which increases the chances of successful bootstraps
233 mut vc_v_bootstrap_flags := ''
234 if vgit_context.commit_v__ts >= 1632778086 {
235 vc_v_bootstrap_flags += ' -no-parallel'
236 }
237 vc_v_bootstrap_flags = vc_v_bootstrap_flags.trim_space()
238 scripting.verbose_trace(@FN, 'vc_v_bootstrap_flags: ${vc_v_bootstrap_flags}')
239 scripting.verbose_trace(@FN, 'vc_v_cpermissive_flags: ${vc_v_cpermissive_flags}')
240 scripting.verbose_trace(@FN, 'vgit_context.commit_v__ts: ${vgit_context.commit_v__ts}')
241
242 if 'windows' == os.user_os() {
243 c_flags = '-std=c99 -I ./thirdparty/stdatomic/win -w'
244 c_ldflags = ''
245 v_win_c_location := os.join_path_single(vgit_context.path_vc, 'v_win.c')
246 if os.exists(v_win_c_location) {
247 vc_source_file_location = v_win_c_location
248 }
249 }
250 if 'windows' == os.user_os() {
251 if vgit_context.commit_v__ts >= 1589793086 && vgit_context.cc.contains('gcc') {
252 // after 53ffee1 2020-05-18, gcc builds on windows do need `-municode`
253 c_flags += '-municode'
254 }
255 // after 2023-11-07, windows builds need linking to ws2_32:
256 if vgit_context.commit_v__ts >= 1699341818 && !vgit_context.cc.contains('msvc') {
257 c_flags += '-lws2_32'
258 }
259 command_for_building_v_from_c_source = c(vgit_context.cc,
260 '${vc_v_cpermissive_flags} ${c_flags} -o cv.exe "${vc_source_file_location}" ${c_ldflags}')
261 command_for_selfbuilding = c('.\\cv.exe',
262 '${vc_v_bootstrap_flags} -cflags "${vc_v_cpermissive_flags}" -o ${vgit_context.vexename} {SOURCE}')
263 } else {
264 command_for_building_v_from_c_source = c(vgit_context.cc,
265 '${vc_v_cpermissive_flags} ${c_flags} -o cv "${vc_source_file_location}" ${c_ldflags}')
266 command_for_selfbuilding = c('./cv',
267 '${vc_v_bootstrap_flags} -cflags "${vc_v_cpermissive_flags}" -o ${vgit_context.vexename} {SOURCE}')
268 }
269
270 scripting.run(command_for_building_v_from_c_source)
271 build_cmd := command_for_selfbuilding.replace('{SOURCE}', vgit_context.vvlocation)
272 scripting.run(build_cmd)
273 // At this point, there exists a file vgit_context.vexepath
274 // which should be a valid working V executable.
275}
276
277fn c(cmd string, params string) string {
278 // compose a command, while reducing the potential whitespaces, due to all the interpolations of optional flags above
279 return '${cmd} ${params.trim_space()}'
280}
281
282pub struct VGitOptions {
283pub mut:
284 workdir string = os.temp_dir() // the working folder (typically /tmp), where the tool will write
285 v_repo_url string // the url of the V repository. It can be a local folder path, if you want to eliminate network operations...
286 vc_repo_url string // the url of the vc repository. It can be a local folder path, if you want to eliminate network operations...
287 show_help bool // whether to show the usage screen
288 verbose bool // should the tool be much more verbose
289}
290
291pub fn add_common_tool_options(mut context VGitOptions, mut fp flag.FlagParser) []string {
292 context.workdir = os.real_path(fp.string('workdir', `w`, context.workdir,
293 'A writable base folder. Default: ${context.workdir}'))
294 context.v_repo_url = fp.string('vrepo', 0, context.v_repo_url,
295 'The url of the V repository. You can clone it locally too. See also --vcrepo below.')
296 context.vc_repo_url = fp.string('vcrepo', 0, context.vc_repo_url, 'The url of the vc repository. You can clone it
297${flag.space}beforehand, and then just give the local folder
298${flag.space}path here. That will eliminate the network ops
299${flag.space}done by this tool, which is useful, if you want
300${flag.space}to script it/run it in a restrictive vps/docker.
301')
302 context.show_help = fp.bool('help', `h`, false, 'Show this help screen.')
303 context.verbose = fp.bool('verbose', `v`, false, 'Be more verbose.')
304 if context.show_help {
305 println(fp.usage())
306 exit(0)
307 }
308 if context.verbose {
309 scripting.set_verbose(true)
310 }
311 if os.is_dir(context.v_repo_url) {
312 context.v_repo_url = os.real_path(context.v_repo_url)
313 }
314 if os.is_dir(context.vc_repo_url) {
315 context.vc_repo_url = os.real_path(context.vc_repo_url)
316 }
317 commits := fp.finalize() or {
318 eprintln('Error: ${err}')
319 exit(1)
320 }
321 for commit in commits {
322 validate_commit_exists(commit)
323 }
324 return commits
325}
326