| 1 | module main |
| 2 | |
| 3 | import os |
| 4 | import log |
| 5 | import flag |
| 6 | import time |
| 7 | import veb |
| 8 | import net.urllib |
| 9 | |
| 10 | // This tool regenerates V's bootstrap .c files |
| 11 | // every time the V master branch is updated. |
| 12 | // if run with the --serve flag it will run in webhook |
| 13 | // server mode awaiting a request to http://host:port/genhook |
| 14 | // available command line flags: |
| 15 | // --work-dir gen_vc's working directory |
| 16 | // --purge force purge the local repositories |
| 17 | // --serve run in webhook server mode |
| 18 | // --port port for http server to listen on |
| 19 | // --log-to either 'file' or 'terminal' |
| 20 | // --log-file path to log file used when --log-to is 'file' |
| 21 | // --dry-run dont push anything to remote repo |
| 22 | // --force force update even if already up to date |
| 23 | |
| 24 | // git credentials |
| 25 | const git_username = os.getenv('GITUSER') |
| 26 | const git_password = os.getenv('GITPASS') |
| 27 | |
| 28 | // repository |
| 29 | // git repo |
| 30 | const git_repo_v = 'github.com/vlang/v' |
| 31 | const git_repo_vc = 'github.com/vlang/vc' |
| 32 | // local repo directories |
| 33 | const git_repo_dir_v = 'v' |
| 34 | const git_repo_dir_vc = 'vc' |
| 35 | |
| 36 | // gen_vc |
| 37 | // name |
| 38 | const app_name = 'gen_vc' |
| 39 | // version |
| 40 | const app_version = '0.1.3' |
| 41 | // description |
| 42 | const app_description = "This tool regenerates V's bootstrap .c files every time the V master branch is updated." |
| 43 | // assume something went wrong if file size less than this |
| 44 | const too_short_file_limit = 5000 |
| 45 | // create a .c file for these os's |
| 46 | const vc_build_oses = [ |
| 47 | 'nix', |
| 48 | // all nix based os |
| 49 | 'windows', |
| 50 | ] |
| 51 | |
| 52 | // default options (overridden by flags) |
| 53 | // gen_vc working directory |
| 54 | const work_dir = '/tmp/gen_vc' |
| 55 | // dont push anything to remote repo |
| 56 | const dry_run = false |
| 57 | // server port |
| 58 | const server_port = 7171 |
| 59 | // log file |
| 60 | const log_file = '${work_dir}/log.txt' |
| 61 | // log_to is either 'file' or 'terminal' |
| 62 | const log_to = 'terminal' |
| 63 | |
| 64 | // errors |
| 65 | const err_msg_build = 'error building' |
| 66 | const err_msg_make = 'make failed' |
| 67 | const err_msg_gen_c = 'failed to generate .c file' |
| 68 | const err_msg_cmd_x = 'error running cmd' |
| 69 | |
| 70 | struct GenVC { |
| 71 | // logger |
| 72 | // flag options |
| 73 | options FlagOptions |
| 74 | mut: |
| 75 | logger &log.Log = unsafe { nil } |
| 76 | // true if error was experienced running generate |
| 77 | gen_error bool |
| 78 | } |
| 79 | |
| 80 | // webhook server |
| 81 | struct WebhookServer { |
| 82 | mut: |
| 83 | gen_vc &GenVC = unsafe { nil } // initialized in init_server |
| 84 | } |
| 85 | |
| 86 | struct Context { |
| 87 | veb.Context |
| 88 | } |
| 89 | |
| 90 | // storage for flag options |
| 91 | struct FlagOptions { |
| 92 | work_dir string |
| 93 | purge bool |
| 94 | serve bool |
| 95 | port int |
| 96 | log_to string |
| 97 | log_file string |
| 98 | dry_run bool |
| 99 | force bool |
| 100 | } |
| 101 | |
| 102 | fn main() { |
| 103 | log.use_stdout() |
| 104 | mut fp := flag.new_flag_parser(os.args.clone()) |
| 105 | fp.application(app_name) |
| 106 | fp.version(app_version) |
| 107 | fp.description(app_description) |
| 108 | fp.skip_executable() |
| 109 | show_help := fp.bool('help', 0, false, 'Show this help screen\n') |
| 110 | flag_options := parse_flags(mut fp) |
| 111 | if show_help { |
| 112 | println(fp.usage()) |
| 113 | exit(0) |
| 114 | } |
| 115 | fp.finalize() or { |
| 116 | eprintln(err) |
| 117 | println(fp.usage()) |
| 118 | return |
| 119 | } |
| 120 | // webhook server mode |
| 121 | if flag_options.serve { |
| 122 | mut server := &WebhookServer{} |
| 123 | veb.run_at[WebhookServer, Context](mut server, port: flag_options.port)! |
| 124 | } else { |
| 125 | // cmd mode |
| 126 | mut gen_vc := new_gen_vc(flag_options) |
| 127 | gen_vc.init() |
| 128 | gen_vc.generate() |
| 129 | } |
| 130 | } |
| 131 | |
| 132 | // new GenVC |
| 133 | fn new_gen_vc(flag_options FlagOptions) &GenVC { |
| 134 | mut logger := &log.Log{} |
| 135 | logger.set_level(.debug) |
| 136 | if flag_options.log_to == 'file' { |
| 137 | logger.set_full_logpath(flag_options.log_file) |
| 138 | } |
| 139 | return &GenVC{ |
| 140 | options: flag_options |
| 141 | logger: logger |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | // WebhookServer init |
| 146 | pub fn (mut ws WebhookServer) init_server() { |
| 147 | mut fp := flag.new_flag_parser(os.args.clone()) |
| 148 | flag_options := parse_flags(mut fp) |
| 149 | ws.gen_vc = new_gen_vc(flag_options) |
| 150 | ws.gen_vc.init() |
| 151 | // ws.gen_vc = new_gen_vc(flag_options) |
| 152 | } |
| 153 | |
| 154 | pub fn (mut ws WebhookServer) index() { |
| 155 | eprintln('WebhookServer.index() called') |
| 156 | } |
| 157 | |
| 158 | // gen webhook |
| 159 | pub fn (mut ws WebhookServer) genhook() veb.Result { |
| 160 | // request data |
| 161 | // println(ws.req.data) |
| 162 | // TODO: parse request. json or urlencoded |
| 163 | // json.decode or net.urllib.parse |
| 164 | ws.gen_vc.generate() |
| 165 | // error in generate |
| 166 | if ws.gen_vc.gen_error { |
| 167 | return ctx.json('{status: "failed"}') |
| 168 | } |
| 169 | return ctx.json('{status: "ok"}') |
| 170 | } |
| 171 | |
| 172 | pub fn (ws &WebhookServer) reset() { |
| 173 | } |
| 174 | |
| 175 | // parse flags to FlagOptions struct |
| 176 | fn parse_flags(mut fp flag.FlagParser) FlagOptions { |
| 177 | return FlagOptions{ |
| 178 | serve: fp.bool('serve', 0, false, 'run in webhook server mode') |
| 179 | work_dir: fp.string('work-dir', 0, work_dir, 'gen_vc working directory') |
| 180 | purge: fp.bool('purge', 0, false, 'force purge the local repositories') |
| 181 | port: fp.int('port', 0, server_port, 'port for web server to listen on') |
| 182 | log_to: fp.string('log-to', 0, log_to, "log to is 'file' or 'terminal'") |
| 183 | log_file: fp.string('log-file', 0, log_file, "log file to use when log-to is 'file'") |
| 184 | dry_run: fp.bool('dry-run', 0, dry_run, 'when specified dont push anything to remote repo') |
| 185 | force: fp.bool('force', 0, false, 'force update even if already up to date') |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | fn (mut gen_vc GenVC) init() { |
| 190 | // purge repos if flag is passed |
| 191 | if gen_vc.options.purge { |
| 192 | gen_vc.purge_repos() |
| 193 | } |
| 194 | } |
| 195 | |
| 196 | // regenerate |
| 197 | fn (mut gen_vc GenVC) generate() { |
| 198 | // set errors to false |
| 199 | gen_vc.gen_error = false |
| 200 | // check if gen_vc dir exists |
| 201 | if !os.is_dir(gen_vc.options.work_dir) { |
| 202 | // try create |
| 203 | os.mkdir(gen_vc.options.work_dir) or { panic(err) } |
| 204 | // still doesn't exist... we have a problem |
| 205 | if !os.is_dir(gen_vc.options.work_dir) { |
| 206 | gen_vc.logger.error('error creating directory: ${gen_vc.options.work_dir}') |
| 207 | gen_vc.gen_error = true |
| 208 | return |
| 209 | } |
| 210 | } |
| 211 | // cd to gen_vc dir |
| 212 | os.chdir(gen_vc.options.work_dir) or {} |
| 213 | // if we are not running with the --serve flag (webhook server) |
| 214 | // rather than deleting and re-downloading the repo each time |
| 215 | // first check to see if the local v repo is behind master |
| 216 | // if it isn't behind there's no point continuing further |
| 217 | if !gen_vc.options.serve && os.is_dir(git_repo_dir_v) { |
| 218 | gen_vc.cmd_exec('git -C ${git_repo_dir_v} checkout master') |
| 219 | // fetch the remote repo just in case there are newer commits there |
| 220 | gen_vc.cmd_exec('git -C ${git_repo_dir_v} fetch') |
| 221 | git_status := gen_vc.cmd_exec('git -C ${git_repo_dir_v} status') |
| 222 | if !git_status.contains('behind') && !gen_vc.options.force { |
| 223 | gen_vc.logger.warn('v repository is already up to date.') |
| 224 | return |
| 225 | } |
| 226 | } |
| 227 | // delete repos |
| 228 | gen_vc.purge_repos() |
| 229 | // clone repos |
| 230 | gen_vc.cmd_exec('git clone --filter=blob:none https://${git_repo_v} ${git_repo_dir_v}') |
| 231 | gen_vc.cmd_exec('git clone --filter=blob:none https://${git_repo_vc} ${git_repo_dir_vc}') |
| 232 | // get output of git log -1 (last commit) |
| 233 | git_log_v := |
| 234 | gen_vc.cmd_exec('git -C ${git_repo_dir_v} log -1 --format="commit %H%nDate: %ci%nDate Unix: %ct%nSubject: %s"') |
| 235 | git_log_vc := |
| 236 | gen_vc.cmd_exec('git -C ${git_repo_dir_vc} log -1 --format="Commit %H%nDate: %ci%nDate Unix: %ct%nSubject: %s"') |
| 237 | // date of last commit in each repo |
| 238 | ts_v := git_log_v.find_between('Date:', '\n').trim_space() |
| 239 | ts_vc := git_log_vc.find_between('Date:', '\n').trim_space() |
| 240 | // parse time as string to time.Time |
| 241 | last_commit_time_v := time.parse(ts_v) or { panic(err) } |
| 242 | last_commit_time_vc := time.parse(ts_vc) or { panic(err) } |
| 243 | // git dates are in users local timezone and v time.parse does not parse |
| 244 | // timezones at the moment, so for now get unix timestamp from output also |
| 245 | t_unix_v := git_log_v.find_between('Date Unix:', '\n').trim_space().int() |
| 246 | t_unix_vc := git_log_vc.find_between('Date Unix:', '\n').trim_space().int() |
| 247 | // last commit hash in v repo |
| 248 | last_commit_hash_v := git_log_v.find_between('commit', '\n').trim_space() |
| 249 | last_commit_hash_v_short := last_commit_hash_v[..7] |
| 250 | // subject |
| 251 | last_commit_subject := git_log_v.find_between('Subject:', '\n').trim_space().replace("'", '"') |
| 252 | // log some info |
| 253 | gen_vc.logger.debug('last commit time (${git_repo_v}): ' + last_commit_time_v.format_ss()) |
| 254 | gen_vc.logger.debug('last commit time (${git_repo_vc}): ' + last_commit_time_vc.format_ss()) |
| 255 | gen_vc.logger.debug('last commit hash (${git_repo_v}): ${last_commit_hash_v}') |
| 256 | gen_vc.logger.debug('last commit subject (${git_repo_v}): ${last_commit_subject}') |
| 257 | // if vc repo already has a newer commit than the v repo, assume it's up to date |
| 258 | if t_unix_vc >= t_unix_v && !gen_vc.options.force { |
| 259 | gen_vc.logger.warn('vc repository is already up to date.') |
| 260 | return |
| 261 | } |
| 262 | // try build v for current os (linux in this case) |
| 263 | gen_vc.cmd_exec('make -C ${git_repo_dir_v}') |
| 264 | v_exec := '${git_repo_dir_v}/v' |
| 265 | // check if make was successful |
| 266 | gen_vc.assert_file_exists_and_is_not_too_short(v_exec, err_msg_make) |
| 267 | // build v.c for each os |
| 268 | for os_name in vc_build_oses { |
| 269 | c_file := if os_name == 'nix' { 'v.c' } else { 'v_win.c' } |
| 270 | v_flags := if os_name == 'nix' { '-os cross' } else { '-os ${os_name}' } |
| 271 | // try generate .c file |
| 272 | gen_vc.cmd_exec('${v_exec} ${v_flags} -o ${c_file} ${git_repo_dir_v}/cmd/v') |
| 273 | // check if the c file seems ok |
| 274 | gen_vc.assert_file_exists_and_is_not_too_short(c_file, err_msg_gen_c) |
| 275 | // embed the latest v commit hash into the c file |
| 276 | gen_vc.cmd_exec('sed -i \'1s/^/#define V_COMMIT_HASH "${last_commit_hash_v_short}"\\n/\' ${c_file}') |
| 277 | // move to vc repo |
| 278 | gen_vc.cmd_exec('mv ${c_file} ${git_repo_dir_vc}/${c_file}') |
| 279 | // add new .c file to local vc repo |
| 280 | gen_vc.cmd_exec('git -C ${git_repo_dir_vc} add ${c_file}') |
| 281 | } |
| 282 | // check if the vc repo actually changed |
| 283 | git_status := gen_vc.cmd_exec('git -C ${git_repo_dir_vc} status') |
| 284 | if git_status.contains('nothing to commit') { |
| 285 | gen_vc.logger.error('no changes to vc repo: something went wrong.') |
| 286 | gen_vc.gen_error = true |
| 287 | } |
| 288 | // commit changes to local vc repo |
| 289 | gen_vc.cmd_exec_safe("git -C ${git_repo_dir_vc} commit -m '[v:master] ${last_commit_hash_v_short} - ${last_commit_subject}'") |
| 290 | // push changes to remote vc repo |
| 291 | gen_vc.cmd_exec_safe('git -C ${git_repo_dir_vc} push https://${urllib.query_escape(git_username)}:${urllib.query_escape(git_password)}@${git_repo_vc} master') |
| 292 | } |
| 293 | |
| 294 | // only execute when dry_run option is false, otherwise just log |
| 295 | fn (mut gen_vc GenVC) cmd_exec_safe(cmd string) string { |
| 296 | return gen_vc.command_execute(cmd, gen_vc.options.dry_run) |
| 297 | } |
| 298 | |
| 299 | // always execute command |
| 300 | fn (mut gen_vc GenVC) cmd_exec(cmd string) string { |
| 301 | return gen_vc.command_execute(cmd, false) |
| 302 | } |
| 303 | |
| 304 | // execute command |
| 305 | fn (mut gen_vc GenVC) command_execute(cmd string, dry bool) string { |
| 306 | // if dry is true then dont execute, just log |
| 307 | if dry { |
| 308 | return gen_vc.command_execute_dry(cmd) |
| 309 | } |
| 310 | gen_vc.logger.info('cmd: ${cmd}') |
| 311 | r := os.execute(cmd) |
| 312 | if r.exit_code < 0 { |
| 313 | gen_vc.logger.error('${err_msg_cmd_x}: "${cmd}" could not start.') |
| 314 | gen_vc.logger.error(r.output) |
| 315 | // something went wrong, better start fresh next time |
| 316 | gen_vc.purge_repos() |
| 317 | gen_vc.gen_error = true |
| 318 | return '' |
| 319 | } |
| 320 | if r.exit_code != 0 { |
| 321 | gen_vc.logger.error('${err_msg_cmd_x}: "${cmd}" failed.') |
| 322 | gen_vc.logger.error(r.output) |
| 323 | // something went wrong, better start fresh next time |
| 324 | gen_vc.purge_repos() |
| 325 | gen_vc.gen_error = true |
| 326 | return '' |
| 327 | } |
| 328 | return r.output |
| 329 | } |
| 330 | |
| 331 | // just log cmd, dont execute |
| 332 | fn (mut gen_vc GenVC) command_execute_dry(cmd string) string { |
| 333 | gen_vc.logger.info('cmd (dry): "${cmd}"') |
| 334 | return '' |
| 335 | } |
| 336 | |
| 337 | // delete repo directories |
| 338 | fn (mut gen_vc GenVC) purge_repos() { |
| 339 | // delete old repos (better to be fully explicit here, since these are destructive operations) |
| 340 | mut repo_dir := '${gen_vc.options.work_dir}/${git_repo_dir_v}' |
| 341 | if os.is_dir(repo_dir) { |
| 342 | gen_vc.logger.info('purging local repo: "${repo_dir}"') |
| 343 | gen_vc.cmd_exec('rm -rf ${repo_dir}') |
| 344 | } |
| 345 | repo_dir = '${gen_vc.options.work_dir}/${git_repo_dir_vc}' |
| 346 | if os.is_dir(repo_dir) { |
| 347 | gen_vc.logger.info('purging local repo: "${repo_dir}"') |
| 348 | gen_vc.cmd_exec('rm -rf ${repo_dir}') |
| 349 | } |
| 350 | } |
| 351 | |
| 352 | // check if file size is too short |
| 353 | fn (mut gen_vc GenVC) assert_file_exists_and_is_not_too_short(f string, emsg string) { |
| 354 | if !os.exists(f) { |
| 355 | gen_vc.logger.error('${err_msg_build}: ${emsg} .') |
| 356 | gen_vc.gen_error = true |
| 357 | return |
| 358 | } |
| 359 | fsize := os.file_size(f) |
| 360 | if fsize < too_short_file_limit { |
| 361 | gen_vc.logger.error('${err_msg_build}: ${f} exists, but is too short: only ${fsize} bytes.') |
| 362 | gen_vc.gen_error = true |
| 363 | return |
| 364 | } |
| 365 | } |
| 366 | |