From c57e957ca38ce9aebb47f71e18d4a9f23a75a097 Mon Sep 17 00:00:00 2001 From: walking dev <104449470+walkingdevel@users.noreply.github.com> Date: Sun, 29 May 2022 18:15:40 +0000 Subject: [PATCH] update repositories after push (#174) --- branch_service.v | 83 ++++++++------------ cli.v | 3 - commit_service.v | 18 +++-- file_service.v | 32 +++++--- git/branch.v | 13 ++-- git/branch_test.v | 3 +- git/push.v | 26 +++++++ git_routes.v | 26 ++++++- repo_routes.v | 18 ++--- repo_service.v | 174 ++++++++++++++++++++++++++---------------- templates/header.html | 4 +- 11 files changed, 240 insertions(+), 160 deletions(-) create mode 100644 git/push.v diff --git a/branch_service.v b/branch_service.v index 2b296f5..e8b5587 100644 --- a/branch_service.v +++ b/branch_service.v @@ -3,79 +3,60 @@ module main import time import git -fn (mut app App) fetch_branches(r Repo) { - branches_output := r.git('branch -av') +fn (mut app App) fetch_branch(repo Repo, branch_name string) { + last_commit_hash := repo.get_last_branch_commit_hash(branch_name) - for branch_output in branches_output.split_into_lines() { - branch_name, last_commit_hash := git.parse_git_branch_output(branch_output) - - branch_data := r.git('log $branch_name -1 --pretty="%aE$log_field_separator%cD" $last_commit_hash') + branch_data := repo.git('log $branch_name -1 --pretty="%aE$log_field_separator%cD" $last_commit_hash') + log_parts := branch_data.split(log_field_separator) - log_parts := branch_data.split(log_field_separator) - author_email := log_parts[0] - committed_at := log_parts[1] + author_email := log_parts[0] + committed_at := time.parse_rfc2822(log_parts[1]) or { + app.info('Error: $err') - user := app.find_user_by_email(author_email) or { - User{ - username: author_email - } - } - - committed_at_date := time.parse_rfc2822(committed_at) or { - app.info('Error: $err') + return + } - return + user := app.find_user_by_email(author_email) or { + User{ + username: author_email } - - app.create_branch(r.id, branch_name, user.username, last_commit_hash, int(committed_at_date.unix)) } + + app.create_branch_or_update(repo.id, branch_name, user.username, last_commit_hash, + int(committed_at.unix)) } -fn (mut app App) update_branches(r &Repo) { - branches_output := r.git('branch -av') +fn (mut app App) fetch_branches(repo Repo) { + branches_output := repo.git('branch -a') for branch_output in branches_output.split_into_lines() { - branch_name, last_commit_hash := git.parse_git_branch_output(branch_output) - - branch_data := r.git('log $branch_name -1 --pretty="%aE$log_field_separator%cD" $last_commit_hash') - - log_parts := branch_data.split(log_field_separator) - author_email := log_parts[0] - committed_at := log_parts[1] + branch_name := git.parse_git_branch_output(branch_output) - user := app.find_user_by_email(author_email) or { - User{ - username: author_email - } - } - - committed_at_date := time.parse_rfc2822(committed_at) or { - app.info('Error: $err') + app.fetch_branch(repo, branch_name) + } +} - return - } +fn (mut app App) create_branch_or_update(repository_id int, branch_name string, author string, hash string, date int) { + branch := sql app.db { + select from Branch where repo_id == repository_id && name == branch_name limit 1 + } - if !app.contains_repo_branch(r.id, branch_name) { - app.create_branch(r.id, branch_name, user.username, last_commit_hash, int(committed_at_date.unix)) - } else { - branch := app.find_repo_branch_by_name(r.id, branch_name) + if branch.id != 0 { + app.update_branch(branch.id, author, hash, date) - app.update_branch(branch.id, user.username, last_commit_hash, int(committed_at_date.unix)) - } + return } -} -fn (mut app App) create_branch(repo_id int, name string, author string, hash string, date int) { - branch := Branch{ - repo_id: repo_id - name: name + new_branch := Branch{ + repo_id: repository_id + name: branch_name author: author hash: hash date: date } sql app.db { - insert branch into Branch + insert new_branch into Branch } } diff --git a/cli.v b/cli.v index 5d4c873..1cfd659 100644 --- a/cli.v +++ b/cli.v @@ -13,9 +13,6 @@ pub fn (mut app App) command_fetcher() { if args.len > 0 { match args[0] { - 'updaterepo' { - app.update_repository() - } 'adduser' { if args.len > 4 { app.register_user(args[1], args[2], args[3], args[4..], false, diff --git a/commit_service.v b/commit_service.v index 7c10364..dc35e12 100644 --- a/commit_service.v +++ b/commit_service.v @@ -60,18 +60,26 @@ fn (commit Commit) get_changes(repo Repo) []Change { return changes } -fn (mut app App) add_commit(repo_id int, hash string, author string, author_id int, message string, date int) { - commit := Commit{ +fn (mut app App) add_commit_if_not_exist(repository_id int, last_hash string, author string, author_id int, message string, date int) { + commit := sql app.db { + select from Commit where repo_id == repository_id && hash == last_hash limit 1 + } + + if commit.id > 0 { + return + } + + new_commit := Commit{ author_id: author_id author: author - hash: hash + hash: last_hash created_at: date - repo_id: repo_id + repo_id: repository_id message: message } sql app.db { - insert commit into Commit + insert new_commit into Commit } } diff --git a/file_service.v b/file_service.v index 964ce57..fe3231a 100644 --- a/file_service.v +++ b/file_service.v @@ -46,27 +46,37 @@ fn (mut app App) find_repository_items(repo_id int, branch string, parent_path s return items } -fn (mut app App) find_repo_file_by_path(repo_id int, branch string, path string) ?File { - parent_path := os.dir(path) - name := path.after('/') - app.info('find file parent_path=$parent_path name=$name') - mut p_path := parent_path - if p_path == '' { - p_path = '.' +fn (mut app App) find_repo_file_by_path(repo_id int, item_branch string, path string) ?File { + mut valid_parent_path := os.dir(path) + item_name := path.after('/') + + if valid_parent_path == '' || valid_parent_path == '/' { + valid_parent_path = '.' } + + app.info('find file repo_id=$repo_id parent_path = $valid_parent_path branch=$item_branch name=$item_branch') + file := sql app.db { - select from File where repo_id == repo_id && parent_path == p_path && branch == branch - && name == name limit 1 + select from File where repo_id == repo_id && parent_path == valid_parent_path + && branch == item_branch && name == item_name limit 1 } + if file.name == '' { return none } + return file } -fn (mut app App) delete_repo_files(repo_id int) { +fn (mut app App) delete_repository_files(repository_id int) { + sql app.db { + delete from File where repo_id == repository_id + } +} + +fn (mut app App) delete_repository_files_in_branch(repository_id int, branch_name string) { sql app.db { - delete from File where repo_id == repo_id + delete from File where repo_id == repository_id && branch == branch_name } } diff --git a/git/branch.v b/git/branch.v index 849265c..43f7834 100644 --- a/git/branch.v +++ b/git/branch.v @@ -1,15 +1,14 @@ module git -// parse_git_branch_with_last_hash parses output from `git branch -av` -// returns the branch name with hash of the last commit -pub fn parse_git_branch_output(output string) (string, string) { - output_parts := output.split(' ').filter(it != '') - +// parse_git_branch_with_last_hash parses output from `git branch -a` +// returns the branch name +pub fn parse_git_branch_output(output string) string { + output_parts := output.fields() asterisk_or_branch_name := output_parts[0] if asterisk_or_branch_name == '*' { - return output_parts[1], output_parts[2] + return output_parts[1] } - return output_parts[0], output_parts[1] + return output_parts[0] } diff --git a/git/branch_test.v b/git/branch_test.v index 0df9b37..f9a0e1a 100644 --- a/git/branch_test.v +++ b/git/branch_test.v @@ -1,8 +1,7 @@ module git fn test_parse_git_branch_output() { - branch_name, last_commit_hash := parse_git_branch_output('* main test another_test') + branch_name := parse_git_branch_output('* main') assert branch_name == 'main' - assert last_commit_hash == 'test' } diff --git a/git/push.v b/git/push.v new file mode 100644 index 0000000..cc577a1 --- /dev/null +++ b/git/push.v @@ -0,0 +1,26 @@ +module git + +pub fn parse_branch_name_from_receive_upload(upload string) ?string { + upload_lines := upload.split_into_lines() + + if upload_lines.len == 0 { + return none + } + + upload_header := upload_lines[0] + header_parts := upload_header.fields() + + if header_parts.len < 3 { + return none + } + + branch_reference := header_parts[2] + branch_name_raw := get_branch_name_from_reference(branch_reference) + branch_name := branch_name_raw.trim_space().trim('\0') + + if branch_name.len == 0 { + return none + } + + return branch_name +} diff --git a/git_routes.v b/git_routes.v index c6e78fd..ce607f1 100644 --- a/git_routes.v +++ b/git_routes.v @@ -52,13 +52,22 @@ fn (mut app App) handle_git_upload_pack(username string, git_repository_name str ['/:user/:repository/git-receive-pack'; post] fn (mut app App) handle_git_receive_pack(username string, git_repository_name string) vweb.Result { + body := app.req.data repository_name := git.remove_git_extension_if_exists(git_repository_name) user := app.find_user_by_username(username) or { return app.not_found() } repository := app.find_repo_by_name(user.id, repository_name) or { return app.not_found() } app.check_git_http_access(username, repository_name) or { return app.ok('') } - git_response := repository.git_smart('receive-pack', app.req.data) + git_response := repository.git_smart('receive-pack', body) + + branch_name := git.parse_branch_name_from_receive_upload(body) or { + app.send_internal_error('Receive upload parsing error') + + return app.ok('') + } + + app.update_repo_after_push(repository.id, branch_name) app.set_git_content_type_headers(.receive) @@ -150,12 +159,21 @@ fn (mut app App) set_git_content_type_headers(service GitService) { } } +fn (mut app App) send_internal_error(custom_message string) { + message := if custom_message == '' { 'Internal Server error' } else { custom_message } + + app.send_custom_error(500, message) +} + fn (mut app App) send_unauthorized() { - app.set_status(401, 'Unauthorized') - app.send_response_to_client(vweb.mime_types['.txt'], '') + app.send_custom_error(401, 'Unauthorized') } fn (mut app App) send_not_found() { - app.set_status(404, 'Not Found') + app.send_custom_error(404, 'Not Found') +} + +fn (mut app App) send_custom_error(code int, text string) { + app.set_status(code, text) app.send_response_to_client(vweb.mime_types['.txt'], '') } diff --git a/repo_routes.v b/repo_routes.v index 5097f4b..d3bb3e0 100644 --- a/repo_routes.v +++ b/repo_routes.v @@ -61,7 +61,7 @@ pub fn (mut app App) handle_repo_delete(user string, repo string) vweb.Result { } if app.form['verify'] == '$user/$repo' { - go app.delete_repo(app.repo.id, app.repo.git_dir, app.repo.name) + go app.delete_repository(app.repo.id, app.repo.git_dir, app.repo.name) } else { app.error('Verification failed') return app.repo_settings(user, repo) @@ -142,7 +142,7 @@ pub fn (mut app App) handle_repo_update(user string, repo string) vweb.Result { } if app.user.is_admin { - app.update_repo_data(mut app.repo) + app.update_repository_data(mut app.repo) app.slow_fetch_files_info('master', '.') } @@ -239,15 +239,15 @@ pub fn (mut app App) handle_new_repo(name string, clone_url string) vweb.Result // Update only cloned repositories if !is_clone_url_empty { - app.update_repository() + app.update_repository(mut app.repo) } return app.redirect('/$app.user.username/repos') } -['/:user/:repo/tree/:branch/:path...'] -pub fn (mut app App) tree(username string, repo string, branch string, path string) vweb.Result { - if !app.exists_user_repo(username, repo) { +['/:user/:repository/tree/:branch/:path...'] +pub fn (mut app App) tree(username string, repository_name string, branch string, path string) vweb.Result { + if !app.exists_user_repo(username, repository_name) { return app.not_found() } @@ -259,7 +259,7 @@ pub fn (mut app App) tree(username string, repo string, branch string, path stri } repo_id := app.repo.id - log_prefix := '$username/$repo' + log_prefix := '$username/$repository_name' app.current_path = '/$path' if app.current_path.contains('/favicon.svg') { @@ -268,7 +268,7 @@ pub fn (mut app App) tree(username string, repo string, branch string, path stri path_parts := path.split('/') - app.path_split = [repo] + app.path_split = [repository_name] app.path_split << path_parts app.is_tree = true @@ -298,7 +298,7 @@ pub fn (mut app App) tree(username string, repo string, branch string, path stri // No files in the db, fetch them from git and cache in db app.info('$log_prefix: caching items in repository with $repo_id') - items = app.cache_repo_files(mut app.repo, branch, app.current_path) + items = app.cache_repository_items(mut app.repo, branch, app.current_path) app.slow_fetch_files_info(branch, app.current_path) } diff --git a/repo_service.v b/repo_service.v index 397c51b..371655b 100644 --- a/repo_service.v +++ b/repo_service.v @@ -23,19 +23,20 @@ enum RepoStatus { clone_done } -fn (mut app App) update_repo_in_db(repo &Repo) { - id := repo.id - desc := repo.description - views_count := repo.views_count - webhook_secret := repo.webhook_secret - tags_count := repo.tags_count - is_public := if repo.is_public { 1 } else { 0 } - open_issues_count := repo.open_issues_count - open_prs_count := repo.open_prs_count - branches_count := repo.branches_count - releases_count := repo.releases_count - contributors_count := repo.contributors_count - commits_count := repo.commits_count +fn (mut app App) save_repository(repository Repo) { + id := repository.id + desc := repository.description + views_count := repository.views_count + webhook_secret := repository.webhook_secret + tags_count := repository.tags_count + is_public := if repository.is_public { 1 } else { 0 } + open_issues_count := repository.open_issues_count + open_prs_count := repository.open_prs_count + branches_count := repository.branches_count + releases_count := repository.releases_count + contributors_count := repository.contributors_count + commits_count := repository.commits_count + sql app.db { update Repo set description = desc, views_count = views_count, is_public = is_public, webhook_secret = webhook_secret, tags_count = tags_count, open_issues_count = open_issues_count, @@ -146,7 +147,7 @@ fn (mut app App) add_repo(repo Repo) { } } -fn (mut app App) delete_repo(id int, path string, name string) { +fn (mut app App) delete_repository(id int, path string, name string) { sql app.db { delete from Repo where id == id } @@ -166,7 +167,7 @@ fn (mut app App) delete_repo(id int, path string, name string) { app.delete_repo_releases(id) app.info('Removed repo releases ($id, $name)') - app.delete_repo_files(id) + app.delete_repository_files(id) app.info('Removed repo files ($id, $name)') app.delete_repo_folder(path) @@ -186,18 +187,17 @@ fn (mut app App) user_has_repo(user_id int, repo_name string) bool { return count >= 0 } -fn (mut app App) update_repository() { - mut r := app.repo +fn (mut app App) update_repository(mut repository Repo) { + repository_id := repository.id - r.analyse_lang(app) + repository.analyse_lang(app) - data := r.git('--no-pager log --abbrev-commit --abbrev=7 --pretty="%h$log_field_separator%aE$log_field_separator%cD$log_field_separator%s$log_field_separator%aN"') + data := repository.git('--no-pager log --abbrev-commit --abbrev=7 --pretty="%h$log_field_separator%aE$log_field_separator%cD$log_field_separator%s$log_field_separator%aN"') app.db.exec('BEGIN TRANSACTION') for line in data.split_into_lines() { args := line.split(log_field_separator) if args.len > 3 { - repo_id := r.id commit_hash := args[0] commit_message := args[3] commit_author := args[4] @@ -210,43 +210,43 @@ fn (mut app App) update_repository() { user := app.find_user_by_email(args[1]) or { User{} } if user.username != '' { - app.add_contributor(user.id, r.id) + app.add_contributor(user.id, repository_id) commit_author_id = user.id } else { empty_user := app.create_empty_user(commit_author, args[1]) - app.add_contributor(empty_user, r.id) + app.add_contributor(empty_user, repository_id) } - app.add_commit(repo_id, commit_hash, commit_author, commit_author_id, commit_message, - int(commit_date.unix)) + app.add_commit_if_not_exist(repository_id, commit_hash, commit_author, commit_author_id, + commit_message, int(commit_date.unix)) } } - app.info(r.contributors_count.str()) - app.fetch_branches(r) + app.info(repository.contributors_count.str()) + app.fetch_branches(repository) - r.commits_count = app.get_count_repo_commits(r.id) - r.contributors_count = app.get_count_repo_contributors(r.id) - r.branches_count = app.get_count_repo_branches(r.id) + repository.commits_count = app.get_count_repo_commits(repository_id) + repository.contributors_count = app.get_count_repo_contributors(repository_id) + repository.branches_count = app.get_count_repo_branches(repository_id) - app.update_repo_commits_count(r.id, r.commits_count) - app.update_repo_contributors_count(r.id, r.contributors_count) + app.update_repo_commits_count(repository_id, repository.commits_count) + app.update_repo_contributors_count(repository_id, repository.contributors_count) // TODO: TEMPORARY - UNTIL WE GET PERSISTENT RELEASE INFO - for tag in app.get_all_repo_tags(r.id) { - app.add_release(tag.id, r.id) + for tag in app.get_all_repo_tags(repository_id) { + app.add_release(tag.id, repository_id) - r.releases_count++ + repository.releases_count++ } - app.update_repo_in_db(r) + app.save_repository(repository) app.db.exec('END TRANSACTION') app.info('Repository updated') } -fn (mut app App) update_repo_data(mut r Repo) { +fn (mut app App) update_repository_data(mut r Repo) { r.git('fetch --all') r.git('pull --all') @@ -282,8 +282,8 @@ fn (mut app App) update_repo_data(mut r Repo) { app.add_contributor(empty_user, r.id) } - app.add_commit(repo_id, commit_hash, commit_author, commit_author_id, commit_message, - int(commit_date.unix)) + app.add_commit_if_not_exist(repo_id, commit_hash, commit_author, commit_author_id, + commit_message, int(commit_date.unix)) } } @@ -293,18 +293,32 @@ fn (mut app App) update_repo_data(mut r Repo) { app.update_repo_commits_count(r.id, r.commits_count) app.update_repo_contributors_count(r.id, r.contributors_count) - app.update_branches(r) - app.update_repo_in_db(r) + app.fetch_branches(r) + app.save_repository(r) app.db.exec('END TRANSACTION') app.info('Repo updated') } +// TODO: tags and other stuff +fn (mut app App) update_repo_after_push(repo_id int, branch_name string) { + mut repo := app.find_repo_by_id(repo_id) + + if repo.id == 0 { + return + } + + app.update_repository(mut repo) + app.delete_repository_files_in_branch(repo_id, branch_name) +} + fn (r &Repo) analyse_lang(app &App) { files := r.get_all_files(r.git_dir) + mut all_size := 0 mut lang_stats := map[string]int{} mut langs := map[string]highlight.Lang{} + for file in files { lang := highlight.extension_to_lang(file.split('.').last()) or { continue } f_text := os.read_file(file) or { '' } @@ -319,8 +333,10 @@ fn (r &Repo) analyse_lang(app &App) { lang_stats[lang.name] = lang_stats[lang.name] + size all_size += size } + mut d_lang_stats := []LangStat{} mut tmp_a := []int{} + for lang, amount in lang_stats { mut tmp := f32(amount) / f32(all_size) tmp *= 1000 @@ -337,15 +353,19 @@ fn (r &Repo) analyse_lang(app &App) { lines_count: amount } } + tmp_a.sort() tmp_a = tmp_a.reverse() + mut tmp_stats := []LangStat{} + for pct in tmp_a { all_with_ptc := r.lang_stats.filter(it.pct == pct) for lang in all_with_ptc { tmp_stats << lang } } + for lang_stat in d_lang_stats { sql app.db { insert lang_stat into LangStat @@ -503,73 +523,79 @@ fn (r &Repo) git(command string) string { return command_output } -fn (r &Repo) parse_ls(ls string, branch string) ?File { - words := ls.fields() - - if words.len < 4 { +fn (r &Repo) parse_ls(ls_line string, branch string) ?File { + ls_line_parts := ls_line.fields() + if ls_line_parts.len < 4 { return none } - typ := words[1] - mut parent_path := os.dir(words[3]) - hash := r.git('log $branch -n 1 --format="%h" -- ${words[3]}') - name := words[3].after('/') - if name == '' { + item_type := ls_line_parts[1] + item_path := ls_line_parts[3] + item_hash := r.git('log $branch -n 1 --format="%h" -- $item_path') + + item_name := item_path.after('/') + if item_name == '' { return none } - if parent_path == name { + + mut parent_path := os.dir(item_path) + if parent_path == item_name { parent_path = '' } - if name.contains('"\\') { + + if item_name.contains('"\\') { // Unqoute octal UTF-8 strings } return File{ - name: name + name: item_name parent_path: parent_path repo_id: r.id - last_hash: hash + last_hash: item_hash branch: branch - is_dir: typ == 'tree' + is_dir: item_type == 'tree' } } // Fetches all files via `git ls-tree` and saves them in db -fn (mut app App) cache_repo_files(mut r Repo, branch string, path string) []File { +fn (mut app App) cache_repository_items(mut r Repo, branch string, path string) []File { if r.status == .caching { app.info('`$r.name` is being cached already') return [] } - mut res := '' + mut repository_ls := '' if path == '.' { r.status = .caching + defer { r.status = .done } } else { - mut p := path - if path != '' { - p += '/' - } - res = r.git('ls-tree --full-name $branch $p') + directory_path := if path == '' { path } else { '$path/' } + repository_ls = r.git('ls-tree --full-name $branch $directory_path') } - lines := res.split('\n') + + // mode type name path + item_info_lines := repository_ls.split('\n') + mut dirs := []File{} // dirs first mut files := []File{} app.db.exec('BEGIN TRANSACTION') - for line in lines { - is_line_empty := validation.is_string_empty(line) - if is_line_empty { + for item_info in item_info_lines { + is_item_info_empty := validation.is_string_empty(item_info) + + if is_item_info_empty { continue } - file := r.parse_ls(line, branch) or { - app.warn('failed to parse $line') + file := r.parse_ls(item_info, branch) or { + app.warn('failed to parse $item_info') continue } + if file.is_dir { dirs << file @@ -578,11 +604,14 @@ fn (mut app App) cache_repo_files(mut r Repo, branch string, path string) []File files << file } } + dirs << files for file in files { app.add_file(file) } + app.db.exec('END TRANSACTION') + return dirs } @@ -625,6 +654,17 @@ fn (mut app App) slow_fetch_files_info(branch string, path string) { } } +fn (r Repo) get_last_branch_commit_hash(branch_name string) string { + git_result := os.execute('git -C $r.git_dir log -n 1 $branch_name --pretty=format:"%h"') + git_output := git_result.output + + if git_result.exit_code != 0 { + eprintln('git log error: $git_output') + } + + return git_output +} + fn (r Repo) git_advertise(service string) string { git_result := os.execute('git $service --stateless-rpc --advertise-refs $r.git_dir') git_output := git_result.output diff --git a/templates/header.html b/templates/header.html index ad0bcf1..d83b6d3 100644 --- a/templates/header.html +++ b/templates/header.html @@ -88,7 +88,9 @@ .tree-path { @for i, p in app.path_split @p - / + @if i < app.path_split.len - 1 + / + @end @end } @end -- 2.39.5