From 9e6aeaf41b0251ebe6d7298147de2a419ddad923 Mon Sep 17 00:00:00 2001 From: Alexander Medvednikov Date: Sat, 16 May 2026 17:03:49 +0300 Subject: [PATCH] clone: import GitHub repo description and contributors Fetch the GitHub description (when the user leaves it blank) and the contributors list in a background thread after a github.com clone. Each contributor is stored as an unregistered shadow user with the GitHub avatar URL; templates render those entries without a profile link. --- github.v | 109 ++++++++++++++++++++++++++++++++++++ repo/repo.v | 22 +++++++- repo/repo_routes.v | 78 ++++++++++++++++++++++++-- static/css/tree.scss | 7 ++- templates/contributors.html | 8 +-- templates/tree.html | 4 ++ 6 files changed, 217 insertions(+), 11 deletions(-) diff --git a/github.v b/github.v index 474704e..6274abb 100644 --- a/github.v +++ b/github.v @@ -30,6 +30,18 @@ struct GitHubLabel { description string } +struct GitHubRepoInfo { + description string +} + +struct GitHubContributor { + login string + avatar_url string + type_ string @[json: 'type'] + html_url string + id int +} + struct GitHubIssue { number int title string @@ -81,6 +93,103 @@ fn (mut app App) find_or_create_github_shadow_user(github_login string) !int { return created.id } +// fetch_github_repo_description returns the GitHub description for a repo, or +// an empty string if it cannot be retrieved. +fn fetch_github_repo_description(clone_url string) string { + owner, name := parse_github_owner_repo(clone_url) or { + eprintln('[github-info] cannot parse github url: ${clone_url}') + return '' + } + url := 'https://api.github.com/repos/${owner}/${name}' + eprintln('[github-info] GET ${url}') + mut req := http.new_request(.get, url, '') + req.add_header(.user_agent, 'gitly') + req.add_header(.accept, 'application/vnd.github+json') + resp := req.do() or { + eprintln('[github-info] request failed: ${err}') + return '' + } + if resp.status_code != 200 { + eprintln('[github-info] non-200 status ${resp.status_code}: ${resp.body#[..200]}') + return '' + } + info := json.decode(GitHubRepoInfo, resp.body) or { + eprintln('[github-info] cannot decode response: ${err}') + return '' + } + return info.description +} + +fn (mut app App) import_github_contributors(repo_id int, clone_url string) ! { + eprintln('[github-contrib] starting for repo_id=${repo_id} clone_url=${clone_url}') + owner, name := parse_github_owner_repo(clone_url) or { + return error('cannot parse github url: ${clone_url}') + } + mut page := 1 + mut total := 0 + for page <= 10 { + url := 'https://api.github.com/repos/${owner}/${name}/contributors?per_page=100&page=${page}' + eprintln('[github-contrib] GET ${url}') + mut req := http.new_request(.get, url, '') + req.add_header(.user_agent, 'gitly') + req.add_header(.accept, 'application/vnd.github+json') + resp := req.do() or { return error('github api request failed: ${err}') } + if resp.status_code != 200 { + return error('github api ${resp.status_code}: ${resp.body}') + } + contributors := json.decode([]GitHubContributor, resp.body) or { + return error('cannot decode github contributors: ${err}') + } + if contributors.len == 0 { + break + } + for c in contributors { + if c.login == '' || c.type_ == 'Bot' { + continue + } + user_id := app.find_or_create_github_shadow_contributor(c.login, c.avatar_url) or { + eprintln('[github-contrib] cannot resolve @${c.login}: ${err}') + continue + } + app.add_contributor(user_id, repo_id) or { + eprintln('[github-contrib] cannot link @${c.login}: ${err}') + continue + } + total++ + } + if contributors.len < 100 { + break + } + page++ + } + app.update_repo_contributor_count(repo_id) or { + eprintln('[github-contrib] cannot update contributor count: ${err}') + } + eprintln('[github-contrib] done: imported ${total} contributors into repo ${repo_id}') +} + +// find_or_create_github_shadow_contributor is like find_or_create_github_shadow_user +// but also stores the GitHub avatar URL when given. +fn (mut app App) find_or_create_github_shadow_contributor(github_login string, avatar_url string) !int { + if u := app.get_user_by_username(github_login) { + return u.id + } + avatar := if avatar_url != '' { avatar_url } else { 'https://github.com/${github_login}.png' } + user := User{ + username: github_login + github_username: github_login + is_github: true + is_registered: false + avatar: avatar + created_at: time.now() + } + app.add_user(user)! + created := app.get_user_by_username(github_login) or { + return error('shadow user not found after insert: ${github_login}') + } + return created.id +} + fn (mut app App) import_github_issues(repo_id int, clone_url string, owner_user_id int) ! { eprintln('[github-import] starting for repo_id=${repo_id} clone_url=${clone_url} owner_user_id=${owner_user_id}') owner, name := parse_github_owner_repo(clone_url) or { diff --git a/repo/repo.v b/repo/repo.v index 60b7a4d..67a5ba7 100644 --- a/repo/repo.v +++ b/repo/repo.v @@ -236,6 +236,19 @@ fn (mut app App) set_repo_status(repo_id int, status RepoStatus) ! { }! } +fn (mut app App) set_repo_description(repo_id int, description string) ! { + sql app.db { + update Repo set description = description where id == repo_id + }! +} + +fn (mut app App) update_repo_contributor_count(repo_id int) ! { + count := app.get_count_repo_contributors(repo_id)! + sql app.db { + update Repo set nr_contributors = count where id == repo_id + }! +} + fn (mut app App) increment_repo_issues(repo_id int) ! { sql app.db { update Repo set nr_open_issues = nr_open_issues + 1 where id == repo_id @@ -1056,9 +1069,14 @@ fn (mut app App) update_repo_primary_branch(repo_id int, branch string) ! { }! } +fn (r &Repo) clone_progress_path() string { + return r.git_dir + '.progress' +} + fn (mut r Repo) clone() { eprintln('R CLONE') - clone_result := git.Git.clone(r.clone_url, r.git_dir) + progress_path := r.clone_progress_path() + clone_result := git.Git.clone_with_progress(r.clone_url, r.git_dir, progress_path) clone_exit_code := clone_result.exit_code if clone_exit_code != 0 { @@ -1068,6 +1086,8 @@ fn (mut r Repo) clone() { } r.status = .done + // progress file is no longer needed after a successful clone + os.rm(progress_path) or {} eprintln('clone done') } diff --git a/repo/repo_routes.v b/repo/repo_routes.v index 9c257a0..cc9e88d 100644 --- a/repo/repo_routes.v +++ b/repo/repo_routes.v @@ -368,11 +368,15 @@ fn clone_repo(new_repo Repo, conf config.Config, import_issues bool, owner_user_ // The tree page will fetch files from git on demand. app.set_repo_status(cloned_repo.id, .done) or { eprintln('cannot set repo status ${err}') } eprintln('clone done, repo available — indexing in background') - // Kick off the GitHub issue import in its own thread with its own DB - // connection so it runs in parallel with indexing and the user can watch - // the issue count grow as imports complete. - if import_issues && cloned_repo.clone_url.contains('github.com') { - spawn bg_import_github_issues(cloned_repo.id, cloned_repo.clone_url, owner_user_id, conf) + // For GitHub clones, also pull the repo description and contributors list + // (the issue import is gated on a separate user opt-in). + if cloned_repo.clone_url.contains('github.com') { + spawn bg_import_github_repo_info(cloned_repo.id, cloned_repo.clone_url, + cloned_repo.description, conf) + if import_issues { + spawn bg_import_github_issues(cloned_repo.id, cloned_repo.clone_url, owner_user_id, + conf) + } } // Index branches, commits, and language stats in the background. app.update_repo_from_fs(mut cloned_repo, true) or { @@ -382,6 +386,31 @@ fn clone_repo(new_repo Repo, conf config.Config, import_issues bool, owner_user_ app.db.close() or {} } +fn bg_import_github_repo_info(repo_id int, clone_url string, existing_description string, conf config.Config) { + eprintln('[github-info] spawned thread for repo_id=${repo_id}') + mut app := &App{ + db: connect_db(conf) or { + eprintln('[github-info] cannot open db connection: ${err}') + return + } + config: conf + } + defer { + app.db.close() or {} + } + if existing_description.trim_space() == '' { + description := fetch_github_repo_description(clone_url) + if description != '' { + app.set_repo_description(repo_id, description) or { + eprintln('[github-info] cannot save description: ${err}') + } + } + } + app.import_github_contributors(repo_id, clone_url) or { + eprintln('[github-contrib] FAILED: ${err}') + } +} + fn bg_import_github_issues(repo_id int, clone_url string, owner_user_id int, conf config.Config) { eprintln('[github-import] spawned thread for repo_id=${repo_id}') mut app := &App{ @@ -399,9 +428,45 @@ fn bg_import_github_issues(repo_id int, clone_url string, owner_user_id int, con pub fn (mut app App) kekw(mut ctx Context) veb.Result { clone_url := '' + clone_progress := '' return $veb.html('templates/cloning_in_process.html') } +// read_clone_progress parses a git `--progress` log file and returns +// the latest output as a single newline-separated string, ready to be +// shown inside a 
 block. Git emits live progress with `\r` and
+// stage transitions with `\n`; we collapse repeated progress lines for
+// the same phase ("Counting objects", "Receiving objects", …) so only
+// the most recent value for each phase remains.
+fn read_clone_progress(progress_path string) string {
+	raw := os.read_file(progress_path) or { return '' }
+	if raw.len == 0 {
+		return ''
+	}
+	lines := raw.replace('\r', '\n').split('\n')
+	mut stages := []string{}
+	mut phase_index := map[string]int{}
+	for raw_line in lines {
+		line := raw_line.trim_space()
+		if line == '' {
+			continue
+		}
+		mut body := line
+		if body.starts_with('remote: ') {
+			body = body[8..]
+		}
+		colon := body.index(':') or { -1 }
+		key := if colon == -1 { body } else { body[..colon].trim_space() }
+		if key in phase_index {
+			stages[phase_index[key]] = line
+		} else {
+			phase_index[key] = stages.len
+			stages << line
+		}
+	}
+	return stages.join('\n')
+}
+
 @['/:username/:repo_name/tree/:branch_name/:path...']
 pub fn (mut app App) tree(mut ctx Context, username string, repo_name string, branch_name string, path string) veb.Result {
 	mut repo := app.find_repo_by_name_and_username(repo_name, username) or {
@@ -409,7 +474,10 @@ pub fn (mut app App) tree(mut ctx Context, username string, repo_name string, br
 		return ctx.not_found()
 	}
 	mut clone_url := ''
+	mut clone_progress := ''
 	if repo.status == .cloning {
+		clone_url = repo.clone_url
+		clone_progress = read_clone_progress(repo.clone_progress_path())
 		return $veb.html('templates/cloning_in_process.html')
 	}
 
diff --git a/static/css/tree.scss b/static/css/tree.scss
index 9741cac..572547e 100644
--- a/static/css/tree.scss
+++ b/static/css/tree.scss
@@ -22,6 +22,7 @@
 .repo-settings-link {
 	font-size: 14px;
 	line-height: 1;
+	margin-left: auto;
 }
 
 /* 1. The repo stats panel */
@@ -305,7 +306,7 @@ a.repo-mode-link:hover {
 /* Two-column layout for the top of the repo (files + sidebar) */
 .repo-layout {
 	display: grid;
-	grid-template-columns: minmax(0, 1fr) 296px;
+	grid-template-columns: minmax(0, 1fr) 160px;
 	gap: 24px;
 	align-items: start;
 }
@@ -494,6 +495,10 @@ a.repo-mode-link:hover {
 	}
 }
 
+.sidebar-contributor--shadow {
+	cursor: default;
+}
+
 @media (max-width: 960px) {
 	.repo-layout {
 		grid-template-columns: minmax(0, 1fr);
diff --git a/templates/contributors.html b/templates/contributors.html
index 321658d..d391273 100644
--- a/templates/contributors.html
+++ b/templates/contributors.html
@@ -15,10 +15,10 @@
 				.contributors-list {
 					@for contributor in contributors
 						.contributors-item {
+							

+								
+							

 							@if contributor.is_registered
-								

-									
-								

 								
@contributor.username

 							@else
 								
@contributor.username

@@ -27,7 +27,7 @@
 					@end
 				}
 			@else
-				
No registered contributors

+				
%no_contributors

 			@end
 		
 
diff --git a/templates/tree.html b/templates/tree.html
index 3121a09..e01ee3b 100644
--- a/templates/tree.html
+++ b/templates/tree.html
@@ -301,6 +301,10 @@
 											
 												@contributor.username
 											
+										@else
+											
+												@contributor.username
+											
 										@end
 									@end
 								
-- 
2.39.5