OverviewFilesCommitsIssuesDocsReleasesPeers

pear-git-v3 commits / 2f7c1d9

coalesce concurrent /repos enrich passes
startPearGit fires a fire-and-forget pre-warm of the enrich cache at boot. Nothing coordinated it with the request path, so a visitor arriving during boot started a SECOND full pass while the first was still running. Neither had populated the cache yet, so both missed and every repo was walked twice — up to 2x CONCURRENCY git processes, on exactly the cold boot where the box is already busy. search-all-repos runs the same full pass on an autocomplete keystroke, so a third and fourth overlapping pass are reachable from a single visitor. getRepoDetails now keeps an in-flight map: a later caller joins the promise already computing that repo instead of forking its own git. Keyed on name alone — a push landing mid-flight leaves the joiner with the marginally older result, which is fine for a cache whose push path calls invalidateRepoActivity() explicitly. getRepoActivityStats() exposes computes/deduped so the coalescing is asserted directly rather than inferred from timing. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2f7c1d9a30b96ab8a98acc1d7d0922f3da15718d
by plankatron <fleeky@gmail.com> · 2026-08-16T16:37:41.000Z

Changed files (2) +171 -1

lib/services/repo-activity.js +48 -1
……@@ -46,6 +46,22 @@ const CONCURRENCY = 8
4646
4747const cache = new Map() // name -> { cachedAt, mtime, data }
4848
49// In-flight computations, name -> Promise<data>. The boot pre-warm and a
50// visitor's render are two independent full passes with nothing between them:
51// neither has populated the cache yet, so without this map both miss and every
52// repo is walked TWICE, forking up to 2x CONCURRENCY git processes on exactly
53// the cold boot where the box is already busy. search-all-repos runs the same
54// full pass per autocomplete keystroke, so more overlapping passes are
55// reachable from one visitor. A late arrival joins the promise already running.
56const inflight = new Map()
57
58// Observability for the above — a test can assert coalescing happened instead
59// of inferring it from timing, and an operator can see whether the cache is
60// actually working. `computes` counts real git passes, `deduped` counts
61// callers that joined one already running.
62let computes = 0
63let deduped = 0
64
4965let persistPath = null
5066let loaded = false
5167let dirty = false
……@@ -97,8 +113,15 @@ export async function configureRepoActivityCache (storageDir) {
97113 * Test seam only — nothing in the daemon calls it. It deliberately does NOT
98114 * touch the cache FILE, only the process-local state.
99115 */
116export function getRepoActivityStats () {
117 return { computes, deduped, cached: cache.size, inflight: inflight.size }
118}
119
100120export function resetRepoActivityCacheForTests () {
101121 cache.clear()
122 inflight.clear()
123 computes = 0
124 deduped = 0
102125 persistPath = null
103126 loaded = false
104127 dirty = false
……@@ -177,6 +200,30 @@ async function getRepoDetails (name, repoPath) {
177200 return cached.data
178201 }
179202
203 // Join a pass already computing this repo rather than forking a second set
204 // of git processes for the same answer. Keyed on name alone: a push landing
205 // mid-flight would leave the joiner with the marginally older result, which
206 // is fine for a cache whose push path calls invalidateRepoActivity()
207 // explicitly. A rejection propagates to both callers, and enrichRepos
208 // already catches it into the zeroed default.
209 const running = inflight.get(name)
210 if (running) {
211 deduped++
212 return running
213 }
214
215 const p = computeRepoDetails(name, repoPath, packMtime)
216 inflight.set(name, p)
217 try {
218 return await p
219 } finally {
220 inflight.delete(name)
221 }
222}
223
224async function computeRepoDetails (name, repoPath, packMtime) {
225 computes++
226
180227 // Best-effort commit-graph so the single log walk below is fast even on a
181228 // cold OS page cache (incremental; no-op when already current). Then ONE
182229 // git spawn (was three) yields count + last-commit + activity buckets.
……@@ -187,7 +234,7 @@ async function getRepoDetails (name, repoPath) {
187234 ])
188235
189236 const data = { description, ...stats }
190 cache.set(name, { cachedAt: now, mtime: packMtime, data })
237 cache.set(name, { cachedAt: Date.now(), mtime: packMtime, data })
191238 dirty = true
192239 return data
193240}
test/repo-activity-inflight-dedup.test.js +123 -0
……@@ -0,0 +1,123 @@
1// Two concurrent enrich passes over the same repo must compute it ONCE.
2//
3// startPearGit fires a fire-and-forget background pre-warm of the /repos
4// enrich cache (lib/pear-git.js, "Pre-warm the /repos landing-page enrich
5// cache"). Nothing coordinates it with the request path, so a visitor who
6// arrives during boot starts a SECOND full enrich over the same repos while
7// the first is still running. Neither has populated the cache yet, so both
8// miss, and every repo is walked twice — up to 2x CONCURRENCY git processes
9// at once, on exactly the cold boot where the box is already busy.
10//
11// `search-all-repos` runs the same full pass on an autocomplete keystroke, so
12// a third and fourth overlapping pass are reachable from a single visitor.
13//
14// The fix is an in-flight map: the second caller joins the first's promise
15// instead of forking its own git. getRepoActivityStats() exposes the two
16// counters so this is observable rather than inferred from timing.
17
18import test from 'brittle'
19import { mkdtemp, rm, mkdir, writeFile } from '#node:fs/promises'
20import { tmpdir } from '#node:os'
21import { join } from '#node:path'
22import process from '#node:process'
23import { execFileSync } from '../lib/_compat-exec.js'
24
25import {
26 enrichRepos,
27 getRepoActivityStats,
28 resetRepoActivityCacheForTests
29} from '../lib/services/repo-activity.js'
30
31function git (cwd, args) {
32 return execFileSync('git', args, {
33 cwd,
34 stdio: ['ignore', 'pipe', 'ignore'],
35 env: {
36 ...process.env,
37 GIT_AUTHOR_NAME: 't',
38 GIT_AUTHOR_EMAIL: 't@t',
39 GIT_COMMITTER_NAME: 't',
40 GIT_COMMITTER_EMAIL: 't@t'
41 }
42 }).toString()
43}
44
45async function makeBareRepo (root, name, n) {
46 const barePath = join(root, name + '.git')
47 const workPath = join(root, name + '-work')
48 await mkdir(barePath, { recursive: true })
49 await mkdir(workPath, { recursive: true })
50 git(barePath, ['init', '--bare', '-q'])
51 git(workPath, ['init', '-q'])
52 for (let i = 0; i < n; i++) {
53 await writeFile(join(workPath, 'f'), 'v' + i)
54 git(workPath, ['add', 'f'])
55 git(workPath, ['commit', '-q', '-m', 'c' + i])
56 }
57 const branch = git(workPath, ['rev-parse', '--abbrev-ref', 'HEAD']).trim()
58 git(workPath, ['remote', 'add', 'origin', barePath])
59 git(workPath, ['push', '-q', 'origin', branch])
60 return barePath
61}
62
63const statusFor = (names) => ({ active: true, repos: names.map((name) => ({ name, serving: true })) })
64const reposMapFor = (root, names) => {
65 const m = new Map()
66 for (const name of names) m.set(name, { path: join(root, name + '.git') })
67 return m
68}
69
70test('concurrent enrich passes share one computation per repo', async (t) => {
71 const root = await mkdtemp(join(tmpdir(), 'ra-inflight-'))
72 t.teardown(() => rm(root, { recursive: true, force: true }))
73
74 const names = ['alpha', 'beta', 'gamma']
75 for (const n of names) await makeBareRepo(root, n, 3)
76
77 // No persist path — this test is about in-flight coalescing, not disk.
78 resetRepoActivityCacheForTests()
79 t.is(getRepoActivityStats().computes, 0, 'counters start clean')
80
81 const status = statusFor(names)
82 const map = reposMapFor(root, names)
83
84 // Model the boot race: pre-warm and a visitor's render, started without an
85 // await between them so both are genuinely in flight.
86 const [warm, request] = await Promise.all([
87 enrichRepos(status, map),
88 enrichRepos(status, map)
89 ])
90
91 const stats = getRepoActivityStats()
92 t.is(stats.computes, names.length, 'each repo computed exactly once across both passes')
93 t.ok(stats.deduped > 0, 'the second pass joined the first instead of forking its own git')
94
95 // Coalescing must not change what either caller sees.
96 t.is(warm.length, names.length, 'pre-warm pass returned every repo')
97 t.is(request.length, names.length, 'request pass returned every repo')
98 for (let i = 0; i < names.length; i++) {
99 t.is(warm[i].name, names[i], 'input order preserved')
100 t.is(warm[i].commits, 3, 'commit count correct in the computing pass')
101 t.is(request[i].commits, 3, 'the joined pass got the same result, not a stub')
102 t.alike(request[i].activity, warm[i].activity, 'both callers see identical activity buckets')
103 }
104})
105
106test('a later pass still hits the cache rather than recomputing', async (t) => {
107 const root = await mkdtemp(join(tmpdir(), 'ra-inflight2-'))
108 t.teardown(() => rm(root, { recursive: true, force: true }))
109
110 await makeBareRepo(root, 'solo', 2)
111 resetRepoActivityCacheForTests()
112
113 const status = statusFor(['solo'])
114 const map = reposMapFor(root, ['solo'])
115
116 await enrichRepos(status, map)
117 t.is(getRepoActivityStats().computes, 1, 'first pass computed')
118
119 // Sequential, after the in-flight entry has been cleared — this must be a
120 // plain cache hit, proving the dedup map does not leak or shadow the cache.
121 await enrichRepos(status, map)
122 t.is(getRepoActivityStats().computes, 1, 'second pass served from cache, no recompute')
123})