/**
* Enrich a sync.status() result with per-repo activity data for the
* /repos landing page: commit counts, last-commit timestamp, a 365-day
* activity bucket array (for the SVG sparkline), and the repo's
* description (git convention: the `description` file in a bare repo).
*
* We shell out to git directly against the bare repo path rather than
* adding new methods to hyper-sync — this keeps the sync service focused
* on swarm/serve logic and avoids coupling display concerns to the
* transport layer.
*
* Cost & caching (the /repos landing page was ~15s cold, ~23ms warm):
*
* - Each repo costs ONE git subprocess (`log --all --format=%at|%H`) for
* count + last-commit + activity, plus a best-effort `commit-graph write`
* so that log walk is fast even on a cold OS page cache. Cold, across N
* repos, this dominates render time — hence the concurrency + caching.
* - Results are cached per-repo keyed on the repo's pack mtime (changes
* on push/repack). The pack mtime IS the correctness signal; a long
* backstop TTL only self-heals silent out-of-band edits.
* - The cache is PERSISTED to disk (configureRepoActivityCache) so a
* daemon restart (OTA update, respawn) doesn't re-pay the cold cost —
* unchanged repos (same pack mtime) hit the on-disk cache.
* - enrichRepos runs the per-repo work with bounded CONCURRENCY instead
* of serially, cutting cold render time roughly N-fold.
* - Pushes fire ref-update-observed, which calls invalidateRepoActivity
* to drop the affected repo precisely (loose-object pushes don't always
* bump the pack mtime).
*/
import { spawn } from '#node:child_process'
import { readFile, writeFile, rename, stat } from '#node:fs/promises'
import { join } from '#node:path'
const DAY_MS = 86400000
const BUCKETS = 365
// Backstop TTL. The pack mtime is the primary freshness key (and pushes
// invalidate precisely), so this only bounds truly-silent out-of-band
// edits. Generous so restart-survival works even after long idle.
const BACKSTOP_TTL_MS = 6 * 60 * 60 * 1000 // 6h
// Bound the parallelism so a big install doesn't fork hundreds of git
// processes at once on a cold render.
const CONCURRENCY = 8
const cache = new Map() // name -> { cachedAt, mtime, data }
// In-flight computations, name -> Promise<data>. The boot pre-warm and a
// visitor's render are two independent full passes with nothing between them:
// neither has populated the cache yet, so without this map both miss and every
// repo is walked TWICE, forking up to 2x CONCURRENCY git processes on exactly
// the cold boot where the box is already busy. search-all-repos runs the same
// full pass per autocomplete keystroke, so more overlapping passes are
// reachable from one visitor. A late arrival joins the promise already running.
const inflight = new Map()
// Observability for the above — a test can assert coalescing happened instead
// of inferring it from timing, and an operator can see whether the cache is
// actually working. `computes` counts real git passes, `deduped` counts
// callers that joined one already running.
let computes = 0
let deduped = 0
let persistPath = null
let loaded = false
let dirty = false
let saveTimer = null
/**
* Point the cache at a storage dir and load any persisted entries. Call
* once at daemon boot (before serving) so the first request — and the
* background pre-warm — reuse cross-restart cache instead of re-running
* git against every repo. Idempotent.
*
* @param {string} storageDir - daemon storage dir (opts.storage)
*/
export async function configureRepoActivityCache (storageDir) {
if (!storageDir) return
persistPath = join(storageDir, 'repo-activity-cache.json')
if (loaded) return
loaded = true
try {
const raw = await readFile(persistPath, 'utf8')
const obj = JSON.parse(raw)
if (obj && obj.entries && typeof obj.entries === 'object') {
for (const [name, entry] of Object.entries(obj.entries)) {
if (entry && entry.data && typeof entry.mtime === 'number') {
// Self-heal: distrust a persisted zero.
//
// execGit used to turn ANY git failure into '', which parsed to
// exactly { commits: 0, lastCommit: null } and got cached and
// persisted as a real answer. The cache key is the pack-dir mtime,
// so unless that repo's pack happens to change again the zero is
// served for ever, across restarts. stortron carried one for
// dc-terrain while `git log --all` returned 514 by hand.
//
// The write path is fixed above, but entries written by an older
// build are still on disk, so drop them on load and recompute.
//
// A genuinely empty repo looks identical, and that is fine: the only
// cost of dropping its entry is ONE cheap recompute that writes the
// same zero straight back. Paying that on a handful of empty repos
// per boot is worth not serving a stale zero for a real one.
if (entry.data.commits === 0 && !entry.data.lastCommit) continue
cache.set(name, {
cachedAt: entry.cachedAt || 0,
mtime: entry.mtime,
data: entry.data
})
}
}
}
} catch {
// No cache file yet (first boot) or unreadable — start empty.
}
}
/**
* Drop ALL in-memory state so the next configureRepoActivityCache() re-reads
* the cache file — i.e. model a daemon restart within one process.
*
* This exists because a test cannot get a fresh copy of this module under
* Bare. `import('./repo-activity.js?fresh=x')` returns a DISTINCT module under
* Node but the SAME object under Bare, which resolves modules by realpath and
* ignores the query. Without this, a "second boot" kept the first boot's
* populated cache and `loaded = true`, so the disk-load path was never
* exercised at all.
*
* Test seam only — nothing in the daemon calls it. It deliberately does NOT
* touch the cache FILE, only the process-local state.
*/
export function getRepoActivityStats () {
return { computes, deduped, cached: cache.size, inflight: inflight.size }
}
export function resetRepoActivityCacheForTests () {
cache.clear()
inflight.clear()
computes = 0
deduped = 0
persistPath = null
loaded = false
dirty = false
if (saveTimer) {
clearTimeout(saveTimer)
saveTimer = null
}
}
/**
* Drop a repo's cached activity so the next render recomputes it. Called
* from ref-update-observed on every push (loose-object pushes don't always
* bump the pack mtime, so we can't rely on the mtime key alone).
*
* @param {string} name - repo name
* @returns {boolean} whether an entry was removed
*/
export function invalidateRepoActivity (name) {
if (!name) return false
const had = cache.delete(name)
if (had) {
dirty = true
schedulePersist()
}
return had
}
/**
* Given a sync status object (from hyper-sync), return an array of repo
* records augmented with { description, commits, lastCommit, activity }.
* Per-repo work runs with bounded concurrency; misses recompute + cache.
*
* @param {object} status - { repos: [...], ... } from sync.status()
* @param {Map} reposMap - sync.repos (name -> { path, ... })
* @returns {Promise<Array>}
*/
export async function enrichRepos (status, reposMap) {
if (!status || !Array.isArray(status.repos) || !reposMap) return []
const out = await mapLimit(status.repos, CONCURRENCY, async (base) => {
const info = reposMap.get(base.name)
const repoPath = info && info.path
if (!repoPath) {
return { ...base, description: '', commits: 0, lastCommit: null, activity: new Array(BUCKETS).fill(0) }
}
const extra = await getRepoDetails(base.name, repoPath).catch(() => ({
description: '',
commits: 0,
lastCommit: null,
activity: new Array(BUCKETS).fill(0)
}))
return { ...base, ...extra }
})
if (dirty) schedulePersist()
return out
}
/**
* Name + description only — NO git, so this is safe on a keystroke path.
*
* Exists for repo autocomplete (search-all-repos), which needs to decide WHICH
* repos match before paying for anything expensive. It used the full
* enrichRepos() for that, walking git over the whole fleet to rank a query and
* then discarding all but 8 results.
*
* Description comes from the cached entry when there is one and a small file
* read otherwise. The cached value is used without re-checking the pack mtime:
* a description is a ranking input, not a correctness one, and re-validating it
* would mean the stat-per-repo this function exists to avoid.
*
* @param {object} status - { repos: [...] } from sync.status()
* @param {Map} reposMap - sync.repos (name -> { path, ... })
* @returns {Promise<Array>} base records plus { description }
*/
export async function describeRepos (status, reposMap) {
if (!status || !Array.isArray(status.repos) || !reposMap) return []
return mapLimit(status.repos, CONCURRENCY, async (base) => {
const cached = cache.get(base.name)
if (cached && cached.data) return { ...base, description: cached.data.description || '' }
const info = reposMap.get(base.name)
const repoPath = info && info.path
if (!repoPath) return { ...base, description: '' }
return { ...base, description: await readDescription(repoPath) }
})
}
async function getRepoDetails (name, repoPath) {
// Cache key: pack mtime (changes on new commits) or 0 if no pack yet.
let packMtime = 0
try {
const packDir = join(repoPath, 'objects', 'pack')
const st = await stat(packDir)
packMtime = st.mtimeMs
} catch {}
const cached = cache.get(name)
const now = Date.now()
// Pack mtime match = fresh (the correctness signal). The TTL is only a
// backstop for silent out-of-band edits, so it's generous and works
// across restarts (cachedAt may predate this process).
if (cached && cached.mtime === packMtime && (now - cached.cachedAt) < BACKSTOP_TTL_MS) {
return cached.data
}
// Join a pass already computing this repo rather than forking a second set
// of git processes for the same answer. Keyed on name alone: a push landing
// mid-flight would leave the joiner with the marginally older result, which
// is fine for a cache whose push path calls invalidateRepoActivity()
// explicitly. A rejection propagates to both callers, and enrichRepos
// already catches it into the zeroed default.
const running = inflight.get(name)
if (running) {
deduped++
return running
}
const p = computeRepoDetails(name, repoPath, packMtime)
inflight.set(name, p)
try {
return await p
} finally {
inflight.delete(name)
}
}
async function computeRepoDetails (name, repoPath, packMtime) {
computes++
// Best-effort commit-graph so the single log walk below is fast even on a
// cold OS page cache (incremental; no-op when already current). Then ONE
// git spawn (was three) yields count + last-commit + activity buckets.
await ensureCommitGraph(repoPath)
const [description, stats] = await Promise.all([
readDescription(repoPath),
commitStats(repoPath)
])
// A FAILED walk is rendered as zeros for this pass but is NEVER cached.
//
// Caching it would make a transient git failure permanent: the cache key is
// the pack-dir mtime, so if the repo's pack does not happen to change again
// the zero is served for ever, across restarts, because it is persisted too.
// dc-terrain sat at commits:0 on stortron for exactly this reason while the
// identical `git log --all` returned 514 by hand; hyper-tps escaped only
// because a later pull happened to move its pack mtime.
//
// Returning zeros keeps the page rendering (it degrades to a card without
// counts, which is what enrichRepos already does on a throw) while the next
// sweep retries.
if (stats === null) {
return { description, commits: 0, lastCommit: null, activity: new Array(BUCKETS).fill(0) }
}
const data = { description, ...stats }
cache.set(name, { cachedAt: Date.now(), mtime: packMtime, data })
dirty = true
return data
}
async function readDescription (repoPath) {
try {
const text = (await readFile(join(repoPath, 'description'), 'utf8')).trim()
if (!text || text.startsWith('Unnamed repository')) return ''
return text
} catch {
return ''
}
}
async function ensureCommitGraph (repoPath) {
// Perf cache only; safe to fail (old git may lack the subcommand, or a
// concurrent gc may hold the lock). Incremental — cheap when current.
await execGit(repoPath, ['commit-graph', 'write', '--reachable']).catch(() => {})
}
// ONE git spawn for all three metrics. `git log --all` lists every commit
// reachable from any ref, newest commit-date first, so:
// - line count = total commits (== the old rev-list --count --all)
// - first line = last commit (== the old log --all -1)
// - %at within 365d = the activity buckets (== the old log --since=365d)
// %at is author time — matches each old per-metric call exactly.
async function commitStats (repoPath) {
const out = await execGit(repoPath, ['log', '--all', '--format=%at|%H'])
// null = the walk FAILED (non-zero exit / spawn error), which is NOT the
// same as an empty repo returning ''. The caller must not cache a failure:
// an empty-repo '' is a real answer, a failure is the absence of one.
if (out === null) return null
const buckets = new Array(BUCKETS).fill(0)
const now = Date.now()
const start = now - BUCKETS * DAY_MS
let commits = 0
let lastCommit = null
for (const line of out.split('\n')) {
const s = line.trim()
if (!s) continue
const bar = s.indexOf('|')
if (bar === -1) continue
const ts = parseInt(s.slice(0, bar), 10)
if (!Number.isFinite(ts)) continue
commits++
if (!lastCommit) lastCommit = { timestamp: ts, hash: s.slice(bar + 1) }
const ms = ts * 1000
if (ms >= start && ms <= now) {
const idx = Math.floor((ms - start) / DAY_MS)
if (idx >= 0 && idx < BUCKETS) buckets[idx]++
}
}
return { commits, lastCommit, activity: buckets }
}
/**
* Run git in a bare repo and capture stdout.
*
* Resolves with stdout text on success, or `null` on failure — a spawn error
* or a non-zero exit. The distinction matters: this used to resolve with
* whatever stdout it had captured REGARDLESS of exit code, with stderr routed
* to 'ignore'. A failing `git log --all` therefore produced '', which parses
* to { commits: 0, lastCommit: null, activity: all-zero } — byte-identical to
* a legitimately empty repo — and got cached and persisted as a real answer.
*
* That is not hypothetical. stortron, 2026-09-02: the mirror was mid
* `git fetch` into dc-terrain (a long-running pull; three other pulls started
* AND finished inside its window) when the reconcile sweep walked the same
* bare repo. git exited non-zero, the failure became commits:0, and it stuck —
* the cache key is the pack-dir mtime, that fetch brought nothing new, so the
* mtime never moved and the poison never expired. hyper-tps was hit by the
* same race and only recovered by luck, when a later pull did change its pack.
*
* stderr is captured (bounded) and logged so the NEXT such failure names
* itself instead of being unrecoverable after the fact.
*/
function execGit (repoPath, args) {
return new Promise((resolve) => {
let proc
try {
proc = spawn('git', ['-C', repoPath, ...args], {
stdio: ['ignore', 'pipe', 'pipe']
})
} catch {
return resolve(null)
}
const chunks = []
const errChunks = []
let errBytes = 0
proc.stdout.on('data', (c) => chunks.push(c))
if (proc.stderr) {
proc.stderr.on('data', (c) => {
// Bounded: a pathological repo must not buffer megabytes of stderr.
if (errBytes < 8192) { errChunks.push(c); errBytes += c.length }
})
}
proc.on('error', () => resolve(null))
proc.on('close', (code) => {
if (code !== 0) {
const msg = Buffer.concat(errChunks).toString('utf8').trim().split('\n')[0]
// eslint-disable-next-line no-console
console.error('[repo-activity] git ' + args[0] + ' exited ' + code +
' in ' + repoPath + (msg ? ': ' + msg : ' (no stderr)'))
return resolve(null)
}
resolve(Buffer.concat(chunks).toString('utf8'))
})
})
}
/**
* Map over items with bounded concurrency, preserving input order.
*/
async function mapLimit (items, limit, fn) {
const out = new Array(items.length)
let next = 0
const workers = new Array(Math.min(limit, items.length)).fill(0).map(async () => {
while (true) {
const i = next++
if (i >= items.length) return
out[i] = await fn(items[i], i)
}
})
await Promise.all(workers)
return out
}
// Debounced atomic persist. Coalesces the burst of cache.set() from a
// single enrichRepos pass into one write (temp file + rename).
function schedulePersist () {
if (!persistPath || saveTimer) return
saveTimer = setTimeout(() => {
saveTimer = null
persistNow().catch(() => {})
}, 1000)
if (saveTimer && typeof saveTimer.unref === 'function') saveTimer.unref()
}
/**
* Force any pending persist to disk immediately. Call on graceful shutdown
* (and used by tests to make the debounced write deterministic).
*/
export async function flushRepoActivityCache () {
if (saveTimer) {
clearTimeout(saveTimer)
saveTimer = null
}
await persistNow()
}
async function persistNow () {
if (!persistPath || !dirty) return
dirty = false
const entries = {}
for (const [name, entry] of cache) entries[name] = entry
const tmp = persistPath + '.tmp'
try {
await writeFile(tmp, JSON.stringify({ version: 1, entries }))
await rename(tmp, persistPath)
} catch {
// Persist is best-effort; a failed write just means a colder next boot.
dirty = true
}
}