/**
* RefReconciler — gip-overlay P2: the bare repo follows the autobase.
*
* The per-repo autobase ref-view is the AUTHORITY for a repo's git refs
* (P1 made it fast-forward-enforced). This service projects that view onto
* the on-disk bare repo so HTTP `git clone/push`, `git log`, hooks and CI
* keep working — the bridge half of the design.
*
* On the tracker's `update` event (and once at start) it reads the view
* via `listRefs()` and forces every ref in the bare repo to match:
*
* - create / fast-forward a ref to the view's oid
* - roll a divergent ref BACK to the view's oid (this is how a push that
* was accepted locally but rejected globally, i.e. non-FF in the
* linearized order, gets undone — the view never moved, so the bare
* repo is forced back)
* - delete a ref it previously materialized once the view drops it
*
* Safety: a ref is never pointed at an object the bare repo does not have
* (that would corrupt the repo). Missing objects are requested through the
* injected `ensureObjects(oids)` hook (hyper-overlay in production, P3);
* if the object is still absent afterwards the ref is left untouched.
*
* It only ever deletes refs IT materialized (tracked in `_materialized`) —
* never refs the autobase has not managed (e.g. an internal `release`
* branch), so it can't clobber out-of-band refs.
*/
import { execFile } from '../_compat-exec.js'
const NOOP_LOG = { warn () {}, info () {} }
export class RefReconciler {
constructor (opts = {}) {
if (!opts.tracker) throw new Error('RefReconciler requires { tracker }')
if (!opts.repoPath) throw new Error('RefReconciler requires { repoPath }')
this._tracker = opts.tracker
this._repoPath = opts.repoPath
this._ensureObjects = opts.ensureObjects || null
// ASYNC by default. The git plumbing below runs once per ref per repo,
// and the periodic reconcileAll sweep drives it across every repo — with
// a blocking spawn that starves the HTTP server for the whole sweep.
// An injected sync shim still works: every call site awaits, and awaiting
// a plain value resolves immediately.
this._git = opts.git || execFile
this._log = opts.logger || NOOP_LOG
// Called after a pass that actually changed a ref in the bare repo. Kept as
// a callback so this service stays unaware of what depends on repo content
// — the manager wires it to the activity cache, which knows the repo name.
this._onChange = typeof opts.onChange === 'function' ? opts.onChange : null
// Called with the newest `timestamp` in the view on EVERY pass, no-op or
// not — it describes the view, not this disk. The /repos ordering key; see
// services/repo-ref-activity.js.
this._onRefActivity = typeof opts.onRefActivity === 'function' ? opts.onRefActivity : null
this._materialized = new Map() // refname -> oid we last wrote to the bare repo
this._started = false
this._running = null // in-flight reconcile (serialize concurrent triggers)
this._queued = false // a trigger arrived while a reconcile was running
// ── Sweep-skip bookkeeping (see needsPass) ───────────────────────────
// A ref we could not materialize because its objects were missing. Until
// that clears, the repo is genuinely out of sync and the sweep MUST retry
// it however quiet the tracker looks.
this._deferred = false
this._lastPassAt = 0
this._lastVersion = null // view.version at the end of the last pass
// Backstop: re-pass even when nothing looks changed, so silent drift (a
// missed `update`, an out-of-band edit to the bare repo) still heals.
// Jittered per instance and re-rolled after every pass, so the repos do
// not all fall due on the same sweep and re-create the burst this avoids.
this._backstopBaseMs = typeof opts.backstopMs === 'number' ? opts.backstopMs : 30 * 60 * 1000
this._backstopMs = this._rollBackstop()
// Debounced update trigger. Two distinct reasons, both measured in prod:
//
// 1. COALESCING. `update` fires on every view advance. On a MIRROR every
// tracker is a replication target, so during initial sync — or a re-drain
// after an unclean stop — all N trackers advance continuously. Each
// reconcile costs a `listRefs()`, which is `await base.update()` PLUS a
// full ref range-scan (hyper-issues/lib/hyper-issues.js:502). Undebounced,
// N reconcilers re-run that for every replication chunk. An ORIGIN never
// sees this: its trackers only advance when someone pushes. That asymmetry
// is why this starved the mirror while the origin sat at ~15% CPU.
//
// 2. YIELDING. setTimeout is a MACROTASK. The reconcile chain awaits values
// that resolve from cache, so back-to-back triggers drain only the
// MICROtask queue — inbound HTTP, accept() and SIGCHLD never run. That is
// exactly the trap reconcileAll hit in 0.1.180, where `await` alone did
// NOT fix it and only an explicit macrotask handoff did (see
// test/reconcile-sweep-yields.test.js). Routing every event-driven
// reconcile through a timer hands the loop back between passes.
//
// start() still reconciles immediately, so first materialization is never
// delayed by this.
this._debounceMs = typeof opts.debounceMs === 'number' ? opts.debounceMs : 1500
this._updateTimer = null
this._onUpdate = () => {
if (this._updateTimer) return // a pass is already scheduled; coalesce into it
this._updateTimer = setTimeout(() => {
this._updateTimer = null
this.reconcile().catch((err) => this._log.warn('ref reconcile failed: ' + err.message))
}, this._debounceMs)
}
}
async start () {
if (this._started) return
this._started = true
this._tracker.base.on('update', this._onUpdate)
await this.reconcile()
}
async stop () {
if (!this._started) return
this._started = false
// Cancel a pending debounced pass so teardown doesn't fire a reconcile
// against a tracker the caller is closing.
if (this._updateTimer) {
clearTimeout(this._updateTimer)
this._updateTimer = null
}
try { this._tracker.base.off('update', this._onUpdate) } catch {}
}
// Serialize reconciles. If a trigger lands mid-run, coalesce to a single
// follow-up pass (the view may have advanced again while we were applying).
async reconcile () {
if (this._running) {
this._queued = true
return this._running
}
this._running = this._reconcileOnce().finally(() => {
this._running = null
if (this._queued) {
this._queued = false
this.reconcile().catch((err) => this._log.warn('ref reconcile failed: ' + err.message))
}
})
return this._running
}
_rollBackstop () {
// 0.75x–1.25x of the base, so N reconcilers spread instead of synchronising.
return Math.round(this._backstopBaseMs * (0.75 + Math.random() * 0.5))
}
_viewVersion () {
try {
const view = this._tracker && this._tracker.base && this._tracker.base.view
return view && typeof view.version === 'number' ? view.version : null
} catch {
return null
}
}
/**
* Does the periodic sweep need to run a pass on this repo?
*
* The sweep is a BACKSTOP — per-repo reconcile is already event-driven off the
* tracker's `update`, so a repo that is materialized and quiet gains nothing
* from another `listRefs()` (an `await base.update()` plus a full ref
* range-scan). Across 146 repos that redundant work degraded the daemon from
* 11ms to ~450ms TTFB for ~57s of every 300s.
*
* Answer true whenever we cannot PROVE there is nothing to do:
* - never completed a pass, or
* - a ref is still waiting on objects, or
* - the linearized view advanced since the last pass, or
* - the view version is unreadable (never skip on a broken signal), or
* - the backstop deadline expired.
*
* @returns {boolean}
*/
needsPass () {
if (this._deferred) return true
if (!this._lastPassAt) return true
if (Date.now() - this._lastPassAt >= this._backstopMs) return true
const v = this._viewVersion()
if (v === null || v !== this._lastVersion) return true
return false
}
async _reconcileOnce () {
// Any ref left unmaterialized this pass re-arms the sweep for this repo.
let deferred = false
const refs = await this._tracker.listRefs() // [{ ref, hash, updatedBy, timestamp }]
// The version whose refs we are about to materialize. Sampled HERE — not at
// the end of the pass — because listRefs() does the `base.update()` that
// advances the view, and any advance AFTER this point is work this pass did
// not do. Recording a later version would skip it forever.
const versionReconciled = this._viewVersion()
const desired = new Map()
for (const r of refs) if (r.hash) desired.set(r.ref, r.hash)
// Newest ref timestamp in the view — the REPLICATED last-activity signal
// that /repos orders on. Reported from the array listRefs() already
// returned, so it costs nothing beyond the pass we are in, and reported
// unconditionally: a mirror that is fully in sync never changes a ref, and
// gating this on `changed` would leave exactly those backends ordering on
// local mtimes. A view with no timestamps (an older writer, or no refs at
// all) reports nothing rather than a zero that would sink the repo.
if (this._onRefActivity) {
let newest = 0
for (const r of refs) {
if (r && typeof r.timestamp === 'number' && r.timestamp > newest) newest = r.timestamp
}
if (newest > 0) {
try { this._onRefActivity(newest) } catch (err) {
this._log.warn('onRefActivity hook failed: ' + (err && err.message))
}
}
}
// Did this pass actually change the bare repo? Consumers cache things keyed
// on repo content (see onChange in the constructor).
let changed = false
// Create / move (incl. roll-back) every ref to match the authoritative view.
for (const [ref, oid] of desired) {
const current = await this._readRef(ref)
if (current === oid) { this._materialized.set(ref, oid); continue }
if (!await this._hasObject(oid)) {
if (this._ensureObjects) {
try {
await this._ensureObjects([oid])
} catch (err) {
this._log.warn(`ensureObjects failed for ${ref}@${oid}: ${err.message}`)
deferred = true
continue
}
}
if (!await this._hasObject(oid)) {
this._log.warn(`skip ${ref}: object ${oid} not present (no source)`)
deferred = true
continue
}
}
await this._updateRef(ref, oid)
this._materialized.set(ref, oid)
changed = true
}
// Delete refs we previously materialized that the view has since dropped.
for (const ref of Array.from(this._materialized.keys())) {
if (!desired.has(ref)) {
await this._deleteRef(ref)
this._materialized.delete(ref)
changed = true
}
}
// Repoint a dangling HEAD. `git init --bare` leaves HEAD -> refs/heads/master,
// but a repo whose default branch is `main` (and which never had a `master`)
// ends up with HEAD pointing at a branch that doesn't exist — `git rev-parse
// HEAD` / `git clone` then find no default branch even though every object +
// ref is present. hyper-sync's scanRepos repoints on scan, but a repo the
// reconciler materialized (object pull, not a scan) never got that. Do it
// here so ANY reconciled repo ends with a valid HEAD. pear-git #3.
// Repair a dangling HEAD first (so a valid local value exists to publish),
// then reconcile it against the replicated default branch.
await this._repointHeadIfDangling()
await this._syncHead()
// Refs that arrive by REPLICATION never pass through ref-update-observed,
// which is the only thing that used to invalidate the repo-activity cache.
// Updating a ref does not restamp objects/pack either, so the cache's
// mtime key still matches and a mirror serves a stale commit count until
// the backstop TTL expires. Measured on the live public backend: cached
// commits 19 vs an actual 20, cached mtime identical to the live pack
// mtime, 5.5h stale and not self-correcting. Every repo that arrives by
// replication is affected, and the LB fronts exactly such a node.
if (changed && this._onChange) {
try { await this._onChange() } catch (err) {
this._log.warn('onChange hook failed: ' + (err && err.message))
}
}
// Record what the sweep needs to decide whether to come back (needsPass).
this._deferred = deferred
this._lastPassAt = Date.now()
this._lastVersion = versionReconciled
this._backstopMs = this._rollBackstop()
}
/**
* Make HEAD — the repo's default branch — follow the REPLICATED value.
*
* HEAD is a symbolic ref, and the ref-view schema stores ref→hash pairs, so
* a symbolic ref was never modelable there and HEAD stayed per-backend
* (hyper-sync sets it on scan; _repointHeadIfDangling repairs it with a
* master→main→first heuristic). Two nodes running that heuristic against
* different materialization orders land on DIFFERENT defaults, and the
* default branch is what the repo page renders and what `git clone` checks
* out. Measured across the live pair: 4 of 142 repos disagreed, and which
* one the public saw depended on which backend the load balancer hit.
*
* It rides the EXISTING generic `set-meta` op (meta:head) rather than a new
* op type, so there is no wire-format change, no schema version bump, and no
* schema-guard interaction — a node running older code simply stores the key
* and ignores it.
*
* Two rules make this safe:
*
* - ADOPT ONLY IF THE TARGET EXISTS LOCALLY. A replicated HEAD naming a
* branch whose ref has not materialized here yet would leave a DANGLING
* HEAD, which is strictly worse than divergence: `git clone` then finds
* no default branch at all. When the target is absent we defer — this
* method runs after every materialization pass, so the next one adopts it.
* - SEED ONLY WHEN THE VIEW HAS NONE. A writable node publishes its local
* HEAD only if no value is recorded yet. Otherwise two writers would each
* keep re-publishing their own answer and the repos would flap forever.
* First writer wins; everyone else converges on it.
*/
async _syncHead () {
if (!this._tracker || typeof this._tracker.getMeta !== 'function') return
let target = null
try {
target = await this._tracker.getMeta('head')
} catch { return }
if (target && typeof target === 'string' && target.startsWith('refs/heads/')) {
// Deferred adoption: never point HEAD at a ref we do not have.
if (!await this._readRef(target)) {
this._log.info('replicated HEAD ' + target + ' not materialized yet — deferring')
return
}
const cur = await this._readSymbolicHead()
if (cur === target) return
try {
await this._git('git', ['--git-dir', this._repoPath, 'symbolic-ref', 'HEAD', target], { encoding: 'utf8' })
this._log.info('HEAD -> ' + target + ' (adopted replicated default branch)')
} catch (err) {
this._log.warn('could not adopt replicated HEAD: ' + (err && err.message))
}
return
}
// No replicated value yet. Seed it from this node IF we can write, so the
// rest of the fleet converges on one answer instead of each guessing.
if (!this._tracker.writable || typeof this._tracker.setMeta !== 'function') return
const cur = await this._readSymbolicHead()
if (!cur || !await this._readRef(cur)) return // local HEAD dangling — nothing worth publishing
try {
await this._tracker.setMeta('head', cur)
this._log.info('published default branch ' + cur + ' to the ref-view')
} catch (err) {
this._log.warn('could not publish HEAD: ' + (err && err.message))
}
}
// Current symbolic HEAD as a full refname, or null when detached/unreadable.
async _readSymbolicHead () {
try {
const out = await this._git('git', ['--git-dir', this._repoPath, 'symbolic-ref', '--quiet', 'HEAD'], { encoding: 'utf8' })
const t = (out || '').toString().trim()
return t.startsWith('refs/heads/') ? t : null
} catch {
return null
}
}
// If HEAD is a symbolic ref pointing at a branch that has no ref, repoint it
// to an existing one (prefer master, then main, then the first) — matching
// hyper-sync/lib/repos.js repointHeadIfDangling. No-op when HEAD is detached
// (a raw SHA) or already points at an existing branch.
async _repointHeadIfDangling () {
const branches = []
for (const ref of this._materialized.keys()) {
if (ref.startsWith('refs/heads/')) branches.push(ref.slice('refs/heads/'.length))
}
if (branches.length === 0) return
let cur = null
try {
const out = await this._git('git', ['--git-dir', this._repoPath, 'symbolic-ref', '--quiet', 'HEAD'], { encoding: 'utf8' })
const t = (out || '').toString().trim()
if (!t.startsWith('refs/heads/')) return
cur = t.slice('refs/heads/'.length)
} catch {
return // detached HEAD (raw SHA) — leave alone
}
if (branches.includes(cur)) return // HEAD already points at an existing branch
const target = branches.includes('master') ? 'master'
: branches.includes('main') ? 'main'
: branches[0]
if (cur === target) return
try {
await this._git('git', ['--git-dir', this._repoPath, 'symbolic-ref', 'HEAD', 'refs/heads/' + target], { encoding: 'utf8' })
this._log.info('repointed dangling HEAD -> ' + target)
} catch {}
}
// ── git plumbing (bare repo via --git-dir) ──────────────────────
async _readRef (ref) {
try {
const out = await this._git('git', ['--git-dir', this._repoPath, 'rev-parse', '--verify', '--quiet', ref], { encoding: 'utf8' })
const oid = (out || '').toString().trim()
return oid || null
} catch {
return null // missing / invalid ref
}
}
async _hasObject (oid) {
try {
await this._git('git', ['--git-dir', this._repoPath, 'cat-file', '-e', oid + '^{object}'], { encoding: 'utf8' })
return true
} catch {
return false
}
}
async _updateRef (ref, oid) {
await this._git('git', ['--git-dir', this._repoPath, 'update-ref', ref, oid], { encoding: 'utf8' })
this._log.info(`reconciled ${ref} -> ${oid}`)
}
async _deleteRef (ref) {
await this._git('git', ['--git-dir', this._repoPath, 'update-ref', '-d', ref], { encoding: 'utf8' })
this._log.info(`reconciled delete ${ref}`)
}
}