/**
* overlay-mirror — make a forge backend a SECOND source for a repo's overlay
* large files (#66 / hyper-tps). The overlay (pgh-overlay) is single-publisher:
* bytes live only on the origin (overlay.json `peerServerKey`). A mirror backend
* (e.g. stortron) has the bare repo but none of the large-file bytes, so it can't
* serve them and they "don't replicate".
*
* This module is the overlay analogue of pear-git-v3's RepoPuller: discover repos
* published elsewhere, fetch their large files by oid into a CONTENT-ADDRESSED
* mirror-spool (<mirrorSpool>/<oid>), and let the v2 serve-from-spool path
* (hyper-overlay 0.2.8 `<syncBaseDir>/<contentHash>`) re-serve them. v2's
* multi-source then makes the mirror a real second source.
*
* The bytes are addressed by the pointer's oid (== blake2b content hash ==
* overlayV2 contentHash), so a forge backend needs NO working tree — it stores
* everything content-addressed in the spool.
*
* Pure/injectable so the logic unit-tests without git, swarm, or overlay.
*/
const POINTER_MAGIC = 'hyper-overlay v1'
/**
* Parse a `hyper-overlay v1` pointer blob → { oid, size } | null.
* hyper-overlay v1
* oid blake2b:<64-hex>
* size <n>
* The oid is returned WITHOUT the `blake2b:` prefix (== overlayV2 contentHash).
*/
export function parsePointer (textOrBuf) {
const text = (textOrBuf && textOrBuf.toString) ? textOrBuf.toString('utf8') : String(textOrBuf || '')
const lines = text.split('\n')
if (lines[0] !== POINTER_MAGIC) return null
let oid = null
let size = null
for (let i = 1; i < lines.length; i++) {
const mo = lines[i].match(/^oid\s+(?:blake2b:)?([0-9a-f]{64})\s*$/)
if (mo) oid = mo[1]
const ms = lines[i].match(/^size\s+(\d+)\s*$/)
if (ms) size = Number(ms[1])
}
return oid ? { oid, size } : null
}
// Pointer blobs are tiny; anything bigger is a real in-git blob, never read.
const MAX_POINTER_BYTES = 512
/**
* Parse `git cat-file --batch` output → Map(oid → content string).
* Each entry is "<oid> <type> <size>\n<size BYTES>\n"; "<oid> missing\n" is
* skipped. Parsed by BYTES (Buffer): sizes are byte counts, and a small blob may
* be binary — decoding to a string first would misalign every later entry.
*/
export function parseCatFileBatch (out) {
const buf = Buffer.isBuffer(out) ? out : Buffer.from(String(out || ''), 'utf8')
const map = new Map()
let i = 0
while (i < buf.length) {
const nl = buf.indexOf(0x0a, i)
if (nl === -1) break
const header = buf.subarray(i, nl).toString('utf8').split(' ')
i = nl + 1
if (header.length !== 3) continue // "<oid> missing" / unexpected
const size = Number(header[2])
if (!Number.isFinite(size) || i + size > buf.length) break
map.set(header[0], buf.subarray(i, i + size).toString('utf8'))
i += size + 1 // content + trailing LF
}
return map
}
/**
* A repo's overlay pointers from `git ls-tree -r -l HEAD` output plus ONE
* `git cat-file --batch` for every blob small enough to be a pointer.
* The old path spawned `git cat-file -p` per blob — ~4,000 forks of a multi-GB
* daemon per pass on stortron (2026-10-11).
*
* lsTree async () => ls-tree text
* catFileBatch async (oids[]) => batch output (Buffer or string)
*/
export async function listPointersFromGit ({ lsTree, catFileBatch }) {
const small = []
for (const line of String(await lsTree()).split('\n')) {
const m = line.match(/^\d+\s+blob\s+([0-9a-f]+)\s+(\d+)\t(.+)$/)
if (m && Number(m[2]) <= MAX_POINTER_BYTES) small.push({ oid: m[1], path: m[3] })
}
if (small.length === 0) return []
const objects = parseCatFileBatch(await catFileBatch(small.map(s => s.oid)))
const pointers = []
for (const s of small) {
const p = parsePointer(objects.get(s.oid) || '')
if (p) pointers.push({ path: s.path, oid: p.oid, size: p.size })
}
return pointers
}
/**
* Wrap an async fn so a call made while a run is in flight joins that run
* instead of starting another. The mirror runs on a 5-min interval; a pass that
* outlived it used to start a second, then a third — on stortron they piled up
* until the daemon wedged (2026-10-11).
*/
export function singleFlight (fn) {
let running = null
return function () {
if (running) return running
running = (async () => {
try { return await fn() } finally { running = null }
})()
return running
}
}
/**
* From a list of {path, oid, size} pointers, the de-duplicated set still missing
* from the content-addressed mirror-spool (<mirrorSpool>/<oid> absent on disk).
*/
export function selectMissingOids (pointers, mirrorSpool, deps) {
const fs = deps.fs
const path = deps.path
const out = []
const seen = new Set()
for (const p of pointers || []) {
if (!p || !p.oid || seen.has(p.oid)) continue
seen.add(p.oid)
let present = false
try { present = fs.statSync(path.join(mirrorSpool, p.oid)).isFile() } catch {}
if (!present) out.push(p)
}
return out
}
/**
* Discover repos this node should MIRROR: a repo published elsewhere (its
* overlay.json `peerServerKey` is someone else's, not our own identity).
*
* @param {object} o
* o.repos string[] — repo names available locally (bare repos)
* o.selfPubkeyHex our identity public key hex (skip repos we publish)
* o.readOverlayJson (repo) => { peerServerKey } | null — reads the repo's
* .pear-git/overlay.json (git, injected)
* @returns {Array<{ repo, peerServerKey }>}
*/
export async function discoverMirrorTargets ({ repos, selfPubkeyHex, readOverlayJson, log }) {
const targets = []
for (const repo of repos || []) {
let cfg = null
try { cfg = await readOverlayJson(repo) } catch {}
if (!cfg || !cfg.peerServerKey) continue
if (cfg.peerServerKey === selfPubkeyHex) continue // we ARE the publisher — nothing to mirror
targets.push({ repo, peerServerKey: cfg.peerServerKey })
}
return targets
}
/**
* Run the mirror: for each target repo, read its pointer oids and fetch the ones
* missing from the spool. The fetch (join publisher topic + pull by oid into
* <mirrorSpool>/<oid>) is injected so this orchestration unit-tests cleanly.
*
* @param {object} o
* o.targets [{ repo, peerServerKey }] (from discoverMirrorTargets)
* o.mirrorSpool content-addressed spool dir
* o.listPointers async (repo) => [{ path, oid, size }] — repo's overlay pointers
* o.fetchOid async (peerServerKey, oid, destPath) => boolean — pull one blob
* o.deps { fs, path }
* o.log? (msg) => void
* @returns {Promise<Array<{ repo, fetched, missing, total }>>}
*/
export async function runOverlayMirror ({ targets, mirrorSpool, listPointers, fetchOid, deps, log }) {
const { path } = deps
const summary = []
for (const t of targets || []) {
let pointers = []
try { pointers = await listPointers(t.repo) } catch (err) {
if (log) log('[overlay-mirror] list pointers failed for ' + t.repo + ': ' + (err && err.message))
continue
}
const missing = selectMissingOids(pointers, mirrorSpool, deps)
let fetched = 0
for (const p of missing) {
const dest = path.join(mirrorSpool, p.oid)
try {
if (await fetchOid(t.peerServerKey, p.oid, dest)) fetched++
} catch (err) {
if (log) log('[overlay-mirror] fetch failed ' + p.oid.slice(0, 12) + '… (' + t.repo + '): ' + (err && err.message))
}
}
summary.push({ repo: t.repo, fetched, missing: missing.length, total: pointers.length })
if (log && (fetched || missing.length)) {
log('[overlay-mirror] ' + t.repo + ': fetched ' + fetched + '/' + missing.length + ' missing (' + pointers.length + ' overlay files)')
}
}
return summary
}