| … | … | @@ -44,7 +44,7 @@ import { OverlayManager } from './overlay-manager.js' |
| 44 | 44 | import { SyncEngine } from 'hyper-overlay/sync-engine' |
| 45 | 45 | import { TransferManager } from 'hyper-overlay/transfer' |
| 46 | 46 | import { MultiRepoOverlayProtocolV2 } from './multi-repo-overlay-protocol.js' |
| 47 | import { parsePointer, discoverMirrorTargets, runOverlayMirror } from './overlay-mirror.js' | |
| 47 | import { discoverMirrorTargets, runOverlayMirror, listPointersFromGit, singleFlight } from './overlay-mirror.js' | |
| 48 | 48 | |
| 49 | 49 | export const DEFAULT_ROOT = path.join(homedir(), '.pgh') |
| 50 | 50 | export const DEFAULT_SOCKET = path.join(DEFAULT_ROOT, 'svc.sock') |
| … | … | @@ -95,20 +95,27 @@ function safeDirSize (dir) { |
| 95 | 95 | // after stdout/stderr EOF, so output is fully captured) and destroy() each |
| 96 | 96 | // stream to free the fds. `spawn` is injected so tests can pass bare-subprocess |
| 97 | 97 | // directly. Proven by test/subprocess-fd-leak.test.js. |
| 98 | export function spawnCaptureNoLeak (spawn, cmd, args) { | |
| 98 | // | |
| 99 | // `input` (optional) is written to the child's stdin, which is then ended (the | |
| 100 | // overlay mirror feeds `git cat-file --batch` its oids this way). `opts.buffer` | |
| 101 | // resolves a Buffer instead of a string — needed when output carries BYTE | |
| 102 | // counts over possibly-binary content. | |
| 103 | export function spawnCaptureNoLeak (spawn, cmd, args, input, opts = {}) { | |
| 99 | 104 | return new Promise((resolve, reject) => { |
| 100 | 105 | const p = spawn(cmd, args) |
| 101 | let out = '' | |
| 106 | const chunks = [] | |
| 102 | 107 | let err = '' |
| 103 | p.stdout.on('data', (d) => { out += d }) | |
| 108 | p.stdout.on('data', (d) => { chunks.push(typeof d === 'string' ? Buffer.from(d) : d) }) | |
| 104 | 109 | p.stderr.on('data', (d) => { err += d }) |
| 105 | 110 | p.on('error', reject) |
| 106 | 111 | p.on('close', (code) => { |
| 107 | 112 | try { p.stdout.destroy() } catch {} |
| 108 | 113 | try { p.stderr.destroy() } catch {} |
| 109 | 114 | try { if (p.stdin) p.stdin.destroy() } catch {} |
| 110 | code === 0 ? resolve(out) : reject(new Error(((err || '') + '').trim() || (cmd + ' exited ' + code))) | |
| 115 | const out = Buffer.concat(chunks) | |
| 116 | code === 0 ? resolve(opts.buffer ? out : out.toString('utf8')) : reject(new Error(((err || '') + '').trim() || (cmd + ' exited ' + code))) | |
| 111 | 117 | }) |
| 118 | if (input != null && p.stdin) p.stdin.end(input) | |
| 112 | 119 | }) |
| 113 | 120 | } |
| 114 | 121 | |
| … | … | @@ -981,7 +988,15 @@ export class Sidecar { |
| 981 | 988 | // .pear-git/overlay.json peerServerKey ≠ our identity), fetch its overlay |
| 982 | 989 | // large files by oid into the content-addressed mirror-spool so this backend |
| 983 | 990 | // re-serves them. Best-effort; never throws into boot/interval. |
| 984 | async _runOverlayMirror () { | |
| 991 | // One pass at a time: the 5-min interval used to start a new pass while the | |
| 992 | // last still ran, and on stortron they piled up until the daemon wedged | |
| 993 | // (2026-10-11). A call during a pass joins it. | |
| 994 | _runOverlayMirror () { | |
| 995 | if (!this._overlayMirrorOnce) this._overlayMirrorOnce = singleFlight(() => this._overlayMirrorPass()) | |
| 996 | return this._overlayMirrorOnce() | |
| 997 | } | |
| 998 | ||
| 999 | async _overlayMirrorPass () { | |
| 985 | 1000 | if (!this._overlayProtocolV2 || !this._overlaySwarm || !this._overlayMirrorSpool) return |
| 986 | 1001 | // ASYNC spawn (NOT spawnSync): this runs in the always-on bare daemon, and |
| 987 | 1002 | // listing pointers shells out to git once per small blob (hundreds for a |
| … | … | @@ -1000,22 +1015,18 @@ export class Sidecar { |
| 1000 | 1015 | const readOverlayJson = async (repo) => { |
| 1001 | 1016 | try { return JSON.parse(await git(repo, ['show', 'HEAD:.pear-git/overlay.json'])) } catch { return null } |
| 1002 | 1017 | } |
| 1018 | // Two spawns per repo: ls-tree + ONE `cat-file --batch` fed the small-blob | |
| 1019 | // oids on stdin. It was one `cat-file -p` per blob — ~4,000 forks of a | |
| 1020 | // multi-GB daemon per pass on stortron (2026-10-11), see listPointersFromGit. | |
| 1003 | 1021 | const listPointers = async (repo) => { |
| 1004 | let out = '' | |
| 1005 | try { out = await git(repo, ['ls-tree', '-r', '-l', 'HEAD']) } catch { return [] } | |
| 1006 | const pointers = [] | |
| 1007 | for (const line of out.split('\n')) { | |
| 1008 | // "<mode> blob <oid> <size>\t<path>" — pointer blobs are tiny, so skip | |
| 1009 | // anything bigger than a pointer to avoid cat-file'ing real in-git blobs. | |
| 1010 | const m = line.match(/^\d+\s+blob\s+([0-9a-f]+)\s+(\d+)\t(.+)$/) | |
| 1011 | if (!m) continue | |
| 1012 | if (Number(m[2]) > 512) continue | |
| 1013 | let blob = '' | |
| 1014 | try { blob = await git(repo, ['cat-file', '-p', m[1]]) } catch { continue } | |
| 1015 | const p = parsePointer(blob) | |
| 1016 | if (p) pointers.push({ path: m[3], oid: p.oid, size: p.size }) | |
| 1017 | } | |
| 1018 | return pointers | |
| 1022 | const gd = path.join(reposDir, repo.endsWith('.git') ? repo : repo + '.git') | |
| 1023 | try { | |
| 1024 | return await listPointersFromGit({ | |
| 1025 | lsTree: () => git(repo, ['ls-tree', '-r', '-l', 'HEAD']), | |
| 1026 | catFileBatch: (oids) => spawnCaptureNoLeak(spawn, 'git', ['--git-dir', gd, 'cat-file', '--batch'], | |
| 1027 | oids.join('\n') + '\n', { buffer: true }) | |
| 1028 | }) | |
| 1029 | } catch { return [] } | |
| 1019 | 1030 | } |
| 1020 | 1031 | |
| 1021 | 1032 | const repos = fs.readdirSync(reposDir).filter(d => d.endsWith('.git')).map(d => d.replace(/\.git$/, '')) |
| … | … | @@ -42,6 +42,75 @@ export function parsePointer (textOrBuf) { |
| 42 | 42 | return oid ? { oid, size } : null |
| 43 | 43 | } |
| 44 | 44 | |
| 45 | // Pointer blobs are tiny; anything bigger is a real in-git blob, never read. | |
| 46 | const MAX_POINTER_BYTES = 512 | |
| 47 | ||
| 48 | /** | |
| 49 | * Parse `git cat-file --batch` output → Map(oid → content string). | |
| 50 | * Each entry is "<oid> <type> <size>\n<size BYTES>\n"; "<oid> missing\n" is | |
| 51 | * skipped. Parsed by BYTES (Buffer): sizes are byte counts, and a small blob may | |
| 52 | * be binary — decoding to a string first would misalign every later entry. | |
| 53 | */ | |
| 54 | export function parseCatFileBatch (out) { | |
| 55 | const buf = Buffer.isBuffer(out) ? out : Buffer.from(String(out || ''), 'utf8') | |
| 56 | const map = new Map() | |
| 57 | let i = 0 | |
| 58 | while (i < buf.length) { | |
| 59 | const nl = buf.indexOf(0x0a, i) | |
| 60 | if (nl === -1) break | |
| 61 | const header = buf.subarray(i, nl).toString('utf8').split(' ') | |
| 62 | i = nl + 1 | |
| 63 | if (header.length !== 3) continue // "<oid> missing" / unexpected | |
| 64 | const size = Number(header[2]) | |
| 65 | if (!Number.isFinite(size) || i + size > buf.length) break | |
| 66 | map.set(header[0], buf.subarray(i, i + size).toString('utf8')) | |
| 67 | i += size + 1 // content + trailing LF | |
| 68 | } | |
| 69 | return map | |
| 70 | } | |
| 71 | ||
| 72 | /** | |
| 73 | * A repo's overlay pointers from `git ls-tree -r -l HEAD` output plus ONE | |
| 74 | * `git cat-file --batch` for every blob small enough to be a pointer. | |
| 75 | * The old path spawned `git cat-file -p` per blob — ~4,000 forks of a multi-GB | |
| 76 | * daemon per pass on stortron (2026-10-11). | |
| 77 | * | |
| 78 | * lsTree async () => ls-tree text | |
| 79 | * catFileBatch async (oids[]) => batch output (Buffer or string) | |
| 80 | */ | |
| 81 | export async function listPointersFromGit ({ lsTree, catFileBatch }) { | |
| 82 | const small = [] | |
| 83 | for (const line of String(await lsTree()).split('\n')) { | |
| 84 | const m = line.match(/^\d+\s+blob\s+([0-9a-f]+)\s+(\d+)\t(.+)$/) | |
| 85 | if (m && Number(m[2]) <= MAX_POINTER_BYTES) small.push({ oid: m[1], path: m[3] }) | |
| 86 | } | |
| 87 | if (small.length === 0) return [] | |
| 88 | const objects = parseCatFileBatch(await catFileBatch(small.map(s => s.oid))) | |
| 89 | const pointers = [] | |
| 90 | for (const s of small) { | |
| 91 | const p = parsePointer(objects.get(s.oid) || '') | |
| 92 | if (p) pointers.push({ path: s.path, oid: p.oid, size: p.size }) | |
| 93 | } | |
| 94 | return pointers | |
| 95 | } | |
| 96 | ||
| 97 | /** | |
| 98 | * Wrap an async fn so a call made while a run is in flight joins that run | |
| 99 | * instead of starting another. The mirror runs on a 5-min interval; a pass that | |
| 100 | * outlived it used to start a second, then a third — on stortron they piled up | |
| 101 | * until the daemon wedged (2026-10-11). | |
| 102 | */ | |
| 103 | export function singleFlight (fn) { | |
| 104 | let running = null | |
| 105 | return function () { | |
| 106 | if (running) return running | |
| 107 | running = (async () => { | |
| 108 | try { return await fn() } finally { running = null } | |
| 109 | })() | |
| 110 | return running | |
| 111 | } | |
| 112 | } | |
| 113 | ||
| 45 | 114 | /** |
| 46 | 115 | * From a list of {path, oid, size} pointers, the de-duplicated set still missing |
| 47 | 116 | * from the content-addressed mirror-spool (<mirrorSpool>/<oid> absent on disk). |
| … | … | @@ -1,6 +1,6 @@ |
| 1 | 1 | { |
| 2 | 2 | "name": "pgh-sidecar", |
| 3 | "version": "0.2.64", | |
| 3 | "version": "0.2.65", | |
| 4 | 4 | "description": "Small optional user-level daemon for pgh \u2014 shared identity vault, OTA polling, device linking, bootstrap seed, NameRegistry host", |
| 5 | 5 | "type": "module", |
| 6 | 6 | "main": "lib/index.js", |
| … | … | @@ -0,0 +1,84 @@ |
| 1 | /** | |
| 2 | * overlay-mirror must not fork the daemon once per blob, and passes must not overlap. | |
| 3 | * | |
| 4 | * 2026-10-11 stortron: listPointers ran one `git cat-file -p` per small blob | |
| 5 | * (~4,000 spawns a pass for hypercore-kb + hyper-tps), a pass outlived its 5-min | |
| 6 | * setInterval, and passes piled up. Fork cost grows with the daemon's RSS, so it | |
| 7 | * fed itself: the main thread spent 98% of 44 h of CPU in the kernel, the UDS | |
| 8 | * accept backlog filled (511/511), and the box ran out of RAM + swap. | |
| 9 | */ | |
| 10 | ||
| 11 | import test from 'brittle' | |
| 12 | import { spawn } from '#node:child_process' | |
| 13 | ||
| 14 | import { parseCatFileBatch, listPointersFromGit, singleFlight } from '../lib/overlay-mirror.js' | |
| 15 | import { spawnCaptureNoLeak } from '../lib/index.js' | |
| 16 | ||
| 17 | const OID = 'a'.repeat(64) | |
| 18 | const OID2 = 'b'.repeat(64) | |
| 19 | const ptr = (oid, size) => 'hyper-overlay v1\noid blake2b:' + oid + '\nsize ' + size + '\n' | |
| 20 | ||
| 21 | test('parseCatFileBatch — oid -> content, contents may contain newlines; missing objects skipped', (t) => { | |
| 22 | const a = ptr(OID, 10) | |
| 23 | const b = 'just text\nwith lines' | |
| 24 | const out = '1111 blob ' + Buffer.byteLength(a) + '\n' + a + '\n' + | |
| 25 | '2222 missing\n' + | |
| 26 | '3333 blob ' + Buffer.byteLength(b) + '\n' + b + '\n' | |
| 27 | const m = parseCatFileBatch(out) | |
| 28 | t.is(m.get('1111'), a) | |
| 29 | t.is(m.get('3333'), b) | |
| 30 | t.absent(m.has('2222')) | |
| 31 | }) | |
| 32 | ||
| 33 | test('parseCatFileBatch — parses by BYTES: a binary blob does not misalign the next entry', (t) => { | |
| 34 | const bin = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0xff, 0xfe, 0x0a, 0x00, 0xc3]) | |
| 35 | const p = Buffer.from(ptr(OID, 3)) | |
| 36 | const out = Buffer.concat([ | |
| 37 | Buffer.from('1111 blob ' + bin.length + '\n'), bin, Buffer.from('\n'), | |
| 38 | Buffer.from('2222 blob ' + p.length + '\n'), p, Buffer.from('\n') | |
| 39 | ]) | |
| 40 | const m = parseCatFileBatch(out) | |
| 41 | t.is(m.get('2222'), p.toString('utf8'), 'pointer after a binary blob parsed intact') | |
| 42 | }) | |
| 43 | ||
| 44 | test('listPointersFromGit — ONE batch call for all small blobs; big blobs never read', async (t) => { | |
| 45 | const lsTree = [ | |
| 46 | '100644 blob 1111 ' + Buffer.byteLength(ptr(OID, 5)) + '\ttex/a.png', | |
| 47 | '100644 blob 2222 12\tREADME.md', | |
| 48 | '100644 blob 3333 ' + Buffer.byteLength(ptr(OID2, 7)) + '\ttex/b.png', | |
| 49 | '100644 blob 4444 900000\tbig/in-git.bin', | |
| 50 | '040000 tree 5555 -\tdir' | |
| 51 | ].join('\n') | |
| 52 | const objects = { 1111: ptr(OID, 5), 2222: 'hello world\n', 3333: ptr(OID2, 7) } | |
| 53 | const batches = [] | |
| 54 | const pointers = await listPointersFromGit({ | |
| 55 | lsTree: async () => lsTree, | |
| 56 | catFileBatch: async (oids) => { | |
| 57 | batches.push(oids) | |
| 58 | return oids.map(o => o + ' blob ' + Buffer.byteLength(objects[o]) + '\n' + objects[o] + '\n').join('') | |
| 59 | } | |
| 60 | }) | |
| 61 | t.is(batches.length, 1, 'one batch, not one spawn per blob') | |
| 62 | t.alike(batches[0], ['1111', '2222', '3333'], 'only blobs small enough to be pointers') | |
| 63 | t.alike(pointers, [{ path: 'tex/a.png', oid: OID, size: 5 }, { path: 'tex/b.png', oid: OID2, size: 7 }]) | |
| 64 | }) | |
| 65 | ||
| 66 | test('singleFlight — a call while one is running does not start a second run', async (t) => { | |
| 67 | let runs = 0 | |
| 68 | let release | |
| 69 | const gate = new Promise(r => { release = r }) | |
| 70 | const pass = singleFlight(async () => { runs++; await gate; return runs }) | |
| 71 | const p1 = pass() | |
| 72 | const p2 = pass() | |
| 73 | t.is(runs, 1, 'second call joined the first') | |
| 74 | release() | |
| 75 | t.is(await p1, 1) | |
| 76 | t.is(await p2, 1) | |
| 77 | await pass() | |
| 78 | t.is(runs, 2, 'a later call runs again') | |
| 79 | }) | |
| 80 | ||
| 81 | test('spawnCaptureNoLeak feeds stdin when given input', async (t) => { | |
| 82 | const out = await spawnCaptureNoLeak(spawn, 'cat', [], 'line one\nline two\n') | |
| 83 | t.is(out, 'line one\nline two\n') | |
| 84 | }) |