/**
* pgh-sidecar — user-level daemon. Owns the shared pgh state + acts as
* the single writer/reader of the release-registry corestore.
*
* Responsibilities:
* - Shared identity vault (hyperident-backed, at ~/.pgh/identity/)
* - TreeStore ownership
* - Release-registry ownership (single corestore writer) + advertiser
* on Hyperswarm
* - DeployServer seeding the tree store on the hyperident topic
* (folded in from the legacy `pgh seed` command)
* - UDS RPC for every pgh CLI operation that touches the registry —
* publish-release, releases-list/latest/get — so pgh CLI processes
* don't fight the sidecar for the RocksDB lock
*
* - OTA polling across installed apps (stubbed; wires in 0.5f)
* - Blind-pairing device linker (stubbed; wires in 0.5c phase 2c)
*
* UDS at ~/.pgh/svc.sock. Intended to run under
* `systemctl --user enable pgh-sidecar` or equivalent.
*/
import process from '#node:process'
import fs from '#node:fs'
import path from '#node:path'
import os, { homedir } from '#node:os'
import b4a from 'b4a'
import Corestore from 'corestore'
import Hyperswarm from 'hyperswarm'
import Protomux from 'protomux'
import crypto from 'hypercore-crypto'
import { Service } from 'hyper-svc'
import { DeployServer } from 'hyper-deploy'
import { TreeStore, DEFAULT_ROOT as PGH_STORE_ROOT } from 'pgh/store'
import { loadOrCreateIdentity, identityRootHex } from 'pgh/identity'
import { ReleaseRegistry } from 'pgh/release-registry'
import { NameRegistry } from 'pgh/name-registry'
import { readSubscriptions as readSubscriptionsFromDisk, readNameSubscriptions, addNameSubscription } from 'pgh/subscriptions'
import { DEFAULT_NAME_REGISTRIES } from 'pgh/bootstrap'
import { OverlayManager } from './overlay-manager.js'
import { SyncEngine } from 'hyper-overlay/sync-engine'
import { TransferManager } from 'hyper-overlay/transfer'
import { MultiRepoOverlayProtocolV2 } from './multi-repo-overlay-protocol.js'
import { discoverMirrorTargets, runOverlayMirror, listPointersFromGit, singleFlight } from './overlay-mirror.js'
export const DEFAULT_ROOT = path.join(homedir(), '.pgh')
export const DEFAULT_SOCKET = path.join(DEFAULT_ROOT, 'svc.sock')
export const DEFAULT_IDENTITY_PATH = path.join(DEFAULT_ROOT, 'identity')
// Re-export pgh's canonical store root so sidecar and pgh CLI agree
// on where trees live (otherwise publish + seed would diverge).
export const DEFAULT_STORE_ROOT = PGH_STORE_ROOT
export const DEFAULT_REGISTRY_STORE = path.join(homedir(), '.config', 'pgh', 'release-registry')
export const DEFAULT_NAME_REGISTRY_STORE = path.join(homedir(), '.config', 'pgh', 'name-registry')
const SIDECAR_VERSION = '0.2.0'
// Store retention: each app's newest N releases stay; trees only older
// releases use are swept whatever their age (pgh store.gc keepReleases).
// The 14-day age gate alone kept 75 hyper-tps .pck copies (~44 GiB) on
// tabertron, 2026-10-11 — that app publishes several times a day.
const KEEP_RELEASES = Math.max(1, parseInt(process.env.PGH_KEEP_RELEASES, 10) || 2)
// Recursive disk size for an on-disk dir. Used for compact() before/after
// reporting and for storage sweep logs. Returns 0 on any error (e.g. dir
// doesn't exist) — sweep logging is best-effort, never load-bearing.
function safeDirSize (dir) {
if (!dir) return 0
let total = 0
function walk (p) {
let stat
try { stat = fs.lstatSync(p) } catch { return }
if (stat.isSymbolicLink()) return
if (stat.isFile()) { total += stat.size; return }
if (stat.isDirectory()) {
let entries
try { entries = fs.readdirSync(p) } catch { return }
for (const e of entries) walk(path.join(p, e))
}
}
try { walk(dir) } catch {}
return total
}
// Spawn a child, capture stdout, and — critically — RELEASE the parent-side
// stdio socketpair fds when it finishes. bare-subprocess backs child stdio with
// socketpairs and does NOT auto-close them on 'exit'; resolving on 'exit' (or
// never destroying the streams) leaks ~1 fd per spawn. The overlay-mirror runs
// ~one git per repo/pointer every 5 min, so that pattern leaked ~89 fds/min and
// climbed the always-on sidecar to 232k fds / 15.8 GB in 43h on stortron
// (2026-07-13). These are anonymous socketpairs — invisible to the hyper-svc
// UdsServer idle reaper, so nothing reclaims them. We listen on 'close' (fires
// after stdout/stderr EOF, so output is fully captured) and destroy() each
// stream to free the fds. `spawn` is injected so tests can pass bare-subprocess
// directly. Proven by test/subprocess-fd-leak.test.js.
//
// `input` (optional) is written to the child's stdin, which is then ended (the
// overlay mirror feeds `git cat-file --batch` its oids this way). `opts.buffer`
// resolves a Buffer instead of a string — needed when output carries BYTE
// counts over possibly-binary content.
export function spawnCaptureNoLeak (spawn, cmd, args, input, opts = {}) {
return new Promise((resolve, reject) => {
const p = spawn(cmd, args)
const chunks = []
let err = ''
p.stdout.on('data', (d) => { chunks.push(typeof d === 'string' ? Buffer.from(d) : d) })
p.stderr.on('data', (d) => { err += d })
p.on('error', reject)
p.on('close', (code) => {
try { p.stdout.destroy() } catch {}
try { p.stderr.destroy() } catch {}
try { if (p.stdin) p.stdin.destroy() } catch {}
const out = Buffer.concat(chunks)
code === 0 ? resolve(opts.buffer ? out : out.toString('utf8')) : reject(new Error(((err || '') + '').trim() || (cmd + ' exited ' + code)))
})
if (input != null && p.stdin) p.stdin.end(input)
})
}
export class Sidecar {
constructor (opts = {}) {
this.root = opts.root || DEFAULT_ROOT
this.socketPath = opts.socketPath || path.join(this.root, 'svc.sock')
this.identityPath = opts.identityPath || path.join(this.root, 'identity')
this.storeRoot = opts.storeRoot || DEFAULT_STORE_ROOT
this.registryStore = opts.registryStore || DEFAULT_REGISTRY_STORE
this.nameRegistryStore = opts.nameRegistryStore || DEFAULT_NAME_REGISTRY_STORE
this.releaseRegistryReplicasDir = opts.releaseRegistryReplicasDir ||
path.join(homedir(), '.config', 'pgh', 'release-registries')
// Bootstrap key for an existing global NameRegistry. When omitted,
// each sidecar creates its own registry on first boot (local to
// this user). For a shared registry across users, ship a hex key
// in pgh config / env PGH_NAME_REGISTRY_KEY.
this.nameRegistryBootstrap = opts.nameRegistryBootstrap ||
process.env.PGH_NAME_REGISTRY_KEY || null
// Comma-separated list of release-registry keys to replicate as
// read-only warm replicas. CLI's `pgh app install pgh://<key>/...`
// queries these via UDS instead of opening a fresh one-shot
// replica on every call. Replication happens in the background
// and stays current via autobase update events — the CLI sees
// whatever the replica has at query time.
// Three sources merged: opts > env var > persistent config file.
// The config file (~/.config/pgh/subscriptions.json) is the canonical
// path — pgh install auto-adds to it. Env var and opts are for tests
// and bootstrap scenarios.
const fromConfig = readSubscriptionsFromDisk()
const fromEnv = process.env.PGH_RELEASE_REGISTRIES
? process.env.PGH_RELEASE_REGISTRIES.split(',').map(s => s.trim()).filter(Boolean)
: []
this.releaseRegistrySubscriptions = opts.releaseRegistrySubscriptions ||
[...new Set([...fromConfig, ...fromEnv])]
// NameRegistry subscriptions — read-only replicas of OTHER publishers'
// name registries so bare app-name resolution works cross-publisher.
// Sourced from subscriptions.json `nameRegistries` + PGH_NAME_REGISTRIES.
//
// When BOTH are empty (a fresh machine — nothing seeds either file) fall
// back to pgh's DEFAULT_NAME_REGISTRIES. Without it a fresh peer boots its
// own empty NameRegistry, `_resolveNameAcross` has nothing to search, and
// the documented `pgh app install <name>` fails on every bare name. Any
// explicit configuration WINS OUTRIGHT rather than merging, so a peer that
// subscribes to its own set of namespace registries never has the default
// forced on it (and `pgh names unsubscribe`-style edits stay meaningful).
const nameFromConfig = readNameSubscriptions()
const nameFromEnv = process.env.PGH_NAME_REGISTRIES
? process.env.PGH_NAME_REGISTRIES.split(',').map(s => s.trim()).filter(Boolean)
: []
const nameConfigured = [...new Set([...nameFromConfig, ...nameFromEnv])]
this.nameRegistrySubscriptions = opts.nameRegistrySubscriptions ||
(nameConfigured.length ? nameConfigured : [...DEFAULT_NAME_REGISTRIES])
this.passphrase = opts.passphrase || null
// Seeding can be disabled in tests (no swarm / no DeployServer)
this.enableSeed = opts.enableSeed !== false
this._identity = null // identity info (keypair + optional rootPubkey + name)
this._treeStore = null // TreeStore
this._svc = null // hyper-svc Service
this._registryStore = null // Corestore for release-registry
this._registry = null // ReleaseRegistry instance
this._nameStore = null // Corestore for name-registry
this._nameRegistry = null // NameRegistry instance
this._nameRegistrySwarm = null
this._deployServer = null // hyper-deploy DeployServer (advertises trees)
this._deployIndexStore = null // persistent Corestore for DeployServer's fileIndex (see _open)
this._registrySwarm = null // Hyperswarm advertising the release-registry
this._releaseReplicas = [] // [{ keyHex, store, registry, swarm }] — warm replicas
this._nameReplicas = [] // [{ keyHex, store, registry, swarm }] — read-only NameRegistry replicas
// Running-app registry. Apps register with us on boot so `pgh app
// resync|start|stop|restart|status` can find their UDS. Persisted
// to disk so sidecar restarts don't lose the map — otherwise every
// app would need its own retry/watch logic, and `pgh app resync`
// fails with "not registered" for the window between sidecar back
// up and app figuring out to re-register.
this._apps = new Map() // name → { name, udsPath, pid, runtime, version, treeHash, registeredAt }
this._appsFile = path.join(this.root, 'apps.json')
// Per-app last-notified version, so `on('update')` firing repeatedly
// doesn't spam `release-ready` RPCs when nothing changed for that app.
this._lastNotifiedVersion = new Map()
// Per-app install mutex. autobase update events fire on every append;
// without this, two near-simultaneous events fire two concurrent
// `pgh app install` runs for the same app — both rmSync the install
// dir and materialize into it, racing each other. The losing install
// ENOTEMPTYs mid-rm and leaves an empty (or partially-populated)
// install dir behind.
this._installInFlight = new Set()
// Per-app persistent-failure backoff. Keyed by app name →
// { token, count, nextRetryAt }. An autoupdate that fails the SAME way every
// tick (un-fetchable tree, a registry Corestore whose fd-lock can't be
// acquired, etc.) must NOT be retried on every autobase 'update' event — that
// zero-backoff tight loop bled the sidecar to 232k fds / 15.8 GB on stortron
// (2026-07-13). We back off exponentially per (name, token); a new token
// (version/tree bump) resets the counter for a fresh attempt.
this._installFailures = new Map()
// pear-git #47: one armed retry per app whose last autoupdate failed. The backoff above only gated
// the NEXT registry 'update' event, so after a failure nothing retried until some unrelated release
// was published — observed for pgh-sidecar 0.2.68 and favela-gateway 0.2.0, both failing on a tree
// the publish had created seconds earlier and fetchable a minute later. The retry re-runs the whole
// release check, so every gate (dedupe, staged, backoff, a newer release) still applies.
this._installRetryTimers = new Map()
// Overlay subsystem — per-repo file metadata index + cross-peer file
// serving. Initialised lazily during start() because it needs the
// identity (for the swarm topic). Stays null when enableSeed === false
// (overlay shares the seeding/network plane).
this._overlayStore = null // Corestore for overlay-files namespaces
this._overlayManager = null // OverlayManager
this._overlaySwarm = null // Hyperswarm joined on overlay topic
this._overlayTopic = null // hash(identity.publicKey + 'pgh-overlay/v1')
// hyper-overlay v2 stack — sole overlay protocol after #14d step 3.
this._overlaySyncEngine = null // SyncEngine over namespace('overlay-sync-v2')
this._overlayTransferManager = null // TransferManager over the unified FileIndex
this._overlayProtocolV2 = null // MultiRepoOverlayProtocolV2
// Set of hex peer-topic strings we've joined for cross-machine fetch.
// Topics persist across calls so subsequent fetches from the same peer
// reuse the connection (and its already-running protocol channel).
this._overlayPeerTopics = new Set()
this._overlayRoot = opts.overlayStoreRoot ||
path.join(homedir(), '.config', 'pgh', 'overlay-store')
this._started = false
}
async start () {
if (this._started) throw new Error('already started')
fs.mkdirSync(this.root, { recursive: true })
fs.mkdirSync(this.identityPath, { recursive: true })
this._identity = await loadOrCreateIdentity(this.identityPath)
this._treeStore = new TreeStore({ root: this.storeRoot })
this._treeStore.init()
// Release-registry — always opened (single writer, held by sidecar
// for the lifetime of this process). pgh CLI talks to it via UDS.
// Startup storage sweep — clean up the small leaks pre-2026-05-13:
// - tree-store `tmp/<hash>-<ts>/` dirs left by crashes mid-materialize
// - migration backup files (refs.json.backup-…, published-index.json.backup-…)
// created by manual scripts; safe to drop after 14 days.
// Single-pass, cheap. Bounded by the size of those dirs.
this._sweepLeftoverState()
fs.mkdirSync(path.dirname(this.registryStore), { recursive: true })
this._registryStore = new Corestore(this.registryStore)
this._registry = new ReleaseRegistry(this._registryStore)
await this._registry.ready()
// Name-registry — global TOFU app-name registry. Each sidecar
// either creates its own (first boot, no bootstrap key) or joins
// an existing one (bootstrap key from env / opts). pgh CLI looks
// names up via UDS, never opens the corestore directly.
fs.mkdirSync(this.nameRegistryStore, { recursive: true })
this._nameStore = new Corestore(this.nameRegistryStore)
this._nameRegistry = new NameRegistry(this._nameStore, {
bootstrap: this.nameRegistryBootstrap
})
await this._nameRegistry.ready()
// Seeding — folded in from the legacy `pgh seed` command. One
// DeployServer on the hyperident topic serves the tree store;
// one Hyperswarm instance advertises the release-registry so
// remote installers can replicate entries.
// Automatic store GC — prune superseded, unreferenced dependency trees
// BEFORE the DeployServer indexes the store. Without this the tree store
// grows unbounded (every publish/install leaves the prior closure behind),
// and the boot index walks all of it (~78k files / ~10 min). A 14-day
// grace gate keeps a rollback window AND protects freshly-published trees
// that no installed app references yet (a 0-age sweep would delete a
// release before anyone installs it). Non-fatal — a gc error must not
// block boot. Re-run on a daily interval (see _gcTimer below) so a
// long-lived sidecar stays bounded without needing a restart.
try {
const r = await this._storeGc()
if (r) console.log('[pgh-sidecar] store gc on boot: pruned ' + r.removed.length + ' superseded tree(s), kept ' + r.kept.length + ' (keep ' + KEEP_RELEASES + ' releases/app)')
} catch (err) {
console.error('[pgh-sidecar] store gc (non-fatal):', err && err.message)
}
this._gcTimer = setInterval(() => {
this._storeGc().then((r) => {
if (r && r.removed.length) console.log('[pgh-sidecar] periodic store gc: pruned ' + r.removed.length + ' tree(s)')
}, (err) => { console.error('[pgh-sidecar] periodic gc (non-fatal):', err && err.message) })
}, 24 * 60 * 60 * 1000)
if (this._gcTimer.unref) this._gcTimer.unref()
// #66 bootstrap catch-up: tar any installed pgh.bootstrap app whose tarball
// is missing/stale into <store>/bootstrap (covers trees installed before
// this existed, or a deleted artifact). Live version bumps are handled
// inline in the autoupdate path. Best-effort, fire-and-forget.
this._reconcileBootstrap().catch(() => {})
if (this.enableSeed) {
const topic = this._identity.publicKey
// Persistent fileIndex store. hyper-deploy's DeployServer defaults to a
// fresh `os.tmpdir()/hyper-deploy-srv-<Date.now()>` corestore on every
// open (index.js:113), which is fine for a small built-release dir but
// pathological here: we serve the ENTIRE content-addressed tree store
// (~34k files / 40+ GB). An ephemeral index forces a full re-hash of the
// whole store on every boot (the multi-minute cold boot, `0 cache hits`)
// AND means published trees are only servable for the lifetime of the
// process that indexed them — the root cause of trees intermittently
// "not found on any peer" after a restart. A persistent store is a pure
// optimization: _scan() still rebuilds the manifest from disk each boot
// (so the served set stays disk-accurate), but reuses cached content
// hashes when size+mtime match — safe because the store's tree dirs are
// content-addressed and immutable. Trees also stay durably registered
// across restarts. We own this store (DeployServer only owns the store
// when it creates the tmp one), so we close it in close().
const deployIndexDir = path.join(this.storeRoot, 'deploy-index')
fs.mkdirSync(deployIndexDir, { recursive: true })
this._deployIndexStore = new Corestore(deployIndexDir)
this._deployServer = new DeployServer(this._treeStore.dependenciesDir, { topic, corestore: this._deployIndexStore })
// Attach error handler BEFORE ready() — peers disconnecting mid-stream
// emit 'error' with ECONNRESET; without a listener the default
// EventEmitter behaviour crashes the process.
this._deployServer.on('error', (err) => {
// [diag-2026-05-04] include stack trace + message identifier when
// available to find which message-type trips secret-stream's 16 MB
// MAX_ATOMIC_WRITE. Drop after diagnosis.
const tag = err.message && err.message.includes('atomic write') ? 'TOOLARGE' : (err.code || 'ERR')
const stack = err.stack ? err.stack.split('\n').slice(0, 4).map(s => s.trim()).join(' | ') : ''
console.error('[pgh-sidecar] deploy-server error (non-fatal) [' + tag + ']:', err.message, stack ? '@' + stack : '')
})
// Surface the initial scan's progress so a slow startup is visible.
// Without this, the sidecar appears hung for the multi-minute walk
// of a large tree store.
this._deployServer.on('index-progress', (i, total) => {
if (i % 5000 === 0 || i === total) {
console.log('[pgh-sidecar] indexing: ' + i + '/' + total + ' files')
}
})
this._deployServer.on('scan-cache-stats', (s) => {
console.log('[pgh-sidecar] scan-cache: ' + s.hits + ' hits, ' +
s.misses + ' misses, ' + s.hashedBytes + ' bytes hashed / ' +
s.totalFiles + ' files')
})
this._deployServer.on('indexed', (files, totalSize) => {
console.log('[pgh-sidecar] initial scan complete: ' + files +
' files, ' + Math.round(totalSize / 1024 / 1024) + ' MB')
})
await this._deployServer.ready()
this._registrySwarm = new Hyperswarm()
this._registrySwarm.on('error', (err) => {
console.error('[pgh-sidecar] registry-swarm error (non-fatal):', err.code || err.message)
})
this._registrySwarm.on('connection', (conn) => {
conn.on('error', () => {}) // absorb ECONNRESET from disconnecting peers
this._registryStore.replicate(conn)
// Multi-source content distribution: ALSO serve tree content by
// hash on the SHARED registry topic, muxed onto this same
// connection (keet many-protocols-one-connection pattern). Any
// peer that has the bytes can serve them — content is
// blake2b-addressed, no autobase writership needed. The DeployServer
// also seeds on its identity topic (above) for backward-compat with
// already-published releases whose serverKey = identity.publicKey;
// this adds the torrent-style shared-topic path so installers pull
// from EVERY sidecar seeding the registry, not just the publisher.
// replicate(conn) claims the Protomux first; serveOnConnection then
// attaches the overlay content channel onto Protomux.from(conn).
// Reuses the DeployServer's live FileIndex/manifest so trees added
// by the incremental rescan in publish-release are served here too.
try { this._deployServer.serveOnConnection(conn) } catch (err) {
console.error('[pgh-sidecar] registry-topic content serve (non-fatal):', err && err.message)
}
})
this._registrySwarm.join(this._registry._base.discoveryKey, { server: true, client: true })
await this._registrySwarm.flush()
// Name-registry advertiser. Remote pgh CLIs replicate this over
// Hyperswarm to resolve bare names. Keeps lock-ownership clean:
// sidecar is the only process that writes to the corestore.
this._nameRegistrySwarm = new Hyperswarm()
this._nameRegistrySwarm.on('error', (err) => {
console.error('[pgh-sidecar] name-registry-swarm error (non-fatal):', err.code || err.message)
})
this._nameRegistrySwarm.on('connection', (conn) => {
conn.on('error', () => {})
this._nameStore.replicate(conn)
})
this._nameRegistrySwarm.join(this._nameRegistry.discoveryKey, { server: true, client: true })
await this._nameRegistrySwarm.flush()
// Read-only replicas of OTHER publishers' name registries, so a
// standalone peer (own empty NameRegistry) can resolve bare app
// names claimed elsewhere. Mirrors the release-replica scheme below.
const ownNameKey = this._nameRegistry && this._nameRegistry.keyHex
for (const keyHex of this.nameRegistrySubscriptions) {
if (keyHex.length !== 64) continue
if (keyHex === ownNameKey) continue // own registry already queried first
const replica = await this._openNameReplica(keyHex)
if (replica) this._nameReplicas.push(replica)
}
// Warm read-only replicas of other publishers' release registries.
// These stay live in the background via autobase update events —
// CLI queries them via UDS instead of opening one-shot replicas.
//
// 2026-05-13: skip subscriptions matching THIS sidecar's own
// registry key. `addSubscription` (called from installViaRegistry
// on every install) used to add the publisher's key blindly — so
// a user installing their own published app added themselves as a
// subscriber, and the sidecar then opened a redundant 100+ MB
// replica of its own writer-side corestore. _findReleaseRegistry
// already checks `this._registry` first, so functionally nothing
// breaks when we skip the self-replica.
const ownKey = this._registry && this._registry.keyHex
for (const keyHex of this.releaseRegistrySubscriptions) {
if (keyHex.length !== 64) continue
if (keyHex === ownKey) {
console.log('[pgh-sidecar] skipping replica for own registry ' + keyHex.slice(0, 12) + '…')
continue
}
const replica = await this._openReleaseReplica(keyHex)
if (replica) this._releaseReplicas.push(replica)
}
// Overlay subsystem — per-repo file metadata index + Protomux
// channel `pgh-overlay/v1` over Hyperswarm for cross-peer file
// requests. Topic is identity-derived but distinct from hyper-deploy's
// topic so the two protocols don't share connections (different
// protomux channel naming conflicts otherwise).
fs.mkdirSync(this._overlayRoot, { recursive: true })
this._overlayStore = new Corestore(this._overlayRoot)
this._overlayManager = new OverlayManager(this._overlayStore)
this._overlayTopic = crypto.hash(Buffer.concat([
this._identity.publicKey,
Buffer.from('pgh-overlay/v1')
]))
// hyper-overlay v2 stack — runs alongside the legacy pgh-overlay/v1
// protocol on the same swarm/mux. Different protocol channel names
// ('hyper-overlay/v2' vs 'pgh-overlay/v1') so they coexist without
// conflicts. UDS handler still uses v1 until #14d step 3.
const overlayFileIndex = await this._overlayManager.getFileIndex()
this._overlaySyncEngine = new SyncEngine(
overlayFileIndex,
this._overlayStore.namespace('overlay-sync-v2')
)
await this._overlaySyncEngine.ready()
this._overlayTransferManager = new TransferManager(overlayFileIndex)
// #66 overlay-mirror: content-addressed spool for repos published
// elsewhere. A forge backend has no working tree, so it fetches a foreign
// repo's large files by oid into <overlay-store>/mirror-spool/<oid> and
// re-serves them via the v2 <syncBaseDir>/<contentHash> resolution
// (hyper-overlay 0.2.8). Passing it as a syncBaseDir is what enables that
// serve path for mirrored blobs.
this._overlayMirrorSpool = path.join(this._overlayRoot, 'mirror-spool')
fs.mkdirSync(this._overlayMirrorSpool, { recursive: true })
this._overlayProtocolV2 = new MultiRepoOverlayProtocolV2(
this._overlaySyncEngine,
this._overlayTransferManager,
this._overlayManager,
{ filePaths: this._overlayManager.filePaths, syncBaseDirs: [this._overlayMirrorSpool] }
)
this._overlayManager.attachSyncEngine(this._overlaySyncEngine)
// Warm caches BEFORE joining the swarm so first-connection resolves
// succeed for already-registered repos and files.
await this._overlayManager.warmRepoCache()
await this._overlayManager.populateFilePaths()
this._overlaySwarm = new Hyperswarm()
this._overlaySwarm.on('error', (err) => {
console.error('[pgh-sidecar] overlay-swarm error (non-fatal):', err.code || err.message)
})
this._overlaySwarm.on('connection', (conn) => {
conn.on('error', () => {})
this._attachOverlayConn(conn)
})
this._overlaySwarm.join(this._overlayTopic, { server: true, client: true })
await this._overlaySwarm.flush()
// #66 overlay-mirror: replicate foreign-published repos' large files into
// the mirror-spool so this backend becomes a SECOND source (the overlay
// analogue of pear-git-v3's RepoPuller). Boot pass + interval; best-effort,
// fire-and-forget so a mirror hiccup never blocks boot.
this._runOverlayMirror().catch(() => {})
this._overlayMirrorTimer = setInterval(() => { this._runOverlayMirror().catch(() => {}) }, 5 * 60 * 1000)
if (this._overlayMirrorTimer && this._overlayMirrorTimer.unref) this._overlayMirrorTimer.unref()
}
// Seed the app registry from disk, pruning entries whose processes
// are gone so stale registrations don't poison `app-lookup`.
await this._loadAppsFromDisk()
this._svc = new Service({
name: 'pgh-sidecar',
uds: this.socketPath
})
this._registerHandlers()
// Hook autobase update events → push `release-ready` RPC to each
// registered app whose pgh.name matches a new release entry. This
// replaces the 30min poll loop in pgh-runtime as the primary OTA
// trigger (poll still runs as a safety net).
this._watchForReleases()
await this._svc.listen()
this._started = true
// One-shot startup compaction across all corestores. Reclaims rocksdb
// tombstones that built up since the last boot. Fire-and-forget;
// logged per-store. Validated by Keet 4.x which exposes the same
// primitive as `core.compact` RPC (id 197) called periodically.
this._compactAll().catch((err) => {
console.warn('[pgh-sidecar] startup compact failed (non-fatal):', err.message)
})
// Daily timer keeps slack from re-accumulating between restarts.
// Refreshed when sidecar runs continuously; cleared on stop().
const COMPACT_INTERVAL_MS = 24 * 60 * 60 * 1000
this._compactTimer = setInterval(() => {
this._compactAll().catch((err) => {
console.warn('[pgh-sidecar] periodic compact failed (non-fatal):', err.message)
})
}, COMPACT_INTERVAL_MS)
// Don't keep the event loop alive just for this timer.
if (this._compactTimer && this._compactTimer.unref) this._compactTimer.unref()
return {
socketPath: this.socketPath,
identityPath: this.identityPath,
storeRoot: this.storeRoot,
registryStore: this.registryStore,
registryKey: this._registry._base.key ? b4a.toString(this._registry._base.key, 'hex') : null,
nameRegistryStore: this.nameRegistryStore,
nameRegistryKey: this._nameRegistry.keyHex,
nameRegistryWritable: this._nameRegistry.writable,
seedTopic: this.enableSeed ? b4a.toString(this._identity.publicKey, 'hex') : null
}
}
/**
* Attach the overlay protocol to one connection, and release it when that
* connection dies.
*
* OverlayProtocol.attach() registers a mux.pair() callback and records the
* mux in `_paired`; `_peers` is pruned by the channel's own onclose, but
* `_paired` is pruned ONLY by detach(). We hold ONE _overlayProtocolV2 for
* the whole daemon and attach it to every inbound overlay connection, so
* without the detach every connection this always-on sidecar ever accepted
* stays pinned — along with its mux and that mux's stream.
*
* Same defect and same shape as hyper-deploy 0.4.11 (DeployServer._onConnection)
* and 0.4.13 (DeployServer.serveOnConnection); this is the third attach site,
* the one the sidecar owns directly. detach() is idempotent, so a close after
* an error path is harmless.
*/
_attachOverlayConn (conn) {
if (!this._overlayProtocolV2) return false
let mux
try {
mux = Protomux.from(conn)
this._overlayProtocolV2.attach(mux)
} catch (err) {
console.error('[pgh-sidecar] overlay-v2 attach failed:', err.message)
return false
}
conn.once('close', () => {
try {
if (this._overlayProtocolV2) this._overlayProtocolV2.detach(mux)
} catch {}
})
return true
}
async stop () {
if (!this._started) return
this._clearInstallRetries()
if (this._compactTimer) {
clearInterval(this._compactTimer)
this._compactTimer = null
}
if (this._gcTimer) {
clearInterval(this._gcTimer)
this._gcTimer = null
}
if (this._svc) {
try { await this._svc.close() } catch {}
this._svc = null
}
if (this._registrySwarm) {
try { await this._registrySwarm.destroy() } catch {}
this._registrySwarm = null
}
if (this._nameRegistrySwarm) {
try { await this._nameRegistrySwarm.destroy() } catch {}
this._nameRegistrySwarm = null
}
for (const r of this._releaseReplicas) {
try { await r.swarm.destroy() } catch {}
try { await r.registry.close() } catch {}
try { await r.store.close() } catch {}
}
this._releaseReplicas = []
for (const r of this._nameReplicas) {
try { await r.swarm.destroy() } catch {}
try { await r.registry.close() } catch {}
try { await r.store.close() } catch {}
}
this._nameReplicas = []
if (this._deployServer) {
try { await this._deployServer.close() } catch {}
this._deployServer = null
}
// We own the DeployServer's fileIndex store (passed via opts.corestore, so
// DeployServer's _ownStore is false and it won't close it). Close it here.
if (this._deployIndexStore) {
try { await this._deployIndexStore.close() } catch {}
this._deployIndexStore = null
}
try { if (this._overlayMirrorTimer) clearInterval(this._overlayMirrorTimer) } catch {}
this._overlayMirrorTimer = null
if (this._overlaySwarm) {
try { await this._overlaySwarm.destroy() } catch {}
this._overlaySwarm = null
}
if (this._overlayProtocolV2) {
try { this._overlayProtocolV2.destroy() } catch {}
this._overlayProtocolV2 = null
}
if (this._overlaySyncEngine) {
try { await this._overlaySyncEngine.close() } catch {}
this._overlaySyncEngine = null
}
this._overlayTransferManager = null
if (this._overlayManager) {
try { await this._overlayManager.close() } catch {}
this._overlayManager = null
}
if (this._overlayStore) {
try { await this._overlayStore.close() } catch {}
this._overlayStore = null
}
if (this._registry) {
try { await this._registry.close() } catch {}
this._registry = null
}
if (this._registryStore) {
try { await this._registryStore.close() } catch {}
this._registryStore = null
}
if (this._nameRegistry) {
try { await this._nameRegistry.close() } catch {}
this._nameRegistry = null
}
if (this._nameStore) {
try { await this._nameStore.close() } catch {}
this._nameStore = null
}
this._started = false
}
get identity () { return this._identity }
get treeStore () { return this._treeStore }
get registry () { return this._registry }
get nameRegistry () { return this._nameRegistry }
get socket () { return this.socketPath }
/**
* Seed `this._apps` from disk on startup. Stale entries (pid gone or
* udsPath absent) are pruned on the way in so `app-lookup` never
* returns a dead registration to a client.
*/
async _loadAppsFromDisk () {
let raw
try { raw = fs.readFileSync(this._appsFile, 'utf8') } catch { return }
let parsed
try { parsed = JSON.parse(raw) } catch { return }
if (!Array.isArray(parsed)) return
let pruned = 0
for (const entry of parsed) {
if (!entry || !entry.name || !entry.udsPath) continue
const alive = this._isAppLive(entry)
if (alive) {
this._apps.set(entry.name, entry)
} else {
pruned++
}
}
if (pruned > 0) this._saveAppsToDisk()
}
_isAppLive (entry) {
// pid alive?
if (entry.pid) {
try { process.kill(entry.pid, 0) } catch { return false }
}
// Abstract-namespace sockets ('@'-prefixed) have no filesystem entry, so
// fs.existsSync always fails — liveness is the pid check above (an app on
// an unrooted-Android/celltron forge binds an abstract udsPath).
if (typeof entry.udsPath === 'string' && entry.udsPath[0] === '@') return true
// udsPath exists?
try { return fs.existsSync(entry.udsPath) } catch { return false }
}
_saveAppsToDisk () {
try {
const arr = Array.from(this._apps.values())
fs.mkdirSync(path.dirname(this._appsFile), { recursive: true })
fs.writeFileSync(this._appsFile, JSON.stringify(arr, null, 2))
} catch (err) {
console.error('[pgh-sidecar] could not persist app registry:', err.message)
}
}
/**
* Subscribe to autobase 'update' events on every live registry (own +
* replicas). On each update, check for every registered app whether
* the latest entry for its name is newer than what we last notified,
* and push a `release-ready` RPC so the app triggers an immediate OTA
* check. Replaces polling for responsiveness; pgh-runtime's periodic
* poll stays as a safety net in case an event gets missed.
*/
_watchForReleases () {
const fire = () => {
this._notifyAppsOfNewReleases().catch((err) => {
console.error('[pgh-sidecar] release-notify failed:', err.message)
})
}
// The same check an armed install retry runs (pear-git #47).
this._fireReleaseCheck = fire
try { this._registry._base.on('update', fire) } catch {}
for (const r of this._releaseReplicas) {
try { r.registry._base.on('update', fire) } catch {}
}
// Fire once on startup so apps that booted before the first autobase
// update see anything we already have.
queueMicrotask(fire)
}
/**
* For every app we know about (registered OR installed-but-quiet),
* check registries for a newer release. If found, INSTALL it (full
* materialize + dep resolve + symlink swap via pgh's installViaRegistry
* — not just a tree fetch), then signal the running daemon (if any)
* to exit so its supervisor respawns against the new install. CLI
* apps (no UDS) get the swap; next invocation picks it up.
*/
async _notifyAppsOfNewReleases () {
const candidates = await this._listUpdateCandidates()
if (candidates.length === 0) return
const registries = [this._registry, ...this._releaseReplicas.map(r => r.registry)]
for (const cand of candidates) {
const { name, currentVersion, currentTreeHash, stagedVersion, registered, udsPath } = cand
let best = null
for (const reg of registries) {
try {
const e = await reg.latest(name)
if (!e) continue
if (!best || (e.publishedAt && (!best.publishedAt || e.publishedAt > best.publishedAt))) {
best = e
}
} catch {}
}
if (!best) continue
const token = best.version + '@' + best.treeHash
if (this._lastNotifiedVersion.get(name) === token) continue
if (currentTreeHash && currentTreeHash === best.treeHash) {
this._lastNotifiedVersion.set(name, token)
continue
}
// Same-version dedupe — multiple replicas can advertise the same
// app version with different treeHashes (each `pgh publish` of
// unchanged source still produces a fresh tree). Without this,
// autoupdate oscillates: round 1 picks tree A, installs, round 2
// picks tree B, installs, round 3 picks A again, forever. Holding
// on the version we already have stops the loop. New version bumps
// still trigger install; only same-version-different-tree is skipped.
if (currentVersion && best.version && currentVersion === best.version) {
this._lastNotifiedVersion.set(name, token)
continue
}
// Already STAGED: the app was running when this version was installed, so
// pgh left it as <name>.next to apply at the app's next launch (pgh
// materializers/atomic.js). Re-installing would only rebuild the same tree.
if (stagedVersion && best.version && stagedVersion === best.version) {
this._lastNotifiedVersion.set(name, token)
continue
}
// Auto-install new release (skip if user explicitly disabled).
const autoupdate = this._isAutoupdateEnabled(name)
if (!autoupdate) {
console.log('[pgh-sidecar] new release for ' + name + ' (' + best.version + ') — autoupdate disabled, skipping')
this._lastNotifiedVersion.set(name, token)
continue
}
// Persistent-failure backoff. If this exact release (name, token) has been
// failing, don't hammer it on every autobase 'update' tick — wait out the
// exponential backoff. A new token (version/tree bump) falls through with a
// fresh counter. See _installFailures in the constructor.
if (this._isInInstallBackoff(name, token)) continue
// Per-app install mutex — see _installInFlight comment in
// constructor. Skip if a previous tick is still installing.
if (this._installInFlight.has(name)) {
console.log('[pgh-sidecar] autoupdate: ' + name + ' install already in flight — skipping concurrent tick')
continue
}
this._installInFlight.add(name)
try {
console.log('[pgh-sidecar] autoupdate: installing ' + name + '@' + best.version + ' (tree ' + best.treeHash.slice(0, 12) + ')')
await this._installRelease(name, best)
console.log('[pgh-sidecar] autoupdate: ' + name + ' installed; notifying daemon if running')
// #66: the freshly-installed app may be a bootstrap app (or its tree
// changed) — refresh <store>/bootstrap so /bootstrap serves the new
// tarball. Idempotent + treeHash-cached; best-effort.
this._reconcileBootstrap().catch(() => {})
if (registered && udsPath) {
this._pushReleaseReady({ name, udsPath }, best).catch(() => {})
}
this._lastNotifiedVersion.set(name, token)
this._installFailures.delete(name)
this._cancelInstallRetry(name)
} catch (err) {
// Record the failure and back off. Don't poison _lastNotifiedVersion so
// a later fire (once the backoff elapses, or a new version appears) gets
// another shot — but no longer on EVERY tick (the leak that melted the
// sidecar). Backoff resets when the release token changes.
const prev = this._installFailures.get(name)
const count = (prev && prev.token === token ? prev.count : 0) + 1
const backoffMs = this._installBackoffMs(count)
this._installFailures.set(name, { token, count, nextRetryAt: Date.now() + backoffMs })
console.error('[pgh-sidecar] autoupdate failed for ' + name + ': ' + err.message +
' (attempt ' + count + '; retrying in ' + Math.round(backoffMs / 1000) + 's)')
// ⛔ And actually retry: the backoff alone only gates the next event (pear-git #47).
this._armInstallRetry(name, backoffMs)
} finally {
this._installInFlight.delete(name)
}
}
}
// pear-git #47: after a failed autoupdate, re-run the release check once the backoff has elapsed. One
// timer per app (a newer failure replaces it); cleared on success and on stop; unref'd so it never holds
// the process open. The small margin lands the check just past nextRetryAt, so the backoff gate admits it.
// Lazily created, so an instance built without the constructor (the stub tests) still works.
_retryTimers () {
if (!this._installRetryTimers) this._installRetryTimers = new Map()
return this._installRetryTimers
}
_armInstallRetry (name, backoffMs) {
this._cancelInstallRetry(name)
const timer = setTimeout(() => {
this._retryTimers().delete(name)
const fire = this._fireReleaseCheck
if (fire) fire()
}, backoffMs + 50)
if (timer.unref) timer.unref()
this._retryTimers().set(name, timer)
}
_cancelInstallRetry (name) {
const timers = this._retryTimers()
const t = timers.get(name)
if (t) { clearTimeout(t); timers.delete(name) }
}
_clearInstallRetries () {
const timers = this._retryTimers()
for (const t of timers.values()) clearTimeout(t)
timers.clear()
}
// Exponential backoff for a persistently-failing autoupdate. Attempt 1 waits
// 30s, then doubles (1m, 2m, 4m, …) capped at 1h. Bounds the retry rate of an
// un-installable release so a stuck app can't drive an unbounded fd/socket
// leak (see _installFailures in the constructor).
_installBackoffMs (count) {
const base = 30000
const cappedExp = Math.min(Math.max(count, 1) - 1, 7) // cap so no overflow
return Math.min(base * Math.pow(2, cappedExp), 3600000)
}
// True while (name, token) is inside its post-failure backoff window. A token
// change (new version/tree) is NOT in backoff — a fresh release always gets an
// immediate attempt. Extracted for testability.
_isInInstallBackoff (name, token) {
const fail = this._installFailures.get(name)
return !!(fail && fail.token === token && Date.now() < fail.nextRetryAt)
}
/**
* Union of running-registered apps (sidecar's in-memory map) plus
* installed-on-disk apps (have a .pgh-install.json). CLIs like pgh
* aren't registered with the sidecar (no daemon to register), but
* they still need autoupdate.
*/
async _listUpdateCandidates () {
const seen = new Map()
let readInstallManifest = null
let listAppNames = null
try {
;({ listAppNames, readInstallManifest } = await import('pgh/app-state'))
} catch {}
for (const [name, app] of this._apps) {
// Registered apps may register without version/treeHash (older
// launchers, missing manifest). Fall back to disk so the
// version-dedupe gate in _notifyAppsOfNewReleases still works
// even when the in-memory entry is partial.
let version = app.version || null
let treeHash = app.treeHash || null
if ((!version || !treeHash) && readInstallManifest) {
try {
const m = readInstallManifest(name) || {}
if (!version) version = m.version || null
if (!treeHash) treeHash = m.treeHash || null
} catch {}
}
seen.set(name, {
name,
currentVersion: version,
currentTreeHash: treeHash,
stagedVersion: this._stagedVersion(name, readInstallManifest),
registered: true,
udsPath: app.udsPath
})
}
if (listAppNames && readInstallManifest) {
try {
for (const n of listAppNames()) {
if (seen.has(n)) continue
const m = readInstallManifest(n) || {}
seen.set(n, {
name: n,
currentVersion: m.version || null,
currentTreeHash: m.treeHash || null,
stagedVersion: this._stagedVersion(n, readInstallManifest),
registered: false,
udsPath: null
})
}
} catch {
// pgh/app-state may not be available in older pgh copies — skip
// the installed-on-disk set and fall back to registered-only.
}
}
return [...seen.values()]
}
// Version of an update pgh STAGED because the app was running (<name>.next,
// which carries its own install manifest), or null.
_stagedVersion (name, readInstallManifest) {
if (!readInstallManifest) return null
try { return (readInstallManifest(name + '.next') || {}).version || null } catch { return null }
}
// Admin setting, apps.json `bootstrapServeAllArches`. Unlike autoupdate this
// defaults to FALSE: holding another arch's ~150MB tarball only pays off on a
// machine that actually answers cold-start HTTPS /bootstrap fetches, i.e. a
// forge backend. Set with `pgh app bootstrap-serve <name> on`.
_isBootstrapServeEnabled (name) {
try {
const cfgPath = path.join(os.homedir(), '.config', 'pgh', 'apps.json')
if (!fs.existsSync(cfgPath)) return false
const cfg = JSON.parse(fs.readFileSync(cfgPath, 'utf8'))
return !!(cfg && cfg[name] && cfg[name].bootstrapServeAllArches === true)
} catch { return false }
}
_isAutoupdateEnabled (name) {
try {
const cfgPath = path.join(os.homedir(), '.config', 'pgh', 'apps.json')
if (!fs.existsSync(cfgPath)) return true
const cfg = JSON.parse(fs.readFileSync(cfgPath, 'utf8'))
if (cfg && cfg[name] && cfg[name].autoupdate === false) return false
return true
} catch { return true }
}
/**
* Do the full install — fetch trees + dep closure + materialize +
* swap symlink + write launcher. Imports from pgh's installViaRegistry
* rather than duplicating. The app's install path ends up pointing at
* the new content-addressable tree.
*/
async _installRelease (name, entry) {
const { run: appRun } = await import('pgh/commands/app')
const url = 'pgh://' + (entry.upgradePublisher || this._inferPublisher(entry) || '') + '/' + name
// Fall back to a direct registry-key URL via our own registry
// key — works because installViaRegistry accepts pgh://<publisherKey>/<name>.
const registryKeyHex = entry.__registryKey || this._findSourceRegistryKey(entry) || null
const args = registryKeyHex
? ['install', 'pgh://' + registryKeyHex + '/' + name, '--name', name]
: ['install', url, '--name', name]
await appRun(null, args, { format: 'json' })
}
// #66: keep <store>/bootstrap populated with the curl-installable artifacts —
// tarballs of the local pgh.bootstrap app trees (the app IS the self-contained
// bootstrap tree) + an install.sh symlink. The forge (pear-git-v3) serves that
// dir at /bootstrap. Event-driven: called after each autoupdate install plus
// once on boot (catch-up for trees installed before this existed). Idempotent +
// treeHash-cached, so non-bootstrap installs are cheap. Best-effort — a
// bootstrap reconcile error must never break install or boot.
async _reconcileBootstrap () {
try {
const { reconcileBootstrapArtifacts } = await import('pgh/bootstrap-artifacts')
await reconcileBootstrapArtifacts(this._treeStore, {
log: (m) => console.log('[pgh-sidecar] ' + m),
// Admin opt-in — only forge backends serve arches they can't build.
// `pgh app bootstrap-serve pgh-sidecar on`
serveAllArches: this._isBootstrapServeEnabled('pgh-sidecar')
})
} catch (err) {
console.error('[pgh-sidecar] bootstrap reconcile (non-fatal):', err && err.message)
}
}
// #66 overlay-mirror: for every bare repo published ELSEWHERE (its
// .pear-git/overlay.json peerServerKey ≠ our identity), fetch its overlay
// large files by oid into the content-addressed mirror-spool so this backend
// re-serves them. Best-effort; never throws into boot/interval.
// One pass at a time: the 5-min interval used to start a new pass while the
// last still ran, and on stortron they piled up until the daemon wedged
// (2026-10-11). A call during a pass joins it.
_runOverlayMirror () {
if (!this._overlayMirrorOnce) this._overlayMirrorOnce = singleFlight(() => this._overlayMirrorPass())
return this._overlayMirrorOnce()
}
async _overlayMirrorPass () {
if (!this._overlayProtocolV2 || !this._overlaySwarm || !this._overlayMirrorSpool) return
// ASYNC spawn (NOT spawnSync): this runs in the always-on bare daemon, and
// listing pointers shells out to git once per small blob (hundreds for a
// game repo). A synchronous spawn would block the event loop the whole time
// (same freeze class as the bootstrap-tar fix). bare-subprocess provides
// async spawn; #node:child_process is mapped to it.
const { spawn } = await import('#node:child_process')
const reposDir = process.env.PEAR_GIT_REPOS || path.join(homedir(), 'git-repos')
if (!fs.existsSync(reposDir)) return
const selfHex = this._identity ? b4a.toString(this._identity.publicKey, 'hex') : null
const git = (repo, args) => {
const gd = path.join(reposDir, repo.endsWith('.git') ? repo : repo + '.git')
return spawnCaptureNoLeak(spawn, 'git', ['--git-dir', gd, ...args])
}
const readOverlayJson = async (repo) => {
try { return JSON.parse(await git(repo, ['show', 'HEAD:.pear-git/overlay.json'])) } catch { return null }
}
// Two spawns per repo: ls-tree + ONE `cat-file --batch` fed the small-blob
// oids on stdin. It was one `cat-file -p` per blob — ~4,000 forks of a
// multi-GB daemon per pass on stortron (2026-10-11), see listPointersFromGit.
const listPointers = async (repo) => {
const gd = path.join(reposDir, repo.endsWith('.git') ? repo : repo + '.git')
try {
return await listPointersFromGit({
lsTree: () => git(repo, ['ls-tree', '-r', '-l', 'HEAD']),
catFileBatch: (oids) => spawnCaptureNoLeak(spawn, 'git', ['--git-dir', gd, 'cat-file', '--batch'],
oids.join('\n') + '\n', { buffer: true })
})
} catch { return [] }
}
const repos = fs.readdirSync(reposDir).filter(d => d.endsWith('.git')).map(d => d.replace(/\.git$/, ''))
const targets = await discoverMirrorTargets({ repos, selfPubkeyHex: selfHex, readOverlayJson })
if (targets.length === 0) return
await runOverlayMirror({
targets,
mirrorSpool: this._overlayMirrorSpool,
listPointers,
fetchOid: (peerServerKey, oid, dest) => this._overlayFetchOid(peerServerKey, oid, dest),
deps: { fs, path },
log: (m) => console.log('[pgh-sidecar] ' + m)
})
}
// Fetch one overlay blob by content hash from a publisher into destPath
// (content-addressed <mirrorSpool>/<oid>). Joins the publisher's overlay topic
// on the existing swarm (same derivation as overlay-locate), waits for a peer,
// pulls via the v2 protocol, and verifies the whole-file hash. Returns boolean.
async _overlayFetchOid (peerServerKey, oid, destPath) {
const peerKey = Buffer.from(peerServerKey, 'hex')
const peerTopic = crypto.hash(Buffer.concat([peerKey, Buffer.from('pgh-overlay/v1')]))
const peerTopicHex = peerTopic.toString('hex')
if (!this._overlayPeerTopics.has(peerTopicHex)) {
const discovery = this._overlaySwarm.join(peerTopic, { server: false, client: true })
try { await discovery.flushed() } catch {}
this._overlayPeerTopics.add(peerTopicHex)
}
for (let waited = 0; this._overlayProtocolV2._peers.size === 0 && waited < 5000; waited += 100) {
await new Promise(r => setTimeout(r, 100))
}
const peers = [...this._overlayProtocolV2._peers.values()]
if (peers.length === 0) return false
try {
await this._overlayProtocolV2.fetchContent(oid, peers, { destPath, timeout: 60000 })
} catch { return false }
try {
const data = fs.readFileSync(destPath)
if (crypto.data(data).toString('hex') !== oid) { try { fs.rmSync(destPath, { force: true }) } catch {}; return false }
} catch { return false }
return true
}
_findSourceRegistryKey (entry) {
// entry.upgrade is a pgh://<key>/name URL — extract the key.
if (entry && typeof entry.upgrade === 'string' && entry.upgrade.startsWith('pgh://')) {
const rest = entry.upgrade.slice('pgh://'.length)
const slash = rest.indexOf('/')
if (slash === 64) return rest.slice(0, slash)
}
// Otherwise check our own registries.
if (this._registry && this._registry._base && this._registry._base.key) {
return b4a.toString(this._registry._base.key, 'hex')
}
for (const r of this._releaseReplicas) {
return r.keyHex
}
return null
}
_inferPublisher (entry) {
if (entry && typeof entry.upgrade === 'string' && entry.upgrade.startsWith('pgh://')) {
return entry.upgrade.slice('pgh://'.length).split('/')[0]
}
return null
}
async _pushReleaseReady (app, entry) {
const { Client } = await import('hyper-svc')
let c = null
try {
c = await Client.connectUds(app.udsPath)
await c.call('release-ready', {
name: entry.name,
version: entry.version,
treeHash: entry.treeHash,
serverKey: entry.serverKey,
publishedAt: entry.publishedAt
})
} catch (err) {
// App not reachable — that's fine. Polling will pick it up later
// if the app comes back.
} finally {
try { if (c) await c.close() } catch {}
}
}
/**
* Startup cleanup of small storage leaks accumulated by older code
* paths. Cheap, single-pass, runs once per sidecar boot.
*
* 1. tree-store `tmp/<...>` orphans: `_materializeInto` writes to a
* `tmp/<hash>-<Date.now()>/` dir then renames to the final tree
* path. A crash mid-write leaves the tmp dir behind. Drop any tmp
* entry older than 1 hour.
* 2. Migration backup files: `refs.json.backup-…` and
* `published-index.json.backup-…` files written by manual scripts.
* Drop any older than 14 days (preserves a window for operator
* recovery but doesn't accumulate forever).
*/
/**
* Tree-store gc with release retention. Runs HERE, not in the CLI: the
* release list carries every entry's full dep closure, far past hyper-svc's
* 16 MiB frame, so only the process holding the registry can read it whole.
* Own registry + warm replicas: a replica app's newest releases are kept too.
*/
async _storeGc ({ dryRun = false, keepReleases = KEEP_RELEASES } = {}) {
const releases = await this._registry.list()
for (const r of this._releaseReplicas) {
try { releases.push(...(await r.registry.list())) } catch {}
}
return this._treeStore.gc({ dryRun, maxAgeDays: 14, keepReleases, releases })
}
_sweepLeftoverState () {
const ONE_HOUR = 60 * 60 * 1000
const FOURTEEN_DAYS = 14 * 24 * ONE_HOUR
const now = Date.now()
// tree-store tmp/ orphans
try {
const tmpDir = path.join(this._treeStore.root || this._treeStore._root || '', 'tmp')
if (tmpDir && fs.existsSync(tmpDir)) {
for (const name of fs.readdirSync(tmpDir)) {
const p = path.join(tmpDir, name)
let st
try { st = fs.statSync(p) } catch { continue }
if ((now - st.mtimeMs) > ONE_HOUR) {
try {
fs.rmSync(p, { recursive: true, force: true })
console.log('[pgh-sidecar] swept tmp orphan: ' + name)
} catch {}
}
}
}
} catch (err) {
console.warn('[pgh-sidecar] tmp sweep failed (non-fatal):', err.message)
}
// Migration backup files in ~/.config/pgh/ + tree-store root
const backupRoots = [
path.dirname(this.registryStore), // typically ~/.config/pgh/
this._treeStore.root || this._treeStore._root || null
].filter(Boolean)
for (const root of backupRoots) {
try {
if (!fs.existsSync(root)) continue
for (const name of fs.readdirSync(root)) {
if (!/\.backup-/.test(name)) continue
const p = path.join(root, name)
let st
try { st = fs.statSync(p) } catch { continue }
if ((now - st.mtimeMs) > FOURTEEN_DAYS) {
try {
fs.rmSync(p, { recursive: true, force: true })
console.log('[pgh-sidecar] swept aged backup: ' + name + ' (' +
((st.size / 1024 / 1024).toFixed(1)) + ' MB)')
} catch {}
}
}
} catch (err) {
console.warn('[pgh-sidecar] backup sweep failed (non-fatal):', err.message)
}
}
// S5 — abandoned swarm-fallback corestores at
// ~/.config/pgh/registries/<key16>/. Created by installViaRegistry's
// fallback path (when sidecar's resolve-release isn't reachable).
// Once the sidecar's own persistent replica at
// ~/.config/pgh/release-registries/<key16>/ is up, the install-time
// corestore is redundant. Drop install-time entries that the sidecar
// is now mirroring (or that match the sidecar's own registry key).
try {
const transientRoot = path.join(path.dirname(this.registryStore), 'registries')
if (fs.existsSync(transientRoot)) {
const ownKey16 = (this._registry && this._registry.keyHex) ?
this._registry.keyHex.slice(0, 16) : null
const replicaRoot = this.releaseRegistryReplicasDir
for (const name of fs.readdirSync(transientRoot)) {
const dir = path.join(transientRoot, name)
// Two safe-to-delete cases:
// (a) the sidecar's own registry key — replicated by us natively
// (b) the sidecar already has its own replica of this key
const isOwn = ownKey16 && name === ownKey16
const sidecarHas = replicaRoot && fs.existsSync(path.join(replicaRoot, name))
if (isOwn || sidecarHas) {
try {
// Best-effort size for the log line; failure is fine.
let before = 0
try {
for (const f of fs.readdirSync(dir, { recursive: true })) {
try {
const st = fs.statSync(path.join(dir, f))
if (st.isFile()) before += st.size
} catch {}
}
} catch {}
fs.rmSync(dir, { recursive: true, force: true })
console.log('[pgh-sidecar] swept transient registry ' + name +
' (' + (before / 1024 / 1024).toFixed(1) + ' MB' +
(isOwn ? ' — own key' : ' — sidecar replica exists') + ')')
} catch {}
}
}
}
} catch (err) {
console.warn('[pgh-sidecar] transient registry sweep failed (non-fatal):', err.message)
}
}
/**
* Run rocksdb compaction across every corestore the sidecar owns +
* every per-subscription replica. Reclaims tombstone slack — the
* release-registry/db grew to 108 MB on tabertron pre-fix even
* though the live keyspace is much smaller, because rocksdb's
* background compaction is lazy on low-write workloads.
*
* Pattern validated by Keet 4.x: same primitive (storage.compact())
* exposed as the `core.compact` RPC at id 197 in keet-core's
* lib/rpc/server.js. Used periodically via redux-saga in keet-store.
*
* Logs per-store before/after when there's measurable reclamation.
* Errors per-store are non-fatal — the rest still try.
*/
async _compactAll () {
const stores = []
if (this._registryStore) stores.push({ name: 'release-registry', store: this._registryStore })
if (this._nameStore) stores.push({ name: 'name-registry', store: this._nameStore })
if (this._overlayStore) stores.push({ name: 'overlay-store', store: this._overlayStore })
for (const r of this._releaseReplicas) {
stores.push({ name: 'replica:' + r.keyHex.slice(0, 12), store: r.store })
}
const t0 = Date.now()
let totalBefore = 0
let totalAfter = 0
for (const { name, store } of stores) {
const dir = store && store.storage && store.storage.path
if (!store || !store.storage || typeof store.storage.compact !== 'function') {
continue
}
const before = dir ? safeDirSize(dir) : 0
try {
await store.storage.compact()
} catch (err) {
console.warn('[pgh-sidecar] compact(' + name + ') failed:', err.message)
continue
}
const after = dir ? safeDirSize(dir) : 0
totalBefore += before
totalAfter += after
const saved = before - after
console.log('[pgh-sidecar] compact ' + name + ': ' +
(before / 1024 / 1024).toFixed(1) + ' MB → ' +
(after / 1024 / 1024).toFixed(1) + ' MB' +
(saved > 0 ? ' (saved ' + (saved / 1024 / 1024).toFixed(1) + ' MB)' :
saved === 0 ? ' (no change)' : ''))
}
const elapsed = Date.now() - t0
const totalSaved = totalBefore - totalAfter
console.log('[pgh-sidecar] compact run done in ' + elapsed + 'ms — total ' +
(totalBefore / 1024 / 1024).toFixed(1) + ' MB → ' +
(totalAfter / 1024 / 1024).toFixed(1) + ' MB' +
(totalSaved > 0 ? ' (saved ' + (totalSaved / 1024 / 1024).toFixed(1) + ' MB)' : ''))
}
/**
* Open a read-only replica of another publisher's ReleaseRegistry.
* Each replica gets its own corestore (under releaseRegistryReplicasDir)
* and its own Hyperswarm so it can replicate independently.
*/
async _openReleaseReplica (keyHex) {
try {
fs.mkdirSync(this.releaseRegistryReplicasDir, { recursive: true })
const replicaDir = path.join(this.releaseRegistryReplicasDir, keyHex.slice(0, 16))
const store = new Corestore(replicaDir)
const registry = new ReleaseRegistry(store, { bootstrap: keyHex })
await registry.ready()
const swarm = new Hyperswarm()
swarm.on('error', (err) => {
console.error('[pgh-sidecar] replica-swarm(' + keyHex.slice(0, 12) + ') error (non-fatal):', err.code || err.message)
})
swarm.on('connection', (conn) => {
conn.on('error', () => {})
store.replicate(conn)
})
swarm.join(registry._base.discoveryKey, { server: true, client: true })
await swarm.flush()
return { keyHex, store, registry, swarm }
} catch (err) {
console.error('[pgh-sidecar] failed to open release-registry replica ' + keyHex.slice(0, 12) + ':', err.message)
return null
}
}
/**
* Open a read-only replica of another publisher's NameRegistry, joined
* on its discoveryKey so it stays warm. Lets `resolve-name` find names
* claimed in registries this sidecar doesn't own. Mirrors
* _openReleaseReplica. Returns { keyHex, store, registry, swarm } or null.
*/
async _openNameReplica (keyHex) {
try {
const dir = path.join(homedir(), '.config', 'pgh', 'name-registries', keyHex.slice(0, 16))
fs.mkdirSync(dir, { recursive: true })
const store = new Corestore(dir)
const registry = new NameRegistry(store, { bootstrap: keyHex })
await registry.ready()
const swarm = new Hyperswarm()
swarm.on('error', (err) => {
console.error('[pgh-sidecar] name-replica-swarm(' + keyHex.slice(0, 12) + ') error (non-fatal):', err.code || err.message)
})
swarm.on('connection', (conn) => {
conn.on('error', () => {})
store.replicate(conn)
})
swarm.join(registry.discoveryKey, { server: true, client: true })
await swarm.flush()
console.log('[pgh-sidecar] name-registry replica live: ' + keyHex.slice(0, 12) + '…')
return { keyHex, store, registry, swarm }
} catch (err) {
console.error('[pgh-sidecar] failed to open name-registry replica ' + keyHex.slice(0, 12) + ':', err.message)
return null
}
}
/**
* Resolve a bare name across the own NameRegistry first, then every
* subscribed read-only replica. Returns the entry or null.
*/
async _resolveNameAcross (name) {
let entry = null
try { entry = await this._nameRegistry.resolve(name) } catch {}
if (entry) return entry
for (const r of this._nameReplicas) {
try {
await r.registry.update()
const e = await r.registry.resolve(name)
if (e) return e
} catch {}
}
return null
}
/**
* Find a live registry (own writable or warm replica) for the given
* registryKeyHex. Returns the ReleaseRegistry instance or null.
*/
_findReleaseRegistry (registryKeyHex) {
if (!registryKeyHex) return null
if (this._registry && this._registry.keyHex === registryKeyHex) {
return this._registry
}
for (const r of this._releaseReplicas) {
if (r.keyHex === registryKeyHex) return r.registry
}
return null
}
_registerHandlers () {
const svc = this._svc
svc.handle('ping', async () => ({
ok: true,
sidecar: 'pgh-sidecar',
version: SIDECAR_VERSION,
uptime: Math.floor(process.uptime())
}))
svc.handle('identity-info', async () => {
const id = this._identity || {}
const devicePubkey = id.publicKeyHex ||
(id.publicKey ? id.publicKey.toString('hex') : null)
// The root, via the one accessor that reads rootPubkeyHex first. Reaching
// for masterPublicKey alone reported the DEVICE key on a linked device
// (org #142) — hyperident sets it, `pgh device join` does not.
const rootPubkey = identityRootHex(id)
return {
devicePubkey,
rootPubkey,
hasMnemonic: !!id.mnemonic,
method: id.method || 'ed25519',
source: id.source || 'raw'
}
})
// `pgh store gc --keep-releases N` lands here (see _storeGc for why).
svc.handle('store-gc', async (ctx, params = {}) => {
const keep = parseInt(params.keepReleases, 10)
const r = await this._storeGc({
dryRun: !!params.dryRun,
keepReleases: Number.isFinite(keep) && keep > 0 ? keep : KEEP_RELEASES
})
return {
candidates: r.candidates,
removed: r.removed,
kept: r.kept.length,
blobsRemoved: r.blobsRemoved
}
})
svc.handle('store-stats', async () => {
return this._treeStore.stats()
})
svc.handle('store-list', async () => {
return this._treeStore.listTrees().map(t => ({
hash: t.hash,
size: t.size,
refs: t.refs,
pinned: t.pinned
}))
})
// ── Release registry handlers ────────────────────────────────
// The sidecar owns the registry corestore. pgh CLI calls these
// over UDS instead of opening the store itself (which would fight
// the sidecar for the RocksDB single-writer lock).
svc.handle('registry-info', async () => ({
keyHex: this._registry._base.key ? b4a.toString(this._registry._base.key, 'hex') : null,
writable: !!this._registry._base.writable,
discoveryKey: this._registry._base.discoveryKey
? b4a.toString(this._registry._base.discoveryKey, 'hex')
: null,
subscribedReplicas: this._releaseReplicas.map(r => r.keyHex)
}))
// Resolve a release entry against ANY live registry the sidecar
// knows about — own writable one OR warm read-only replicas. The
// replicas stay fresh via autobase update events in the
// background, so no one-shot swarm + setTimeout dance needed here.
svc.handle('resolve-release', async (ctx, params) => {
if (!params || !params.registryKey || !params.name) {
throw new Error('resolve-release requires { registryKey, name, version? }')
}
const registry = this._findReleaseRegistry(params.registryKey)
if (!registry) {
return { found: false, reason: 'registry not subscribed on this sidecar', registryKey: params.registryKey }
}
await registry.update()
const entry = params.version
? await registry.get(params.name, params.version)
: await registry.latest(params.name)
return { found: !!entry, entry: entry || null }
})
svc.handle('publish-release', async (ctx, params) => {
if (!params || !params.entry) throw new Error('publish-release requires { entry }')
const e = params.entry
if (!e.name || !e.version || !e.treeHash) {
throw new Error('entry missing required fields { name, version, treeHash }')
}
await this._registry.add(e)
// Re-scan DeployServer's FileIndex so the freshly-staged tree
// bytes become servable to peers. INCREMENTAL: pass the just-
// staged tree hash so the scan walks only that subdir. Legacy
// full-scan was O(total bytes in store) and on a 24 GB store
// took multiple minutes per publish — long enough that
// `pgh app install` called immediately after publish-release
// returned would error with "tree not found on 1 peer(s)".
// hyper-deploy >= 0.4.3 understands { newHashes }; older
// versions ignore opts and run the legacy full scan.
if (this._deployServer && typeof this._deployServer.rescan === 'function') {
// Index every tree staged by this publish: root + each dep's
// base tree + each dep's per-arch prebuild tree. Pre-0.2.18
// this only indexed the root, leaving dep trees on disk but
// unreachable to NetworkFetcher.getTree (the call
// `pgh app install` makes when fetching the closure).
const newHashes = [e.treeHash]
for (const dep of (e.deps || [])) {
if (dep.tree) newHashes.push(dep.tree)
if (dep.prebuilds) {
for (const h of Object.values(dep.prebuilds)) {
if (h) newHashes.push(h)
}
}
}
try {
const r = await this._deployServer.rescan({ newHashes })
if (r && r.addedFiles !== undefined) {
console.log('[pgh-sidecar] incremental rescan: +' + r.addedFiles +
' files, +' + r.addedBytes + ' bytes across ' +
newHashes.length + ' tree(s) (root ' + e.treeHash.slice(0, 12) + ')')
}
} catch (err) {
console.warn('[pgh-sidecar] DeployServer rescan failed:', err.message)
}
}
return {
ok: true,
name: e.name,
version: e.version,
treeHash: e.treeHash,
registryKey: b4a.toString(this._registry._base.key, 'hex')
}
})
svc.handle('remove-release', async (ctx, params = {}) => {
// Un-register an app (retire). Drops it from the replicated release
// registry the sidecar owns, so it stops being advertised/installable
// and stops polluting cross-app conflict detection. version omitted ⇒
// remove every version of the app.
if (!params.name) throw new Error('remove-release requires { name }')
let removed = 0
if (params.version) {
await this._registry.remove(params.name, params.version)
removed = 1
} else {
const entries = await this._registry.list(params.name)
for (const e of entries) { await this._registry.remove(e.name, e.version); removed++ }
}
return { ok: true, name: params.name, removed }
})
svc.handle('releases-list', async (ctx, params) => {
const filters = params || {}
const own = await this._registry.list(filters.name)
const fromReplicas = []
for (const r of this._releaseReplicas) {
try { fromReplicas.push(...(await r.registry.list(filters.name))) } catch {}
}
return [...own, ...fromReplicas]
})
// Slim projection of releases-list, for callers that only DISPLAY releases.
//
// Every registry entry embeds its full dep closure — measured 2026-08-07,
// ONE pear-git-v3 entry is ~120 KB (285 deps, each with a per-arch prebuild
// hash map). So `releases-list` has to serialise that once per version:
// `pgh releases list pear-git-v3` never returns, and pear-git-v3's
// `tree-registry-list` (which reads the store directly) could not be used to
// back a UI even after its lock conflict was fixed.
//
// `releases-list` is deliberately left intact — install/resolve paths need
// `deps`. This one carries identity + provenance only, plus depCount so a
// caller can still show closure size without paying for it.
svc.handle('releases-summary', async (ctx, params) => {
const filters = params || {}
const slim = (e) => ({
name: e.name,
version: e.version,
treeHash: e.treeHash,
publishedAt: e.publishedAt,
publisherDevicePubkey: e.publisherDevicePubkey,
runtime: e.runtime || null,
upgrade: e.upgrade || null,
depCount: Array.isArray(e.deps) ? e.deps.length : 0
})
const out = (await this._registry.list(filters.name)).map(slim)
for (const r of this._releaseReplicas) {
try { out.push(...(await r.registry.list(filters.name)).map(slim)) } catch {}
}
return out
})
svc.handle('releases-latest', async (ctx, params) => {
if (!params || !params.name) throw new Error('releases-latest requires { name }')
let best = await this._registry.latest(params.name)
// Also check warm replicas of other publishers' registries — without
// this, a subscriber's own registry has no entries so OTA never sees
// anything. Prefer the newest publishedAt across own + replicas.
for (const r of this._releaseReplicas) {
try {
await r.registry.update()
const e = await r.registry.latest(params.name)
if (!e) continue
if (!best || (e.publishedAt && (!best.publishedAt || e.publishedAt > best.publishedAt))) best = e
} catch {}
}
return best || null
})
svc.handle('releases-get', async (ctx, params) => {
if (!params || !params.name || !params.version) {
throw new Error('releases-get requires { name, version }')
}
const entry = await this._registry.get(params.name, params.version)
return entry || null
})
svc.handle('seed-status', async () => ({
enabled: this.enableSeed,
topic: this.enableSeed && this._identity
? b4a.toString(this._identity.publicKey, 'hex')
: null,
deployServer: this._deployServer ? 'running' : 'off',
registrySwarm: this._registrySwarm ? 'advertising' : 'off',
nameRegistrySwarm: this._nameRegistrySwarm ? 'advertising' : 'off',
treeCount: this._treeStore
? fs.readdirSync(this._treeStore.dependenciesDir).filter(e => !e.startsWith('.')).length
: 0
}))
// ── Name registry handlers ───────────────────────────────────
svc.handle('name-registry-info', async () => ({
keyHex: this._nameRegistry.keyHex,
writable: this._nameRegistry.writable,
discoveryKey: this._nameRegistry.discoveryKey
? b4a.toString(this._nameRegistry.discoveryKey, 'hex')
: null
}))
svc.handle('resolve-name', async (ctx, params) => {
if (!params || !params.name) throw new Error('resolve-name requires { name }')
// Own registry first, then subscribed read-only replicas.
const entry = await this._resolveNameAcross(params.name)
return entry || null
})
// Subscribe to another publisher's NameRegistry by autobase key:
// persists to subscriptions.json AND opens a live read-only replica
// immediately (no sidecar restart). Idempotent.
svc.handle('subscribe-name-registry', async (ctx, params) => {
const keyHex = params && params.registryKey
if (!keyHex || typeof keyHex !== 'string' || keyHex.length !== 64) {
throw new Error('subscribe-name-registry requires { registryKey } (64-char hex)')
}
const ownNameKey = this._nameRegistry && this._nameRegistry.keyHex
if (keyHex === ownNameKey) {
return { ok: true, alreadyOwn: true, subscriptions: this._nameReplicas.map(r => r.keyHex) }
}
const persisted = addNameSubscription(keyHex)
let opened = false
if (!this._nameReplicas.some(r => r.keyHex === keyHex)) {
const replica = await this._openNameReplica(keyHex)
if (replica) { this._nameReplicas.push(replica); opened = true }
else throw new Error('failed to open name-registry replica for ' + keyHex.slice(0, 12))
}
return { ok: true, persisted, opened, subscriptions: this._nameReplicas.map(r => r.keyHex) }
})
// List name-registry subscriptions (own key + replica keys).
svc.handle('name-subscriptions', async () => ({
own: this._nameRegistry ? this._nameRegistry.keyHex : null,
replicas: this._nameReplicas.map(r => ({ keyHex: r.keyHex, writable: r.registry.writable }))
}))
// The identity acting on a name claim, in the shape NameRegistry's write
// methods take. A signed (v2) op needs the device secret AND the keet
// attestation chain; an identity with neither — the raw/legacy fallback,
// which has no chain to offer — degrades to the legacy string form so it
// keeps working exactly as before rather than throwing. See org #143.
this._nameSigner = () => {
const id = this._identity
const root = identityRootHex(id)
if (!id || !id.chain || !id.secretKey) return root
return {
identityRoot: root,
devicePubkey: id.publicKeyHex,
deviceSecretKey: id.secretKey,
chain: id.chain
}
}
svc.handle('claim-name', async (ctx, params) => {
if (!params || !params.name) throw new Error('claim-name requires { name }')
if (!this._nameRegistry.writable) {
throw new Error('name registry is not writable on this sidecar (join via invite first)')
}
const registryKeyHex = this._registry._base.key
? b4a.toString(this._registry._base.key, 'hex')
: null
const devicePubkey = this._identity && this._identity.publicKeyHex
// The anchor a name is owned by, and what `update-name` will authorise
// against — it MUST be the root, or rotating the device orphans the
// claim (org #142).
const identityRoot = identityRootHex(this._identity)
const publisher = {
devicePubkey,
registryKey: params.registryKey || registryKeyHex,
serverKey: params.serverKey || devicePubkey,
identityRoot
}
await this._nameRegistry.claim({
name: params.name,
publisher,
description: params.description || ''
})
return await this._nameRegistry.resolve(params.name)
})
// Repair a claim written before org #142, whose identityRoot holds this
// device's key instead of the root. Only the anchoring device can do it,
// and only to its own chain's root — the registry enforces both; the
// checks here exist to give a real error instead of an op that apply
// silently drops.
svc.handle('reanchor-name', async (ctx, params) => {
if (!params || !params.name) throw new Error('reanchor-name requires { name }')
if (!this._nameRegistry.writable) {
throw new Error('name registry is not writable on this sidecar (join via invite first)')
}
const existing = await this._nameRegistry.resolve(params.name)
if (!existing) throw new Error('name not claimed: ' + params.name)
const signer = this._nameSigner()
if (typeof signer === 'string') {
throw new Error('this identity has no attestation chain, so it cannot prove which root owns the name')
}
const anchor = existing.publisher.identityRoot
if (anchor !== signer.identityRoot && anchor !== signer.devicePubkey) {
throw new Error(
'name "' + params.name + '" is anchored to ' + String(anchor).slice(0, 12) +
'… — this device is ' + String(signer.devicePubkey).slice(0, 12) +
'…; only the anchoring key can re-anchor'
)
}
// `force` re-emits the op even when THIS peer's view already shows the
// claim re-anchored. It exists because a peer running code without an
// OP_REANCHOR_NAME branch consumes the op and advances past it — a new
// op type does not degrade gracefully the way a new field does, and
// upgrading afterwards does not re-run history. A fresh op re-applies on
// peers still showing the device anchor and is skipped by peers already
// on the root, since apply tests `claim.identityRoot === op.devicePubkey`
// against each peer's OWN view. Idempotent by construction.
if (anchor === signer.identityRoot && !params.force) {
return { ok: true, alreadyAnchored: true, entry: existing }
}
await this._nameRegistry.reanchor(params.name, signer)
const entry = await this._nameRegistry.resolve(params.name)
// Under force the local view legitimately does not change — the op is
// for peers that are behind — so only assert progress when this peer was
// the one that needed it.
if (anchor === signer.devicePubkey && entry.publisher.identityRoot !== signer.identityRoot) {
throw new Error('re-anchor did not apply for "' + params.name + '" (op refused)')
}
return { ok: true, forced: !!params.force, entry }
})
svc.handle('release-name', async (ctx, params) => {
if (!params || !params.name) throw new Error('release-name requires { name }')
await this._nameRegistry.release(params.name, this._nameSigner())
return { ok: true, released: params.name }
})
svc.handle('update-name', async (ctx, params) => {
if (!params || !params.name) throw new Error('update-name requires { name }')
if (!this._nameRegistry.writable) {
throw new Error('name registry is not writable on this sidecar (join via invite first)')
}
const existing = await this._nameRegistry.resolve(params.name)
if (!existing) throw new Error('name not claimed: ' + params.name)
// Fail here rather than appending an op apply will drop — a dropped op
// is indistinguishable from replication lag at the call site.
const signer = this._nameSigner()
const mine = typeof signer === 'string' ? signer : signer.identityRoot
if (existing.publisher.identityRoot !== mine) {
throw new Error(
'name "' + params.name + '" is owned by ' +
String(existing.publisher.identityRoot).slice(0, 12) + '… — this identity is ' +
String(mine).slice(0, 12) + '…'
)
}
const updates = {}
if (params.registryKey) updates.registryKey = params.registryKey
if (params.serverKey) updates.serverKey = params.serverKey
if (params.description !== undefined) updates.description = params.description
if (Object.keys(updates).length === 0) {
throw new Error('update-name requires at least one of { registryKey, serverKey, description }')
}
await this._nameRegistry.updateName(params.name, updates, signer)
return await this._nameRegistry.resolve(params.name)
})
svc.handle('list-names', async (ctx, params) => {
// Merge own registry + subscribed replicas, dedup by name (own wins).
const seen = new Map()
const own = await this._nameRegistry.list(params || {})
for (const e of (own || [])) if (e && e.name) seen.set(e.name, e)
for (const r of this._nameReplicas) {
try {
await r.registry.update()
const list = await r.registry.list(params || {})
for (const e of (list || [])) if (e && e.name && !seen.has(e.name)) seen.set(e.name, e)
} catch {}
}
return [...seen.values()]
})
// Stubs — declared to make the UDS surface predictable. Real logic
// lands in later phases.
svc.handle('check-updates', async () => ({
implemented: false,
note: 'OTA polling lands with 0.5f (pgh-runtime)'
}))
svc.handle('device-invite', async () => ({
implemented: false,
note: 'Device linking lands when pgh-sidecar grows the blind-pairing wiring'
}))
// ── App registry ────────────────────────────────────────────
//
// Apps that pgh-install'd + launched register themselves on boot so
// the pgh CLI can find their UDS for commands like `pgh app resync`
// / `pgh app status` without either side hardcoding paths. Lookup
// is by app name (the `--name` used at install time). Unregister
// happens on app SIGTERM.
svc.handle('app-register', async (ctx, params = {}) => {
if (!params.name) throw new Error('app-register requires { name }')
if (!params.udsPath) throw new Error('app-register requires { udsPath }')
const entry = {
name: params.name,
udsPath: params.udsPath,
pid: params.pid || null,
runtime: params.runtime || null,
version: params.version || null,
treeHash: params.treeHash || null,
registeredAt: Date.now()
}
this._apps.set(params.name, entry)
this._saveAppsToDisk()
return { ok: true, name: params.name }
})
svc.handle('app-unregister', async (ctx, params = {}) => {
if (!params.name) throw new Error('app-unregister requires { name }')
const existed = this._apps.delete(params.name)
if (existed) this._saveAppsToDisk()
return { ok: true, existed }
})
svc.handle('app-lookup', async (ctx, params = {}) => {
if (!params.name) throw new Error('app-lookup requires { name }')
const entry = this._apps.get(params.name)
if (!entry) return { found: false }
return { found: true, ...entry }
})
svc.handle('app-list-running', async () => {
return { apps: Array.from(this._apps.values()) }
})
// ── Overlay (git-LFS-style large-asset routing) ──────────
//
// pgh CLI's `pgh overlay <init|track|sync|clean|smudge|status>` calls
// these handlers via UDS. Storage is the per-repo FileIndex managed by
// OverlayManager; cross-peer fetches go through this sidecar's overlay
// swarm (Protomux channel pgh-overlay/v1).
svc.handle('overlay-init', async (ctx, params = {}) => {
if (!this._overlayManager) throw new Error('overlay disabled (enableSeed=false)')
if (!params.repo) throw new Error('overlay-init requires { repo }')
if (!params.repoRoot) throw new Error('overlay-init requires { repoRoot }')
return this._overlayManager.initRepo(params.repo, params.repoRoot)
})
svc.handle('overlay-info', async (ctx, params = {}) => {
if (!this._overlayManager) throw new Error('overlay disabled')
if (!params.repo) throw new Error('overlay-info requires { repo }')
const info = await this._overlayManager.getRepoInfo(params.repo)
return {
...info,
topicHex: this._overlayTopic ? this._overlayTopic.toString('hex') : null,
serverKey: this._identity ? Buffer.from(this._identity.publicKey).toString('hex') : null
}
})
svc.handle('overlay-put-file', async (ctx, params = {}) => {
if (!this._overlayManager) throw new Error('overlay disabled')
if (!params.repo) throw new Error('overlay-put-file requires { repo }')
if (!params.path) throw new Error('overlay-put-file requires { path }')
if (!params.oid) throw new Error('overlay-put-file requires { oid }')
const meta = {
oid: params.oid,
size: params.size || 0,
mtime: params.mtime || 0,
executable: !!params.executable,
linkname: params.linkname || null
}
// chunkmap is optional [{ hash, offset, length }] — when present, the
// sidecar persists it on the FileIndex's chunkmap:<path> key. Used by
// the chunked-transfer protocol (#14d).
if (Array.isArray(params.chunkmap)) meta.chunkmap = params.chunkmap
return this._overlayManager.putFile(params.repo, params.path, meta)
})
svc.handle('overlay-get-file', async (ctx, params = {}) => {
if (!this._overlayManager) throw new Error('overlay disabled')
if (!params.repo || !params.path) throw new Error('overlay-get-file requires { repo, path }')
return this._overlayManager.getFile(params.repo, params.path)
})
svc.handle('overlay-delete-file', async (ctx, params = {}) => {
if (!this._overlayManager) throw new Error('overlay disabled')
if (!params.repo || !params.path) throw new Error('overlay-delete-file requires { repo, path }')
return this._overlayManager.deleteFile(params.repo, params.path)
})
svc.handle('overlay-list-files', async (ctx, params = {}) => {
if (!this._overlayManager) throw new Error('overlay disabled')
if (!params.repo) throw new Error('overlay-list-files requires { repo }')
const files = await this._overlayManager.listFiles(params.repo)
return { repo: params.repo, files }
})
svc.handle('overlay-hash-file', async (ctx, params = {}) => {
// Compute oid+size+mtime for a working-tree file. Used by the clean
// filter to determine the pointer contents at `git add` time.
if (!this._overlayManager) throw new Error('overlay disabled')
if (!params.repo || !params.path) throw new Error('overlay-hash-file requires { repo, path }')
return this._overlayManager.hashWorkingTreeFile(params.repo, params.path)
})
svc.handle('overlay-status', async (ctx, params = {}) => {
if (!this._overlayManager) throw new Error('overlay disabled')
return this._overlayManager.status()
})
// overlay-locate — the sole cross-machine surface for asset retrieval.
// Returns the absolute working-tree path so a co-located CLI can read
// bytes directly via fs — UDS never carries asset bytes (#14e).
//
// Modes:
// { oid } local-only lookup; returns
// { found: false } if not local.
// { oid, peer, ensure: true } if not local, joins the peer's
// overlay topic on the existing
// swarm, polls local FileIndex
// until the v2 auto-sync writes
// the file to <repoRoot>/<relPath>,
// then returns the absPath.
//
// CLI reads bytes via `fs.createReadStream(absPath)` — works for
// files of any size (no Buffer/base64 ceiling).
svc.handle('overlay-locate', async (ctx, params = {}) => {
if (!this._overlayManager) throw new Error('overlay disabled')
if (!params.oid) throw new Error('overlay-locate requires { oid }')
const fs = await import('#node:fs')
const path = await import('#node:path')
const tryLocal = async () => {
const found = await this._overlayManager.findByOid(params.oid)
if (!found || !found.repoRoot) return null
try {
const abs = path.join(found.repoRoot, found.path)
const stat = fs.statSync(abs)
if (!stat.isFile()) return null
return {
found: true,
oid: params.oid,
repo: found.reponame,
path: found.path,
repoRoot: found.repoRoot,
absPath: abs,
size: stat.size,
mtime: Math.floor(stat.mtimeMs)
}
} catch {
return null
}
}
// Local fast path
const localHit = await tryLocal()
if (localHit) return { ...localHit, source: 'local' }
// No ensure → just report not-found.
if (!params.ensure || !params.peer) {
return { found: false, oid: params.oid }
}
if (!this._overlaySwarm) throw new Error('overlay-locate: swarm not running')
// Join the peer's overlay topic on the existing swarm. The already-
// attached MultiRepoOverlayProtocolV2 picks up new connections and
// auto-syncs offered files into <repoRoot>/<relPath> via
// OverlayManager.getDiskPathSync. Topics persist across calls so
// future ensures from the same peer skip re-discovery.
const peerKey = Buffer.from(params.peer, 'hex')
const peerTopic = crypto.hash(Buffer.concat([peerKey, Buffer.from('pgh-overlay/v1')]))
const peerTopicHex = peerTopic.toString('hex')
if (!this._overlayPeerTopics.has(peerTopicHex)) {
const discovery = this._overlaySwarm.join(peerTopic, { server: false, client: true })
try { await discovery.flushed() } catch {}
this._overlayPeerTopics.add(peerTopicHex)
}
// Poll for arrival in our local FileIndex.
const TIMEOUT_MS = (typeof params.timeoutMs === 'number' && params.timeoutMs > 0)
? params.timeoutMs
: 60000
const POLL_INTERVAL = 250
const startedAt = Date.now()
while (Date.now() - startedAt < TIMEOUT_MS) {
const hit = await tryLocal()
if (hit) return { ...hit, source: 'peer' }
await new Promise(r => setTimeout(r, POLL_INTERVAL))
}
throw new Error('EOVERLAYNOPEER: timed out fetching oid ' + params.oid.slice(0, 12))
})
}
}
/**
* Main entrypoint — used by bin/pgh-sidecar.
*/
export async function main (opts = {}) {
const sidecar = new Sidecar(opts)
const info = await sidecar.start()
console.log('pgh-sidecar listening:')
console.log(' socket: ' + info.socketPath)
console.log(' identity: ' + info.identityPath)
console.log(' store: ' + info.storeRoot)
const shutdown = async () => {
console.log('shutting down...')
await sidecar.stop()
process.exit(0)
}
process.on('SIGINT', shutdown)
process.on('SIGTERM', shutdown)
return sidecar
}