/**
* Sync service — managed wrapper around hyper-sync (git over Hyperswarm).
*
* hyper-sync is CommonJS. We bridge it via createRequire.
*
* The repos directory at `<storage>/git-repos/` is scanned for bare git
* repos (*.git directories).
*
* ## Dedicated Hyperswarm
*
* hyper-sync speaks the git smart protocol over **raw connection bytes**
* (see `_onConnection` — parses `service\0repo\0` from the byte stream).
* That's incompatible with protomux-framed protocols like corestore
* replication: corestore's control frames contain nulls that hyper-sync's
* header parser treats as a git header, fails to resolve the repo, and
* calls `conn.destroy()`.
*
* pear-git v1 documents and follows the same rule in
* `pear-git/pear-git-server.js`:
*
* // Each swarm MUST be separate: corestore.replicate() sends Protomux
* // framing that corrupts raw streams, and shared swarms cause data
* // delivery failures [...] (connection dedup + listener conflicts)
*
* So the sync service creates its OWN Hyperswarm. The orchestrator passes
* `hyperswarmOpts` (bootstrap, keyPair, etc. — e.g. the testnet bootstrap
* for tests) which we forward to the new Hyperswarm instance. The shared
* hyper-svc swarm continues to handle corestore replication alone.
*
* Only started when the orchestrator has swarm enabled.
*/
import { mkdir } from '#node:fs/promises'
import { watch, existsSync } from '#node:fs'
import { join } from '#node:path'
import HyperSync from 'hyper-sync'
// Deep import: hyper-sync exposes scanRepos/repoInfo/readRefs from this
// module but re-exports none of them from its index. repoInfo is the
// per-repo half of scanRepos — see attachRepoRefresh below for why the
// whole-dir version is the wrong tool after a single pull.
import hyperSyncRepos from 'hyper-sync/lib/repos.js'
import Hyperswarm from 'hyperswarm'
import { enrichRepos } from './repo-activity.js'
import { mergeRefActivity } from './repo-ref-activity.js'
// execFile (async), not execFileSync — this runs inside the daemon's event loop
// and spawnSync would stall every in-flight HTTP request for the child's life.
import { execFile } from '../_compat-exec.js'
import { ensureExportMarker } from '../repo-access.js'
/**
* Give a hyper-sync server a `refreshRepo(name)` — re-stat ONE repo and
* update its entry in the in-memory map that status() serves.
*
* WHY THIS EXISTS. `sync.repos` is a cache, and after boot almost nothing
* refreshes it:
*
* - the repos-dir watcher below is a NON-RECURSIVE `watch(reposDir)`, so it
* fires when a repo directory appears or disappears and never when refs
* are written INSIDE an existing `<repo>.git`;
* - RepoPuller fetches into existing repos continuously on a mirror and its
* only hook is onMount -> scheduleMirrorReannounce.
*
* So a mirror served boot-time branch counts, pack sizes and lastUpdated for
* repos it had since pulled. Measured on the live fleet 2026-09-05: stortron,
* up 3 days, reported blender-bridge lastUpdated 2026-06-12 while its own disk
* said 2026-09-04 15:57 — refs 2 seconds behind the origin's. Since /repos
* orders on lastUpdated, four actively-pushed repos sat mid-page on one
* backend and top-of-page on the other.
*
* PER-REPO, deliberately. The obvious fix is `sync.scan()`, but scanRepos()
* walks every repo SYNCHRONOUSLY (364ms of a 409ms scan across 147 live
* repos) and a mirror pulls repos one at a time — one full walk per pulled
* repo would stall the event loop through the whole drain. repoInfo() is the
* same computation for a single repo, and it takes the previous record so the
* dirSize walk is skipped when the footprint has not moved.
*
* Only refreshes repos ALREADY in the map: adding one is the watcher's job
* (it sees the new directory) and going through here would bypass the
* retired-repo prune that every scan path applies.
*
* @param {object} sync - the hyper-sync server
* @param {string} reposDir - bare-repos directory
*/
export function attachRepoRefresh (sync, reposDir) {
sync.refreshRepo = (name) => {
if (!name || !sync.repos || typeof sync.repos.get !== 'function') return null
const prev = sync.repos.get(name)
if (!prev) return null
try {
const info = hyperSyncRepos.repoInfo(join(reposDir, name + '.git'), name, prev)
if (!info) return null
sync.repos.set(name, info)
return info
} catch {
// A repo that vanished mid-pull, or an unreadable dir. The stale entry
// is better than dropping the repo out of the listing entirely.
return null
}
}
return sync
}
export async function startSyncService (opts) {
const { reposDir, keypair, hyperswarmOpts } = opts
if (!reposDir) throw new Error('sync service requires reposDir')
await mkdir(reposDir, { recursive: true })
// Give hyper-sync its OWN Hyperswarm — sharing with corestore replication
// corrupts raw git streams. Pass the daemon's keypair and any caller-provided
// options (testnet bootstrap, etc.) through.
//
// UNLESS the caller disabled the swarm. `offline: true` means NO NETWORK: no
// Hyperswarm is constructed, nothing is announced, no peer is dialled. This
// used to be impossible to express — the orchestrator passed
// `hyperswarmOpts: null` when its swarm was off, but `{...(null || {})}` is
// just `{}`, so a live DHT-connected swarm was built anyway and pulled from
// real peers. A daemon started with `swarm: false` was still networking.
//
// Local function is unaffected: scan/serve/info/handleInfoRefs all still work,
// so HTTP git serving and the repos map behave identically offline.
const ownSwarm = opts.offline
? null
: (() => {
const swarmCtorOpts = { ...(hyperswarmOpts || {}) }
if (keypair && !swarmCtorOpts.keyPair) swarmCtorOpts.keyPair = keypair
const s = new Hyperswarm(swarmCtorOpts)
s.on('error', (err) => {
// eslint-disable-next-line no-console
if (err && err.code !== 'ECONNRESET') console.error('[sync/swarm]', err.message)
})
return s
})()
// `false`, not null: hyper-sync distinguishes "no swarm, do not make one"
// (false) from "none injected, make your own" (null/undefined). Passing null
// here is exactly the bug this fixes.
const syncOpts = { repos: reposDir, swarm: opts.offline ? false : ownSwarm }
if (keypair) syncOpts.keypair = keypair
// Push authorization hook (issue #37, item 2). Forwarded to hyper-sync:
// git-upload-pack (read) stays open; git-receive-pack (push) is gated.
// The daemon supplies a fn that resolves the peer's swarm noise key to
// an IdentityRegistry user and requires write access.
if (typeof opts.authorize === 'function') syncOpts.authorize = opts.authorize
const sync = new HyperSync(syncOpts)
sync.on('error', (err) => {
// eslint-disable-next-line no-console
if (err.code !== 'ECONNRESET') console.error('[sync]', err.message)
})
await sync.ready()
// Tombstone gate. This is resurrection path 3: scan() re-populates the map
// from whatever bare dirs exist on disk and serve() re-announces them, so a
// retired repo came back on the next scan/watch tick. We prune AFTER scan
// and BEFORE serve, which keeps the policy here in pgv3 — hyper-sync stays
// a dumb transport and needs no change.
//
// Pruning drops the in-memory entry ONLY. The bare directory is untouched;
// removing bytes is a manual operator step on each peer, and this daemon
// must never gain the ability to do it. See services/retired-repos.js.
const isRetired = typeof opts.isRetired === 'function' ? opts.isRetired : () => false
const pruneRetired = () => {
if (!sync.repos || typeof sync.repos.keys !== 'function') return 0
let pruned = 0
for (const name of [...sync.repos.keys()]) {
if (isRetired(name)) { sync.repos.delete(name); pruned++ }
}
return pruned
}
// The gate belongs to scan(), not to whoever calls it.
//
// hyper-sync's scan() does not merge — it REPLACES `this.repos` with a
// fresh map built from the repos directory, and status() iterates exactly
// that map. Pruning after the boot scan and after each watcher rescan was
// therefore not enough: pruneRetired is a closure reachable from nowhere
// else, so the three sites that scan directly — the 'scan-repos' and
// 'create-repo' RPCs below, and the post-pull rescan in pear-git.js — put
// every tombstoned bare dir straight back into the listing. Production,
// 2026-09-02: the origin booted with 15 tombstones and listed 132 repos;
// two `scan-repos` calls later it listed 148, and the SSR /repos page
// advertised repos the operator had retired. (They stayed UNSERVED —
// serve() is only reached through paths that prune — so it leaked into
// the listing, not onto the swarm.)
//
// Wrapping the instance method fixes all three call sites and any future
// one. scan() returns `this.repos` itself, so pruning before the return
// gives every caller the filtered map.
const rawScan = sync.scan.bind(sync)
sync.scan = async (...args) => {
const repos = await rawScan(...args)
pruneRetired()
return repos
}
// Exposed for the case where the tombstone set is populated AFTER this
// service has already scanned.
//
// On an ORIGIN that cannot happen: pear-git.js folds the daemon's own
// RepoEventLog before starting us, so the set is complete by the boot scan.
// On a MIRROR it is the normal case — the mirror's own log carries no
// tombstones, so the set is EMPTY here, and it only fills in later when
// RepoPuller folds the ORIGIN's log. Without a way to re-prune, the mirror
// ends up knowing the tombstones while still listing the repos.
//
// (stortron, 2026-09-02, right after 0.1.247 shipped: `pgh repo retired`
// returned all 15 names while `pgh repo` still listed 148. A manual
// `pgh repo scan` dropped it to 133 — the set was right, only the
// already-scanned map was stale.)
//
// This prunes the in-memory map only; it does NOT re-walk the repos dir, so
// the caller pays nothing for the disk scan it does not need.
sync.pruneRetired = pruneRetired
// Per-repo re-stat, for the paths that change ONE repo's bytes without
// touching the repos dir itself — RepoPuller's fetch above all.
attachRepoRefresh(sync, reposDir)
// Populate the in-memory map. We don't call serve() here — the caller
// attaches listeners (repoEventLog's on('serve') hook, etc.) AFTER we
// return, so serve() must run after listener wire-up or the initial
// 'serve' emits get missed. See pear-git.js where serve() is called
// explicitly once the sync server is fully wired.
await sync.scan().catch(() => {})
// Watch the repos directory so repos added after boot (new clone, mkdir,
// rename-in) get picked up without a restart. Debounced because a single
// clone can emit many rapid events. _serveOne is idempotent (guards on
// this._topics.has(name)) so re-announcing existing repos is free.
let rescanTimer = null
const cancelRescan = () => {
if (rescanTimer) {
clearTimeout(rescanTimer)
rescanTimer = null
}
}
const reposWatcher = watch(reposDir, { persistent: false }, () => {
if (rescanTimer) return
rescanTimer = setTimeout(async () => {
rescanTimer = null
// Skip if the sync server is shutting down — the debounce timer may
// have been queued after close() began, and serve() would fail on
// the destroyed swarm.
if (sync.closing || sync.closed) return
try {
await sync.scan() // prunes retired names — see the wrapper above
await sync.serve()
} catch (err) {
if (sync.closing || sync.closed) return
// eslint-disable-next-line no-console
console.error('[sync] rescan/serve failed:', err && err.message)
}
}, 500)
})
reposWatcher.on('error', (err) => {
// eslint-disable-next-line no-console
console.error('[sync] repos-dir watcher error:', err && err.message)
})
reposWatcher.cancelRescan = cancelRescan
// Expose the dedicated swarm + watcher to the caller so it can tear them
// down — hyper-sync's `_ownSwarm = false` when swarm is injected, so
// hyper-sync itself will not destroy it on close().
sync._dedicatedSwarm = ownSwarm
sync._reposWatcher = reposWatcher
return sync
}
/**
* RPC handlers for the sync service.
*/
export function syncHandlers (svc) {
return [
{
name: 'list-repos',
fn: async () => {
const sync = svc._sync
if (!sync) return { repos: [], active: false }
const status = sync.status()
// Annotate with the REPLICATED last-ref-update timestamp, which is what
// /repos orders on. status() only carries the local mtime, and that is
// a property of this machine — the two load-balanced backends never
// agreed on it. See services/repo-ref-activity.js.
return { active: true, ...status, repos: mergeRefActivity(status.repos) }
}
},
{
// Same as list-repos but augments each repo with description,
// commits, lastCommit, and a 365-day activity bucket array for the
// /repos landing-page sparkline. Shelling out to git per repo so
// we cache results keyed on pack mtime — see repo-activity.js.
//
// `names` restricts the git work to a subset. The landing page orders
// on the cheap `lastUpdated` from status() and then needs git data for
// ONE page of cards — walking every repo just to pick the top 25 was
// the cold cost this avoids (147 repos, 28.3s cold, to render 25).
// Omitted means the whole fleet, so existing callers are unchanged.
//
// Filtering against the real repo list means git fan-out stays bounded
// by fleet size no matter what a caller sends — the same bound as the
// no-args call this RPC already accepted anonymously.
name: 'list-repos-detailed',
fn: async (ctx, params = {}) => {
const sync = svc._sync
if (!sync) return { repos: [], active: false }
const status = sync.status()
let subject = { ...status, repos: mergeRefActivity(status.repos) }
if (Array.isArray(params.names)) {
const wanted = new Set(params.names)
subject = { ...subject, repos: subject.repos.filter((r) => wanted.has(r.name)) }
}
// enrichRepos spreads the base record through, so lastRefUpdate rides
// along and the fragment route orders the same way the first page did.
const repos = await enrichRepos(subject, sync.repos)
return { active: true, ...status, repos }
}
},
{
name: 'repo-info',
fn: async (ctx, params = {}) => {
if (!params.name) throw new Error('repo-info requires { name }')
const sync = svc._sync
if (!sync) throw new Error('sync service not active (no swarm)')
const info = sync.info(params.name)
if (!info) throw new Error(`repo "${params.name}" not found`)
return info
}
},
{
name: 'serve-repo',
fn: async (ctx, params = {}) => {
if (ctx.transport !== 'uds' && (!ctx.identity || !ctx.identity.publicKey)) {
throw Object.assign(new Error('authentication required'), { code: 'EAUTH', status: 401 })
}
if (!params.name) throw new Error('serve-repo requires { name }')
const sync = svc._sync
if (!sync) throw new Error('sync service not active (no swarm)')
await sync.serve(params.name)
return { name: params.name, serving: true }
}
},
{
name: 'serve-all-repos',
fn: async (ctx) => {
if (ctx.transport !== 'uds' && (!ctx.identity || !ctx.identity.publicKey)) {
throw Object.assign(new Error('authentication required'), { code: 'EAUTH', status: 401 })
}
const sync = svc._sync
if (!sync) throw new Error('sync service not active (no swarm)')
await sync.serve()
return { serving: sync.serving }
}
},
{
name: 'scan-repos',
fn: async (ctx) => {
if (ctx.transport !== 'uds' && (!ctx.identity || !ctx.identity.publicKey)) {
throw Object.assign(new Error('authentication required'), { code: 'EAUTH', status: 401 })
}
const sync = svc._sync
if (!sync) throw new Error('sync service not active (no swarm)')
const repos = await sync.scan()
return { repos: repos.size }
}
},
{
// Host a new bare repo in one step. Replaces the four-command dance
// (git init --bare / symbolic-ref HEAD / scan / push / scan), which is
// easy to get half-right — a skipped symbolic-ref leaves clones with no
// checked-out branch, and a skipped scan leaves the repo unserved.
//
// `pgh repo create` exists only in the pear-git BUNDLED CLI, not in the
// installed pgh, which is why hosting a repo has meant init-bare by hand.
// This is the daemon half that gives installed pgh the same verb.
name: 'create-repo',
fn: async (ctx, params = {}) => {
if (ctx.transport !== 'uds' && (!ctx.identity || !ctx.identity.publicKey)) {
throw Object.assign(new Error('authentication required'), { code: 'EAUTH', status: 401 })
}
const sync = svc._sync
if (!sync) throw new Error('sync service not active (no swarm)')
const reposDir = svc._reposDir
if (!reposDir) throw new Error('reposDir not resolved on this daemon')
// The name becomes BOTH a filesystem path under reposDir and a
// swarm-announced repo id, so it is validated, not just trusted.
// Rejects separators, traversal, and leading dots.
const name = String(params.name || '')
if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(name) || name.includes('..')) {
throw new Error('invalid repo name: ' + JSON.stringify(name))
}
const branch = String(params.branch || 'main')
if (!/^[a-zA-Z0-9][a-zA-Z0-9._/-]*$/.test(branch)) {
throw new Error('invalid branch name: ' + JSON.stringify(branch))
}
const dir = join(reposDir, name + '.git')
if (existsSync(dir)) throw new Error('repo already exists: ' + name)
await execFile('git', ['init', '--bare', dir])
// Public by default — preserves today's behaviour, where every repo is
// readable. Without this, `pgh repo create` would produce a repo the
// read gate refuses to serve, making hosting a two-step people forget.
// Opt out with `pgh repo visibility <name> private`.
ensureExportMarker(reposDir, name)
// HEAD is a PER-BACKEND symbolic-ref and does NOT replicate P2P, so a
// multi-backend forge still needs this set on each one. Surfaced in the
// result rather than silently assumed.
await execFile('git', ['--git-dir=' + dir, 'symbolic-ref', 'HEAD', 'refs/heads/' + branch])
// scan() registers + installs the post-receive hook; serve() announces.
// Done inline (not left to the 500ms reposWatcher debounce) so the RPC
// returns only once the clone URL actually works. The watcher will also
// fire — both are idempotent, so the redundant pass is free.
await sync.scan()
await sync.serve()
return { name, branch, path: dir, serving: true, headReplicates: false }
}
}
]
}