/**
* startPearGit — boot the pear-git-v3 daemon.
*
* - identity layer: loads/creates an Ed25519 keypair from
* `<storage>/identity.json` (mode 0o600)
* - mounts: orgs, issues, overlay, IdentityRegistry
* - sessions: in-memory bearer tokens with 24h sliding renewal
* - challenges: in-memory replay-protected challenges from anproto
* - handlers: org, issue, overlay, auth, dump-keys, users, etc.
* - SSR pages: overview, orgs, issues, files, login (+ /api/auth/*)
* - auth model: mutating handlers require IdentityRegistry admin/write
* perms; UDS callers bypass (local user is trusted);
* intrinsic owner (envelope root = daemon's hyperident
* root) bypasses the registry lookup
* - TOFU bootstrap: on a writable registry, first HTTP user gets admin;
* subsequent users get read. On a non-writable registry,
* unknown callers get a synthesized "Pending Writer"
* ephemeral user with ['read'] perms.
* - swarm: when opts.swarm is set, joins a hyperswarm topic and
* replicates orgs/issues/IdentityRegistry to peers
*
* Multi-peer setup:
* 1. Daemon A boots with no bootstrap keys → creates fresh orgs/issues
* 2. Operator runs `pgh dump-keys` (or queries the dump-keys handler) on A
* 3. Daemon B boots with opts.mounts = { orgs: { bootstrapKey: Akeys.orgs }, ... }
* and the SAME opts.swarm.topic
* 4. Both daemons join the swarm topic, find each other, replicate
* 5. Device-invite from A to B pre-authorizes B's local key on every shared
* Autobase via ADD_WRITER; B then writes to the shared views as a peer.
*
* Overlay stays local per daemon — hyper-deploy handles cross-peer file
* / release distribution.
*
* Usage (single-daemon):
*
* import { startPearGit } from 'pear-git-v3'
*
* const node = await startPearGit({
* storage: '/var/lib/pear-git',
* uds: '/var/run/pear-git/svc.sock',
* http: { port: 4040 }
* })
*
* Usage (joining an existing deployment):
*
* const node = await startPearGit({
* storage: '/var/lib/pear-git-mirror',
* http: { port: 4041 },
* swarm: { topic: '<32 bytes hex>' },
* mounts: {
* orgs: { bootstrapKey: '<from primary>' },
* issues: { bootstrapKey: '<from primary>' },
* trust: { coreKey: '<from primary>' }
* }
* })
*/
import process from '#node:process'
import { join, dirname } from '#node:path'
import { mkdir, readdir } from '#node:fs/promises'
import fs from '#node:fs'
import os from '#node:os'
import b4a from 'b4a'
import { signWriterVouch } from 'hyper-rooms'
import crypto from 'hypercore-crypto'
import Corestore from 'corestore'
import Hyperbee from 'hyperbee'
import { Service } from 'hyper-svc'
import { HyperOrgs } from 'hyper-orgs'
import { issuesSchema } from 'hyper-issues'
import { IssueRegistry } from './issue-registry.js'
import { migrateStore } from './services/migrate/boot.js'
import { fileVersionStore } from './services/migrate/version-file.js'
import { announceSchemaVersion, checkSchemaGuard } from './services/schema-guard.js'
import { TARGET_VERSION } from './services/migrate/migrations.js'
import { HyperOverlay as HyperOverlayV2 } from 'hyper-overlay'
import { IdentityRegistry } from 'hyper-identity/auth/identity-registry'
import { Challenges } from 'anproto'
import { loadOrCreateIdentity } from './identity.js'
import { SessionStore } from './auth/sessions.js'
import { extractIdentity, extractToken, requireAdmin } from './auth/check.js'
import { canPushGit } from './auth/git-acl.js'
import { publishRefUpdate } from './gip-publish.js'
import { execFile } from './_compat-exec.js'
import { listDiskRefs } from './services/ref-publish-reconciler.js'
import { isValidRepoName } from './repo-name.js'
import { createRepoAccess, ensureExportMarker } from './repo-access.js'
import { RefReconcilerManager } from './services/ref-reconciler-manager.js'
import { VdriveSeeder } from './services/vdrive-seeder.js'
import { orgHandlers } from './handlers/orgs.js'
import { issueHandlers } from './handlers/issues.js'
import { releaseHandlers } from './handlers/releases.js'
import { attachmentHandlers } from './handlers/attachments.js'
import { pghTreeHandlers } from './handlers/pgh-tree.js'
import { ReleaseManager } from './release-manager.js'
import { authHandlers } from './handlers/auth.js'
import { profileHandlers } from './handlers/profile.js'
import { accessTokenHandlers } from './handlers/access-tokens.js'
import { sshKeyHandlers } from './handlers/ssh-keys.js'
import { attachSshResyncTrigger } from './auth/ssh-resync-trigger.js'
import { userHandlers } from './handlers/users.js'
import { startDeployService, deployHandlers } from './services/deploy.js'
import { configureRepoActivityCache, enrichRepos as enrichReposForWarm, invalidateRepoActivity, flushRepoActivityCache } from './services/repo-activity.js'
import { configurePushIndex } from './services/push-index.js'
import { configureRefActivity, flushRefActivity } from './services/repo-ref-activity.js'
import { configureIssueIndex, backfillIssueIndex } from './services/issue-index.js'
import { startSyncService, syncHandlers } from './services/sync.js'
import { RepoEventLog } from './services/repo-event-log.js'
import { foldRetired } from './services/retired-repos.js'
import { maybeAppendCreatedEntry } from './services/repo-created-entry.js'
import { OrgEventLog } from './services/org-event-log.js'
import { ReleaseEventLog } from './services/release-event-log.js'
import { DEFAULT_ROOT as PGH_STORE_ROOT } from 'pgh/store'
import { RepoPuller } from './services/repo-mirror.js'
import { installRpcAuditLog } from './services/rpc-audit-log.js'
import { OrgPuller } from './services/org-mirror.js'
import { RepoFollower } from './services/repo-follower.js'
import { PendingPairings } from './services/pending-pairings.js'
import { FollowRegistry } from './follow-registry.js'
import { FollowsRegistry } from './follows-registry.js'
import { decode as decodeFollowPayload, parseUrl as parseFollowUrl } from './services/follow-payload.js'
import { repoHandlers, registerGitRoutes } from './handlers/repos.js'
import { deviceHandlers } from './handlers/devices.js'
import { followHandlers } from './handlers/follows.js'
import { registerRoutes } from './web/router.js'
/**
* @param {object} opts
* @param {string} opts.storage - daemon state directory. Layout:
* <storage>/identity.json - daemon-owned (mode 0o600)
* <storage>/store/ - corestore-owned (don't put files here)
* @param {string} [opts.uds] - UNIX socket path (optional, omit to disable)
* @param {object} [opts.http] - HTTP options { port, host } (optional)
* @param {object} [opts.swarm] - Swarm options. { topic, swarm? } where
* topic is a 32-byte buffer or 64-char hex string. swarm is an optional
* passthrough to Hyperswarm (e.g. { bootstrap } for testnet).
* @param {object} [opts.mounts] - Bootstrap keys for joining an existing
* deployment. All optional — omit to create fresh state.
* @param {object} [opts.mounts.orgs] - { bootstrapKey: hex }
* @param {object} [opts.mounts.issues] - { bootstrapKey: hex }
* @param {object} [opts.mounts.trust] - { coreKey: hex } (dormant — trust-map retired in F.6)
* @returns {Promise<{ svc, corestore, identity, orgs, issues, overlay, trust, close }>}
*/
export async function startPearGit (opts = {}) {
if (!opts.storage) throw new Error('startPearGit requires opts.storage')
// Phase 4 — mirror daemons boot with a deploymentBundle produced by
// `pgh device invite --mirror` on the primary. Expanding it here keeps
// the call-sites simple (`startPearGit({ storage, deploymentBundle })`)
// while reusing the existing mounts/mirror/swarm plumbing below.
// Anything the caller specifies explicitly still wins.
if (opts.deploymentBundle) opts = expandDeploymentBundle(opts)
// Make sure the top-level storage dir exists. Corestore will create its
// own subdir, but we need the parent first so we can put identity.json
// alongside it. Corestore wipes its OWN directory on ready(), which is
// why daemon state lives one level up.
await mkdir(opts.storage, { recursive: true })
// Load the persisted /repos landing-page enrich cache before serving so
// the first request (and the background pre-warm below) reuse cross-restart
// results instead of re-running git against every repo. Rebuildable derived
// data — safe to lose. See lib/services/repo-activity.js.
await configureRepoActivityCache(opts.storage).catch(() => {})
// Load the persisted push-attribution index (public profile "repos pushed").
// Rebuildable derived data — safe to lose. See lib/services/push-index.js.
await configurePushIndex(opts.storage).catch(() => {})
// Load the persisted replicated last-ref-update index — the /repos ordering
// key. The boot ensureFor walk below repopulates it from the trackers, but
// not before the first visitor arrives, and without a value the page falls
// back to local mtimes, which is the cross-backend disagreement this exists
// to fix. Rebuildable derived data — safe to lose. See
// lib/services/repo-ref-activity.js.
await configureRefActivity(opts.storage).catch(() => {})
// Load the persisted issue-author index (public profile "recent issues").
// Rebuildable derived data — safe to lose. See lib/services/issue-index.js.
await configureIssueIndex(opts.storage).catch(() => {})
// Load (or create on first boot) the daemon's persistent service identity.
// Lives at <storage>/identity.json with mode 0o600.
//
// PRIMARY (no deploymentBundle): identity = hyperident-derived (stable
// across storage resets). This is the pubkey peers expect us at:
// - swarm topic for hyper-svc RPC + autobase replication
// - serverKey for git punch:// URLs
// - cross-machine peer discovery
//
// MIRROR (deploymentBundle present): identity = linked-device.json
// (newDeviceKp received from the device-invite ceremony). This must be
// the keypair that the source pre-computed and addWritered, otherwise
// mirror's IdentityRegistry autobase writer key won't match what the
// primary registered → mirror stays "Not writable" on every mutating
// RPC. See test/device-invite-fix-proof.test.js for the mechanical
// proof (PASS: with keyPair=newDeviceKp, mirror.local.key matches
// source's pre-computed key; CONTROL: without keyPair, mirror is not
// writable — reproduces today's prod bug exactly).
// (Cross-machine discovery is unaffected: bundle.swarm.topic and
// bundle.sourceServerKey are explicit in the bundle, not derived from
// the mirror's identity.)
const identity = await loadOrCreateIdentity(opts.storage, {
useLinkedDevice: !!opts.deploymentBundle
})
// The autobase writer keypair is NOT pinned here. Pinning a writer
// keypair from outside (e.g. ~/.pgh/linked-device.json — see commit
// 589410b) creates a self-invite trap: the same daemon issues the
// invite AND consumes it locally, so the new writer never gets a
// length>0 head from a remote peer, never graduates from
// pendingIndexers to indexers, ack timer fires uselessly, consensus
// stalls. Canonical pattern: corestore auto-derives the local writer
// per namespace; `pgh device invite/join` from a different machine
// attests the auto-derived key as an indexer. See
// memory/multi-writer-autobase-pattern.md rule 1.
// Corestore gets its OWN subdirectory. It will create CORESTORE and db/
// inside <storage>/store/. The orchestrator must NOT put files inside
// that subdir — Corestore considers it private and may overwrite.
// PEAR_GIT_ALLOW_BACKUP lets the daemon open a COPIED/backed-up store —
// hypercore-storage's device-file guard otherwise rejects a store that
// was moved to a new inode ("was moved unsafely"). Only for running on a
// backup/migration copy; never two live daemons on the same copy.
const corestoreOpts = (opts.allowBackup || process.env.PEAR_GIT_ALLOW_BACKUP)
? { allowBackup: true } : {}
const corestore = new Corestore(join(opts.storage, 'store'), corestoreOpts)
await corestore.ready()
// Mount the domain modules onto the same corestore. Each picks its own
// namespace so the underlying cores don't collide. Bootstrap keys (if
// provided) make the modules join an existing deployment instead of
// creating fresh state.
const mountOpts = opts.mounts || {}
// Canonical upstream pattern: orgs / issue trackers let corestore
// auto-generate the local writer per-namespace ({ name: 'local' }).
// Passing a shared keyPair would deadlock — two Autobases with the
// same keyPair derive identical manifests, corestore returns the
// same Hypercore, and hypercore's `exclusive: true` mutex locks the
// second opener forever. See autobase/lib/boot.js (exclusive:true
// default), corestore `_auth()` (manifest ignores namespace),
// hypercore `lockExclusive()` (mutex).
//
// Identity autobase KEEPS keyPair below: it is the only Autobase
// using identity.keyPair → no collision. The benefit is that its
// root is deterministically derivable from sourceServerKey in the
// deployment bundle (no separate identity bootstrap key to ship).
//
// Mirror write-back for orgs / issues is follow-up: mirrors post
// their auto-gen localKeys to primary via RPC after boot; primary
// appends addWriter ops in each autobase.
// Per-org-autobase model (issue #35): HyperOrgs no longer mounts a
// single shared orgs autobase by bootstrapKey. It owns N per-org
// autobases, re-opened from its local index on ready(). Peer mode =
// joinOrg(invite); mirror mode = mountOrg() per primary.listOrgKeys().
// Linked-core org tracker (#42): compose issuesSchema onto every org
// Room so org issues live IN the org autobase and replicate WITH it.
// A peer that joins only the org record (peer mode) gets its issues
// for free — no separate issues:org:<name> autobase to convey.
const orgs = new HyperOrgs(corestore, { namespace: 'orgs', extraSchemas: [issuesSchema] })
// Mirrors run in strict-bootstrap mode: a tracker is never lazily
// created against an auto-derived local root, only against a known
// bootstrap key. The bootstrap arrives via deployment bundle on boot
// or RepoEventLog/OrgEventLog post-boot. Without strict mode a mirror
// could mint a sibling autobase before the canonical key replicates,
// forking issue history per-machine (the divergence repaired in
// 0.1.57). Primary stays permissive — it OWNS canonical roots.
const issueRegistry = new IssueRegistry(corestore, {
repoBootstrapKeys: mountOpts.issueRepos || {},
orgBootstrapKeys: mountOpts.issueOrgs || {},
indexPath: join(opts.storage, 'issue-registry-index.json'),
requireBootstrap: !!(opts.origin || opts.mirror),
// Linked-core (#42): org issues live in the org's own Room. When the
// org Room is mounted, getForOrg wraps it instead of opening a
// separate issues:org:<name> autobase.
orgRoomProvider: (name) => orgs.getOrgRoom(name),
// Self-heal (#3, Increment 2): when a repo tracker's autobase errors,
// IssueRegistry reopens it under a fresh namespace; re-attach the
// RefReconciler to the new tracker. Lazy — the manager is created below.
onResync: (repoName) => (svc._refReconcilers ? svc._refReconcilers.resyncFor(repoName) : null)
})
// IdentityRegistry — hyper-identity's Autobase-backed user/identity store.
// Source of truth for user profiles (name + picture), perms, and linked
// identities (browser, SSH key, etc.). Replaces the legacy trust-map.
// Trust-map's on-disk hypercore still exists as historical data — we
// just stop reading/writing it. See spec/identity-ops.md for the
// migration + policy rationale.
const identityDbCore = corestore.namespace('identity-db').get({ name: 'db' })
await identityDbCore.ready()
const identityDb = new Hyperbee(identityDbCore, {
keyEncoding: 'utf-8',
valueEncoding: 'json'
})
await identityDb.ready()
// Identity autobase — root-key + writer-keypair policy:
//
// PRIMARY: No keyPair, no bootstrapKey. Corestore auto-derives the
// local writer per namespace (`{ name: 'local' }`).
// Autobase mints its root from that local writer on first
// boot; subsequent boots find the existing root in the
// corestore and reuse it. The root key gets baked into
// deployment bundles shipped to mirrors.
//
// MIRROR: bootstrapKey = primary's autobase ROOT key (from
// bundle.identity.bootstrapKey, set on primary as
// `b4a.toString(identityRegistry.base.key, 'hex')`).
//
// keyPair = identity.{publicKey,secretKey} (which on a
// mirror = newDeviceKp from linked-device.json — the
// keypair received during the device-invite ceremony).
// This makes mirror.base.local.key match
// Hypercore.key({signers:[{publicKey: newDeviceKp.pub}]})
// which is exactly what DeviceLinker.createInvite
// pre-computes and addWriters as `autobaseLocalKey`. So
// after replication picks up the addWriter op, mirror is
// writable. Without this keyPair, mirror's local writer is
// corestore-auto-derived and unrelated to newDeviceKp; the
// addWriter goes nowhere; mirror stays not-writable. See
// test/device-invite-fix-proof.test.js for the mechanical
// proof + control case.
//
// No keyPair-collision concern: identity is the only
// autobase here using a pinned keyPair (orgs/issues/etc
// stay corestore-auto-derived per the keet pattern), so
// there's no manifest collision across mounts.
const mirrorBootstrapKey =
(mountOpts.identity && mountOpts.identity.bootstrapKey) || null
const identityRegistry = new IdentityRegistry(
corestore.namespace('identity'),
{
db: identityDb,
bootstrapKey: mirrorBootstrapKey,
keyPair: mirrorBootstrapKey
? { publicKey: identity.publicKey, secretKey: identity.secretKey }
: undefined,
// Mirror only: adopt the origin's `_system` core instead of deriving one.
// Null on the origin (it IS the system) and on any bundle predating the
// field, both of which behave as before.
fastForwardTo: (mountOpts.identity && mountOpts.identity.fastForwardTo) || null
}
)
await orgs.ready()
// Schema migration (#52): version-gated, in-place, BEFORE serving. On a
// store written by the old single-orgs daemon it upgrades orgs → per-org
// (#37) and replays each org's separate issue tracker into its room (#42),
// writing into the v2 `orgs` just readied above. No-op once the store
// records TARGET_VERSION (a fresh store is stamped at target on first boot).
// Fail-stop: a throw aborts boot rather than serving half-migrated data.
// NOTE: mirror propagation of NEWLY-migrated orgs (orgEventLog announce) is
// a cross-machine follow-up (#52 Phase 4.4) — the origin serves them locally.
{
const versionStore = fileVersionStore(opts.storage)
// Mirrors (booted from an origin's deploymentBundle / with a mounted
// identity bootstrapKey) must NOT self-migrate — they mount the origin's
// migrated per-org autobases via OrgPuller. Only the ORIGIN migrates.
const isMirror = !!opts.deploymentBundle || !!mirrorBootstrapKey
const migration = await migrateStore({
corestore,
newOrgs: orgs,
getVersion: versionStore.getVersion,
setVersion: versionStore.setVersion,
isMirror,
log: (m) => console.log('[migrate] ' + m)
})
if (!migration.skipped && migration.result && migration.result.ran && migration.result.ran.length) {
console.log('[migrate] schema v' + migration.startVersion + ' → v' + migration.version)
}
}
// Pre-mint issue trackers for every org on boot (#123). An org learned via
// orgs-autobase replication had no tracker until someone hit its page; lazy
// creation produced sibling autobases per peer, so issues never converged.
// Walk orgs.list() and wrap each org's room so its tracker key lands in the
// index → dump-keys / device-invite bundles / resync-from-origin include it.
// Under linked-core (#42) the tracker IS the org room (already mounted by
// orgs.ready()). Fail-soft per org so one bad org can't abort boot.
for (const org of await orgs.list()) {
try {
await issueRegistry.getForOrg(org.name)
} catch (err) {
console.log('[boot] pre-mint org tracker ' + org.name + ' failed: ' + err.message)
}
}
// issueRegistry has no top-level ready() — each tracker readies on first access
await identityRegistry.ready()
// Repo-event log — one Hypercore per daemon. The writing daemon
// (primary) appends a "created" or "updated" entry every time a repo
// starts being served or receives a push. Mirror daemons open this
// same core by key (see opts.mirror.repoEventLogKey below) and
// replicate it via the shared swarm → a mirror gets a live,
// persistent, delta-replicated feed of what the primary serves.
//
// GC/compaction deferred — see plan O-11. At v3 scale (10s-100s of
// repos, a few pushes per day) the log grows at ~100 B × events/day,
// negligible for years.
const repoEventLog = new RepoEventLog({ corestore })
await repoEventLog.ready()
// Tombstones (#135). Folded from the local log at boot; the RepoPuller keeps
// folding replicated entries live. Derived state — nothing persisted, so it
// rebuilds identically every boot. Shared by all four gate sites so they
// agree on what is retired.
//
// ⚠ This set only makes the daemon IGNORE a repo (stop listing / serving /
// pulling / re-creating). It never deletes bytes — removing a bare repo is
// a manual operator step on each peer, and this daemon must never gain the
// capability. See services/retired-repos.js and test/retired-repos.test.js.
const retiredRepos = await foldRetired(repoEventLog)
if (retiredRepos.size > 0) {
console.error('[retired] ' + retiredRepos.size + ' repo(s) tombstoned: ' + retiredRepos.names().join(', '))
}
// Sister log for orgs. Same architecture as RepoEventLog: single-
// writer Hypercore on primary, read-only replicas on mirrors. Carries
// each org's RECORD autobase key (orgBootstrapKey) so mirrors mountOrg()
// it. Org issues live IN that record autobase (linked-core #42), so they
// replicate with the org — no separate per-org issue tracker key to
// convey (issueBootstrapKey stays null on org events).
const orgEventLog = new OrgEventLog({ corestore })
await orgEventLog.ready()
// Sister log for RELEASES. Same architecture again: single-writer Hypercore
// here, read-only replica on each mirror. ReleaseManager appends one entry per
// index mutation (publish/promote/unpin/delete) and a mirror reconciles the
// named repo on each entry. This is what let the ReleasePuller's 15-minute
// polling interval be DELETED rather than tuned — the origin always had the
// event (every mutation already rewrites _index.json), it just had no way to
// tell anyone.
//
// Deliberately NOT a new op code on repoEventLog: RepoPuller._enqueue treats
// every non-OP_RETIRED entry as "git-pull this repo", so release ops appended
// there would make every already-deployed mirror spuriously git-fetch.
//
// Carries no bytes and is never truncated (~60 B/entry, GC deferred as with
// its two sisters). Entries are wake-ups, so replay is harmless: the mirror
// always re-reads the origin's authoritative _index.json to decide anything.
const releaseEventLog = new ReleaseEventLog({ corestore })
await releaseEventLog.ready()
// In-band schema guard (#52 Phase 4.4, Keet's interrupt pattern): if the
// replicated org log already declares a schema version NEWER than this code
// supports, this node is running stale code — fail-stop rather than serve
// stale data. Best-effort at boot (a mirror's marker may not have replicated
// yet — OrgPuller's live check catches markers that arrive while running).
{
const guard = await checkSchemaGuard(orgEventLog, TARGET_VERSION)
if (guard.behind) {
throw new Error('[schema-guard] this daemon supports schema v' + guard.supported +
' but the replicated org log declares v' + guard.declared +
' — refusing to serve stale data; upgrade this node.')
}
}
// Auth state — in-memory, evaporates on restart
const sessions = new SessionStore({ ttl: 24 * 60 * 60 * 1000 })
const challenges = new Challenges({ ttl: 5 * 60 * 1000 })
// ctxFactory: hyper-svc calls this for every /rpc/* HTTP request so we
// can populate ctx.identity from the session cookie / Authorization header.
// Without this, /rpc/<mutating-method> over HTTP would always fail because
// requireWriter sees no identity.
//
// F.7.5 — also captures the request's origin-level metadata (Host header,
// TCP peer address, User-Agent) so auth-verify can stamp these onto a
// newly-created Identity record. Host + UA are client-controlled — display
// only. remoteAddress is server-observed; safe for trust heuristics.
const httpCtxFactory = (req) => {
const origin = {
host: (req.headers && req.headers.host) || null,
userAgent: (req.headers && req.headers['user-agent']) || null,
// req.socket is null by the time this runs on the /rpc/ path — bare-http1
// clears it when the request message ends, and hyper-svc builds its ctx
// after awaiting the body. The router's request listener stamps
// _peerAddress synchronously at emit time, before that happens.
remoteAddress: (req.socket && req.socket.remoteAddress) || req._peerAddress || null
}
const session = extractIdentity(req, sessions)
if (!session) return { identity: null, origin }
return {
identity: { publicKey: session.publicKey, _token: extractToken(req) },
origin
}
}
// Default swarm topic: hash(identityPublicKey + "pear-git-v3"). Every
// daemon deterministically joins a topic derived from its own identity.
// Peers connect by knowing the server's identity key. Override with
// opts.swarm.topic if you need a custom topic.
// Pass opts.swarm = false to disable swarm entirely (tests do this).
//
// Captured as a named flag because `opts` is SHADOWED inside several inner
// functions that take their own `opts` parameter (maybeBootReversePuller is
// one). Writing `opts.swarm === false` in those bodies silently reads the
// wrong object. Everything that touches the network must consult this.
const networkDisabled = opts.swarm === false
let swarmOpt = null
if (opts.swarm !== false) {
swarmOpt = opts.swarm || {}
if (!swarmOpt.topic) {
swarmOpt.topic = crypto.hash(b4a.concat([
identity.publicKey,
b4a.from('pear-git-v3')
]))
}
}
const svc = new Service({
name: 'pear-git-v3',
corestore,
uds: opts.uds || null,
http: opts.http ? { ...opts.http, prefix: opts.prefix || '', ctxFactory: httpCtxFactory } : null,
swarm: swarmOpt,
// Anonymous-dispatch gate (org #12). hyper-svc dispatched every registered
// method to anyone, over HTTP and swarm-rpc, with no allowlist — and this
// daemon is internet-reachable via https://7rnx.net/git/rpc/, so the whole
// RPC surface was world-callable. Default-deny with per-handler opt-in at
// registration; the opted-in set is ANONYMOUS_METHODS below.
//
// Denials need no separate audit wiring: installRpcAuditLog wraps dispatch,
// so an EDENIED throw is already recorded like any other failed call.
anonymousPolicy: opts.anonymousPolicy || 'deny',
// Identity semantics live here, not in hyper-svc. FOUR ctx shapes reach
// dispatch and THREE are non-null, so `if (!ctx.identity) deny` would wave
// every swarm peer through and misread UDS:
// uds { trust:'local' } local operator
// http+sess { publicKey } a real user
// http anon null not
// swarm-rpc { type:'swarm', trust:'network' } NOT a user
// Envelope-authenticated peer calls (pgh peer call → POST /api/rpc) are
// verified by the router BEFORE dispatch and arrive carrying a publicKey,
// so they are not anonymous and are unaffected.
isAnonymous: (ctx) => {
if (!ctx) return true
if (ctx.transport === 'uds') return false
const id = ctx.identity
if (!id) return true
if (id.trust === 'network') return true // swarm peer, not a user
return !id.publicKey
}
})
// RPC audit log — an append-only record of every dispatch, on every
// transport. Installed IMMEDIATELY after the Service is built so no call
// can slip in unrecorded during the rest of boot. Until this existed the
// control plane was entirely unlogged: /rpc/<method> and /api/rpc accepted
// mutating calls (join-repo, create-access-token, consume-link-token,
// ref-update-observed) and left no daemon-side trace, so answering "who
// called what" meant correlating an upstream proxy log against
// side-effects. See services/rpc-audit-log.js. Fails open — a logging
// problem can never fail an RPC. Registry is passed as a getter because
// identityRegistry is assigned a few lines below this.
installRpcAuditLog(svc, {
path: join(opts.storage, 'rpc-audit.jsonl'),
registry: () => svc.identityRegistry
})
svc.mount('orgs', orgs)
svc.issueRegistry = issueRegistry
// Persisted outstanding blind-pairing invites — re-registered on boot so an
// already-shared invite survives a daemon restart.
svc._pendingPairings = new PendingPairings(join(opts.storage, 'pending-pairings.json'))
// Mount identityRegistry so hyper-svc's swarm transport calls
// `identityRegistry.base.replicate(conn)` on every incoming peer.
// Without this, the users/devices/revocations autobase replicates
// only via `corestore.replicate` — unreliable for writer cores. A
// mirror daemon boots with the primary's bootstrapKey but then sees
// no updates because the base-level replication hook is missing.
svc.mount('identity', identityRegistry)
svc.identityRegistry = identityRegistry
// Expose identity on the service so handlers and SSR pages can read it.
// Only the public half — secretKey stays in the closure.
svc.identity = {
publicKey: identity.publicKey,
publicKeyHex: identity.publicKeyHex,
rootPubkeyHex: identity.rootPubkeyHex || null,
createdAt: identity.createdAt
}
// Stash auth-related state on the service so handlers and the router
// can reach it without ad-hoc closure plumbing.
svc.sessions = sessions
svc.challenges = challenges
// URL prefix + external URL — used by the router (strip prefix on
// incoming requests) and SSR pages (generate correct links).
svc._prefix = opts.prefix || ''
svc._externalUrl = opts.externalUrl || null
// Per-room swarm rendezvous (Keet activateRoom — issue #34). Each org
// and each repo is its own autobase ("room"); to let a peer find the
// owner for a SPECIFIC room without sharing the daemon's identity
// topic, we join that room's discoveryKey on the SHARED swarm. One
// Hyperswarm, many topics — replication still rides the single
// connection handler (corestore.replicate + mount.replicate). This is
// how peer-mode scope works: a peer joins only the room topics it was
// invited to, so other orgs/repos are unreachable by construction.
// Idempotent + guarded (no-op when swarm is disabled, e.g. tests).
svc.activateRoom = (discoveryKey, opts = {}) => {
if (!discoveryKey) return
if (!svc.swarm || typeof svc.swarm.joinTopic !== 'function') return
try {
svc.swarm.joinTopic(discoveryKey, { server: true, client: true, ...opts })
} catch (err) {
// eslint-disable-next-line no-console
console.error('[pear-git-v3] activateRoom failed:', err && err.message)
}
}
// Wire a forceResync side-store tracker into swarm replication. Its base lives
// in a dedicated corestore OUTSIDE the main corestore.replicate, so it needs:
// (1) mount → hyper-svc replicates tracker.base on every FUTURE connection;
// (2) replicate on EXISTING connections; (3) join its room topic. Shared by the
// repo-resync RPC and the boot re-attach of persisted side stores (idempotent).
svc.wireSideStoreTracker = (repoName, tracker, gen) => {
if (!tracker || !tracker.base) return
const mountName = 'resync:' + repoName + ':' + (gen || 0)
try { if (!svc.hasMount(mountName)) svc.mount(mountName, tracker) } catch {}
try {
const conns = (svc.swarm && svc.swarm.connections) ? svc.swarm.connections : []
for (const conn of conns) { try { tracker.base.replicate(conn) } catch {} }
} catch {}
try { if (tracker.discoveryKey) svc.activateRoom(tracker.discoveryKey) } catch {}
}
// A vdrive lives in its OWN corestore (<releases>/<repo>/.vdrive), outside the
// main corestore.replicate — same shape as a resync side-store, so it needs the
// same three steps: mount (hyper-svc replicates it on FUTURE connections, via
// instance.replicate), replicate on the EXISTING ones, and join its topic.
// server-only: an origin has every byte and needs nothing from peers.
// hyper-svc has no unmount, and a re-announce opens a NEW corestore — so mount
// a stable per-repo holder once and swap the store inside it. Otherwise the
// old, closed store stays mounted and hyper-svc replicates it on every
// subsequent connection.
svc._vdriveMounts = new Map()
svc.wireVdrive = (repo, opened) => {
if (!opened || !opened.store) return
const mountName = 'vdrive:' + repo
let holder = svc._vdriveMounts.get(repo)
if (!holder) {
holder = {
store: null,
replicate (conn) { if (holder.store) holder.store.replicate(conn) }
}
svc._vdriveMounts.set(repo, holder)
try { if (!svc.hasMount(mountName)) svc.mount(mountName, holder) } catch {}
}
holder.store = opened.store
try {
const conns = (svc.swarm && svc.swarm.connections) ? svc.swarm.connections : []
for (const conn of conns) { try { opened.store.replicate(conn) } catch {} }
} catch {}
try { svc.activateRoom(opened.discoveryKey, { server: true, client: false }) } catch {}
}
// Detach without unmounting: the holder stays, its store goes null, so a
// connection arriving before the next announce simply replicates nothing.
svc.unwireVdrive = (repo) => {
const holder = svc._vdriveMounts.get(repo)
if (holder) holder.store = null
}
// Ring the doorbell and KEEP ringing until admitted (writable) or a deadline.
// A single optimistic request-writer op can be rolled back if no indexer acks
// it in that exact drain window (autobase golden rule) — keet re-rings the same
// way (_signal.wait). Background + non-blocking. Each re-ring re-anchors the
// optimistic tip so an intermittently-connected owner can admit it; once
// writable, the extra ops are idempotent no-ops. Returns true if started.
svc.ringDoorbellUntilAdmitted = (repoName, tracker, voucherKeyPair) => {
if (!voucherKeyPair || !tracker || !tracker.room || typeof tracker.room.requestWriter !== 'function') return false
const newKey = tracker.localKeyHex
let voucherSig, voucherSignerKey
try {
voucherSig = signWriterVouch(newKey, voucherKeyPair.secretKey)
voucherSignerKey = b4a.toString(voucherKeyPair.publicKey, 'hex')
} catch { return false }
const label = (svc.identity && svc.identity.publicKeyHex ? svc.identity.publicKeyHex.slice(0, 16) : 'peer') + ':rejoin'
const MAX_ATTEMPTS = 90 // ~7.5 min at 5s — long enough for DHT connect + ack
let attempts = 0
const tick = async () => {
try {
if (tracker.base && tracker.base.writable) return // admitted — done
// Stop if this side-store tracker was superseded by a newer resync.
if (svc.issueRegistry && svc.issueRegistry._repos.get(repoName) !== tracker) return
if (attempts++ >= MAX_ATTEMPTS) {
// eslint-disable-next-line no-console
console.error('[pear-git-v3] doorbell for ' + repoName + ' gave up after ' + MAX_ATTEMPTS + ' rings (manual repo-add-writer fallback)')
return
}
await tracker.room.requestWriter({ newWriterKey: newKey, voucherSignerKey, voucherSig, userId: label })
try { await tracker.base.update() } catch {}
if (tracker.base && tracker.base.writable) return
} catch {}
setTimeout(() => { tick().catch(() => {}) }, 5000)
}
tick().catch(() => {})
return true
}
// Bootstrap artifacts dir — where install.sh + <app>-bootstrap-bare-*.tar.gz
// live. Served at /bootstrap/* by lib/handlers/bootstrap.js, so a fresh
// machine can `curl <host>/bootstrap/install.sh | sh`.
//
// #66: this dir is OWNED + POPULATED by pgh-sidecar, not the forge. The
// sidecar installs the pgh.bootstrap app trees (pgh + pgh-sidecar) and tars
// them into <store>/bootstrap on install/OTA (see pgh/bootstrap-artifacts).
// The forge is a pure static server over that dir — no reconcile, no network.
// Default: <store>/bootstrap. Override via opts.bootstrapDir or
// PEAR_GIT_BOOTSTRAP_DIR env (tests inject a dir directly). Absent → 503.
svc._bootstrapDir = opts.bootstrapDir ||
process.env.PEAR_GIT_BOOTSTRAP_DIR ||
join(PGH_STORE_ROOT, 'bootstrap')
// Attachment spool — persistent home for bytes uploaded via /api/upload.
// hyper-overlay's putFile stores chunks as type:'local' pointers into this
// dir; deleting/moving these files invalidates the chunk-registry. Files
// are content-addressed by oid so re-uploads of identical content are
// deduplicated naturally. No GC yet — future work walks file: bee entries
// and unlinks unreferenced oids.
svc._attachmentSpoolDir = join(opts.storage, 'attachments-spool')
await mkdir(svc._attachmentSpoolDir, { recursive: true })
// V2 overlay — content-addressed cross-peer file serving. The canonical
// bytes stay on disk in the spool dir; V2 holds only metadata + chunk maps
// (no blob storage → no Hyperdrive-style balloon). Registered attachments
// become fetchable by content hash from any peer. Its `attachProtocol(mux)`
// is auto-called per swarm connection by hyper-svc (swarm.js). This is the
// cross-peer path for /files/raw; V1 `overlay` still backs local reads.
const overlayV2 = new HyperOverlayV2(corestore, {
namespace: 'overlay-v2',
syncBaseDirs: [svc._attachmentSpoolDir],
destDir: join(opts.storage, 'attachments-fetch')
})
await overlayV2.ready()
svc.mount('overlayV2', overlayV2)
// gip-overlay consumer: per-repo bare-repo reconcilers. Receiving nodes
// materialize their bare repos from the autobase ref-view, fetching objects
// via the overlay bundle chain. reposDir is resolved here (independent of
// the sync service — gip replaces the RepoPuller read path) and set on svc
// so the producer (ref-update-observed) and the post-receive hook share it.
const gipReposDir = opts.reposDir || join(opts.storage, 'git-repos')
svc._reposDir = gipReposDir
// Read-access choke point (see lib/repo-access.js). ONE instance, shared by
// every repo-serving surface, so `pgh repo visibility` can move them all at
// once instead of promising something the code cannot deliver.
//
// Starts in OBSERVE mode: it computes the decision and reports `wouldDeny`
// without acting on it, because enforcing immediately would un-serve every
// repo that predates the marker. Flip with PEAR_GIT_READ_GATE=enforce once a
// backfill has written the marker fleet-wide.
svc.repoAccess = createRepoAccess({
reposDir: gipReposDir,
isRetired: (name) => !!(svc._retiredRepos && svc._retiredRepos.has(name)),
isValidName: isValidRepoName,
enforce: process.env.PEAR_GIT_READ_GATE === 'enforce',
// Identity-based read grants. Consulted ONLY by repoAccess.async, and
// ONLY for a repo that is not marked public — so this never runs on the
// hot path for public repos.
//
// Policy lives in hyper-orgs, which SYSTEM.md names as THE access control
// layer ("do not duplicate"); repo-access is just the enforcement choke
// point. `read` became expressible there only recently — before that,
// checkAccess({requiredPermission:'read'}) returned false unconditionally
// because the ladder started at `write`.
//
// Returns strictly true/false and never throws — repoAccess.async treats a
// rejection as DENY, and an unreachable registry must not open a private
// repo.
isReader: async (repoName, ctx) => {
// Two ways in. Surfaces that own credential extraction (HTTP smart-git,
// SSH) resolve through auth/git-acl.js resolveGitReader and pass a
// ctx.userId — that path also applies the REVOCATION gate, which a raw
// pubkey lookup does not. Surfaces holding only a session pubkey may pass
// ctx.identity instead.
let user = null
if (ctx && ctx.userId) {
user = await identityRegistry.getUser(ctx.userId)
} else {
const pk = ctx && ctx.identity ? ctx.identity.publicKey : null
if (!pk) return false // anonymous callers only ever see public repos
user = await identityRegistry.getUserByIdentity(pk)
}
if (!user || !user.userId) return false
// A merged-away or deactivated account keeps its grants on paper; neither
// should still open a private repo.
if (user.mergedInto) return false
if (await identityRegistry.isDeactivated(user.userId)) return false
return await orgs.checkAccess({
userId: user.userId,
resourceName: repoName,
requiredPermission: 'read'
}) === true
}
})
const refReconcilers = new RefReconcilerManager({
issueRegistry, overlay: overlayV2, reposDir: gipReposDir,
// Lazy — the sync server is created later; the ensureObjects fallback pulls
// missing git objects over punch:// via it.
getSync: () => svc._sync,
// Lazy too: svc._retiredRepos is folded from the RepoEventLog further down.
// Without this a retired repo whose tracker is still in
// issue-registry-index.json is re-created by `git init --bare` on boot.
isRetired: (name) => !!(svc._retiredRepos && svc._retiredRepos.has(name))
})
svc._refReconcilers = refReconcilers
// "<repo> <ref>" → last disk sha this daemon kicked a CI build for. Guards
// the auto-build trigger in ref-update-observed (org #12): that handler is
// anonymously reachable, so without a dedup a caller could force unbounded
// rebuilds. In-memory by design — a restart re-builds the current sha once,
// which is the pre-existing behaviour, and nothing persistent is at stake.
svc._lastBuiltRef = new Map()
// #66: NO bootstrap reconciler here. <store>/bootstrap is produced by
// pgh-sidecar (it owns the store + installs the bootstrap app trees); the
// forge only serves it statically via handlers/bootstrap.js.
// Restart support: start reconcilers for repos already materialized on disk.
if (fs.existsSync(gipReposDir)) {
;(async () => {
try {
for (const d of fs.readdirSync(gipReposDir)) {
if (!d.endsWith('.git')) continue
try { await refReconcilers.ensureFor(d.replace(/\.git$/, '')) } catch {}
}
} catch {}
})()
}
// Origin-side ref reconcile (Keet "make sure each peer is in the right
// state"). Event-driven publishRefUpdate can MISS a push: the daemon was
// down when it landed, or it was a file:// push whose post-receive hook
// fired into the void. The bare repo then advances on DISK but the autobase
// ref-view — the P2P source of truth mirrors replicate — never moves, so the
// commit never propagates and nothing replays it. On boot, compare each
// writable repo's on-disk refs to what it has announced and re-announce
// anything disk is ahead on. Writer/FF-guarded inside (a no-op where the ref
// is already announced or a non-FF divergence). Origin only: a mirror's disk
// FOLLOWS the view (RefReconciler), so it has no missed pushes to heal.
// Fire-and-forget so it never blocks boot.
if (opts.sync !== false && !opts.deploymentBundle && fs.existsSync(gipReposDir)) {
;(async () => {
try {
if (!svc.identity) return
const { reconcileAllRepoRefs } = await import('./services/ref-publish-reconciler.js')
await reconcileAllRepoRefs({
reposDir: gipReposDir,
issueRegistry,
overlay: overlayV2,
spoolDir: join(opts.storage, 'gip-bundles'),
userId: svc.identity.publicKeyHex,
// Tombstone gate — do not re-adopt a retired repo still on disk.
isRetired: retiredRepos.isRetired,
fs,
log: { info: (m) => console.error(m), warn: (m) => console.error(m) }
})
} catch (err) {
// eslint-disable-next-line no-console
console.error('[ref-reconcile] boot pass failed:', err && err.message)
}
})()
}
// Bootstrap key surface — read by the SSR overview page and the
// dump-keys handler. trustCore is held in the closure (not on the
// TrustMap instance) so we expose its key here for convenience.
// Bootstrap keys for the daemon-level Autobases. Per-repo and per-org
// issue trackers have their own keys, exposed via issueRegistry.keys()
// and the dump-keys handler below.
// Bug fix (2026-04-20): identity used to report base.local.key — the
// LOCAL writer's core key, unique per daemon. Peers joining on that
// value couldn't replicate because Autobase requires the SHARED root
// key (base.key). Mirrors ended up on sibling autobases, never
// becoming writable despite ADD_WRITER on the primary. Fix: report
// base.key (autobase root). For the primary (sole writer), base.key
// === base.local.key so tabertron's bundles stay compatible; for
// mirrors the previously-wrong value is replaced with the real root.
svc.bootstrapKeys = {
// Per-org model (#35): there is no single orgs autobase to bootstrap.
// Mirror mode replicates orgs by following the primary's per-org key
// list (orgs.listOrgKeys()) via the list-org-keys RPC — wired in the
// mirror-follow step. Until then this stays null; the deploymentBundle
// / expandDeploymentBundle paths already guard on b.orgs.bootstrapKey.
orgs: null,
identity: b4a.toString(identityRegistry.base.key, 'hex'),
repoEventLog: repoEventLog.keyHex,
orgEventLog: orgEventLog.keyHex,
// Conveyed through dump-keys (anonymous over swarm) as well as the
// deployment bundle, so a mirror paired BEFORE this log existed can learn
// the key at boot without re-pairing.
releaseEventLog: releaseEventLog.keyHex
}
// Stash the log on svc so the git-receive-pack post-close hook (in
// lib/handlers/repos.js) can append 'updated' entries after a push.
svc._repoEventLog = repoEventLog
svc._releaseEventLog = releaseEventLog
svc._retiredRepos = retiredRepos
// Same story for the org log — handlers/orgs.js create-org appends
// op=0 entries here after orgs().create(). Mirrors watch the log and
// bind tracker keys via OrgPuller.
svc._orgEventLog = orgEventLog
// Generic status handler for any RPC client
svc.handle('status', async (ctx) => {
const userCount = (await identityRegistry.listUsers().catch(() => [])).length
return {
name: svc.name,
transport: ctx.transport,
identity: svc.identity.publicKeyHex,
mounts: svc.listMounts(),
handlers: svc.listHandlers(),
orgs: (await orgs.list()).length,
issues: await issueRegistry.counts(),
users: { total: userCount },
sessions: { active: sessions.size },
swarm: svc.swarm
? { topic: b4a.toString(svc.swarm.topic, 'hex'), connections: svc.swarm.connections }
: null
}
}, { anonymous: true })
// dump-keys: returns the bootstrap keys for joining this daemon's
// deployment. UDS-only because exposing them over HTTP would let any
// reader try to construct a peer. (The keys themselves are public — joining
// as a peer doesn't grant write access — but it's polite to gate them.)
svc.handle('dump-keys', async (ctx) => {
// Block HTTP (anonymous callers). UDS (local operator) and swarm
// (peers that already know our serverKey — linked devices, mostly)
// both allowed. The keys themselves are public — joining as a peer
// doesn't grant write access — and any device we've linked already
// received these same keys in its origin-bundle, so handing them
// out again over swarm is not a new disclosure.
if (ctx.transport === 'http') {
const err = new Error('dump-keys is not available over HTTP')
err.code = 'EAUTH'
err.status = 403
throw err
}
const issueKeys = await issueRegistry.keys()
return {
orgs: { bootstrapKey: svc.bootstrapKeys.orgs },
issues: issueKeys, // { repos: { name: keyHex }, orgs: { name: keyHex } }
// fastForwardTo = OUR `_system` core key, so a resyncing mirror can adopt
// our system state instead of deriving its own (keet ships the same value
// in every room invitation — keet-core/index.js:1991). Fail-soft: null on
// older autobase without getSystemKey(), and consumers treat it as absent.
identity: {
bootstrapKey: svc.bootstrapKeys.identity,
fastForwardTo: (() => {
try {
const b = identityRegistry.base
if (!b || typeof b.getSystemKey !== 'function') return null
const k = b.getSystemKey()
return k ? b4a.toString(k, 'hex') : null
} catch { return null }
})()
},
repoEventLog: { coreKey: svc.bootstrapKeys.repoEventLog },
orgEventLog: { coreKey: svc.bootstrapKeys.orgEventLog },
releaseEventLog: { coreKey: svc.bootstrapKeys.releaseEventLog }
}
}, { anonymous: true }) // swarm peers: resync-from-origin calls this on the origin
// #66 NOTE: the old `bootstrap-register` RPC is gone. The local-tar reconciler
// (above) tars the already-present pgh-store app trees on each forge — there is
// no publish-side artifact to register into overlayV2 anymore.
// ── Anonymous-callable methods (org #12) ────────────────────────────────
//
// Everything NOT listed here requires an authenticated caller (or UDS). The
// set is deliberately tiny and evidence-driven rather than a guess at what
// "looks public": the rpc-audit log on both prod backends shows the only
// methods ever reached anonymously are ref-update-observed (the hook) and
// the two probes used to demonstrate the vulnerability.
//
// ref-update-observed is the load-bearing entry. The post-receive hook is a
// shell script that curls http://127.0.0.1:<port>/rpc/ref-update-observed
// with no credentials, on all 142 served repos — 26 such calls in the audit
// log. Gating it would silently break ref propagation on every push. It is
// safe to expose because 0.1.187 made it read ref state from THIS daemon's
// bare repo instead of the request body, so a forged call is at most a
// redundant, idempotent re-check. Moving the hook to UDS
// (curl --unix-socket) and dropping this entry is the follow-up.
//
// status is a legitimate remote health endpoint and was deliberately left
// open when the debug/repair handlers were gated in 0.1.188.
//
// Getting this set WRONG is visible and cheap: a missing entry surfaces as a
// 403 recorded in rpc-audit.jsonl, not as a silent hole.
//
// The rest of the set is the PUBLIC BROWSING surface. These are not reached
// via /rpc/ — the SSR router and REST routes dispatch them internally with
// the visitor's (anonymous) ctx, so gating them breaks public pages rather
// than closing a hole. All are reads, which matches the policy git-acl.js
// already states for git itself: "reads stay open — anonymous clone/fetch,
// CI, mirror replication, and public browsing must keep working".
//
// list-repos-detailed the /repos landing page (web/pages/repos.js)
// search-all-repos /api/search/repos + /search (autocomplete)
// search-code-all /api/search/code + /search
// search-users /api/search/users
// whoami /api/auth/me — must answer anonymously, since
// "am I logged in?" is exactly what a logged-OUT
// browser asks
//
// Found by enumerating every svc.dispatch() call site under lib/web/ rather
// than by reasoning about which names look public — the first pass missed
// all five, because the audit log only records /rpc/ traffic and these
// arrive through internal dispatch.
//
// The last two are for SWARM PEERS, not for browsers. `resync-from-origin`
// (the mirror-recovery path) opens a Client.connectSwarm to the origin and
// calls dump-keys then list-repos on it — see the handler further down, whose
// own comment says it "requires the origin to allow swarm callers (they do —
// swarm ctx is distinct from http)". A swarm ctx carries trust:'network' and
// is therefore anonymous by this daemon's predicate, so without these two
// entries the ORIGIN refuses its own mirror and recovery breaks.
//
// dump-keys gated by the handler itself to non-HTTP callers, so opting it
// in here grants swarm + UDS only — the HTTP block still stands.
// list-repos becomes readable by an anonymous HTTP caller too. That is the
// cost, and it is small: the /repos page already publishes the
// same list to anonymous visitors via SSR.
//
// Both were missed on the first pass because I checked that `pgh peer call`
// uses HTTP and then assumed nothing else used swarm-rpc. The audit log could
// not have caught it either — it records /rpc/ HTTP, not swarm dispatch.
const ANONYMOUS_METHODS = new Set([
'ref-update-observed',
'status',
'list-repos-detailed',
'search-all-repos',
'search-code-all',
'search-users',
'whoami',
'dump-keys',
'list-repos',
// Read-only view of the pgh release registry, for the SSR Releases tab.
// The registry is public by construction (a replicated Autobase whose key
// ships in every published package.json as `pgh.upgrade`), so gating reads
// adds friction, not secrecy. The handler SELF-GATES the one thing that
// would otherwise leak: an anonymous caller must pass { name }, and that
// name goes through repoAccess, so a private repo's published versions stay
// hidden. Trusted (UDS / writer) callers keep the unfiltered listing.
'tree-registry-list',
// Release discovery for a ReleasePuller. A puller reaches its origin over
// swarm-RPC, and a swarm ctx carries trust:'network' — without this entry
// the origin refuses its own mirror, exactly as it once refused
// resync-from-origin. The handler filters its response by repoAccess for
// anonymous callers, so a private repo's versions and hashes stay hidden.
'release-index'
])
const handleAll = (entries) => {
for (const { name, fn } of entries) {
svc.handle(name, fn, { anonymous: ANONYMOUS_METHODS.has(name) })
}
}
handleAll(orgHandlers(svc))
handleAll(issueHandlers(svc))
handleAll(releaseHandlers(svc))
handleAll(attachmentHandlers(svc))
handleAll(authHandlers(svc))
handleAll(profileHandlers(svc))
handleAll(accessTokenHandlers(svc))
// SSH push path (classic-git-auth Phase 2, same-account / userland model).
// Opt-in via PGV3_SSH_PUSH=1 (or an explicit PGV3_AUTHORIZED_KEYS). When on,
// the daemon manages a marked block in its OWN ~/.ssh/authorized_keys and
// self-installs the wrapper launcher in ~/.local/bin — no separate account,
// no root. Each forced command carries the daemon's own socket + repos dir,
// so the wrapper (running as the daemon's user) is fully self-configuring.
// Without the flag, add-ssh-key still registers keys (punch:// works) but no
// authorized_keys block is written.
const sshPushEnabled = process.env.PGV3_SSH_PUSH === '1' || !!process.env.PGV3_AUTHORIZED_KEYS
let sshAuthorizedKeys = null
let sshWrapperPath = process.env.PGV3_GIT_SHELL || null
if (sshPushEnabled) {
const home = os.homedir()
sshAuthorizedKeys = process.env.PGV3_AUTHORIZED_KEYS || join(home, '.ssh', 'authorized_keys')
if (!sshWrapperPath) {
try {
const { ensureGitShellLauncher } = await import('./auth/ssh-setup.js')
const appDir = dirname(process.argv[1] || join(home, 'pear-git-v3'))
const out = await ensureGitShellLauncher(join(home, '.local', 'bin', 'pgv3-git-shell'), {
runtime: process.execPath,
script: join(appDir, 'bin', 'pgv3-git-shell.js')
})
sshWrapperPath = out.launcherPath
} catch (err) {
// eslint-disable-next-line no-console
console.error('[ssh-keys] launcher install failed:', err && err.message)
}
}
}
handleAll(sshKeyHandlers(svc, {
authorizedKeysPath: sshAuthorizedKeys,
wrapperPath: sshWrapperPath,
sock: opts.uds || null,
reposDir: gipReposDir
}))
handleAll(userHandlers(svc))
handleAll(deployHandlers(svc))
handleAll(syncHandlers(svc))
handleAll(repoHandlers(svc))
handleAll(deviceHandlers(svc))
handleAll(pghTreeHandlers(svc))
handleAll(followHandlers(svc))
// Boot resync: regenerate the SSH authorized_keys managed block from the
// current ssh-key identities (heals after a restart, or keys enrolled while
// the daemon was down). No-op unless PGV3_AUTHORIZED_KEYS is configured.
if (typeof svc._resyncAuthorizedKeys === 'function') {
svc._resyncAuthorizedKeys().then((r) => {
if (r && !r.skipped) console.log('[ssh-keys] authorized_keys synced: ' + r.count + ' key(s)')
}).catch((err) => console.error('[ssh-keys] boot resync failed:', err && err.message))
}
// Replication-triggered resync (#74 Phase 1). The local add/remove handlers
// only resync the daemon that took the write, so a key enrolled on a peer
// (already in the replicated IdentityRegistry view) stayed blind in this
// daemon's authorized_keys until its next boot — the #73 propagation gap.
// attachSshResyncTrigger watches the registry autobase's 'update' (fires on
// local append AND remote replication) and debounces a rebuild. Gated on
// SSH push being configured — resync is a no-op otherwise, no point
// attaching a listener we'd never act on.
if (sshAuthorizedKeys && typeof svc._resyncAuthorizedKeys === 'function') {
svc._sshResyncDetach = attachSshResyncTrigger({
base: identityRegistry._base,
resync: () => svc._resyncAuthorizedKeys(),
onSynced: (r) => {
if (r && !r.skipped) console.log('[ssh-keys] authorized_keys resynced after update: ' + r.count + ' key(s)')
},
onError: (err) => console.error('[ssh-keys] update resync failed:', err && err.message)
})
}
// Re-establish the grouped-repos watcher for EVERY org on boot — both
// orgs we joined and orgs we own. orgHandlers() set svc._autoMountOrgRepos
// above; the join-org handler only wires the watcher for orgs joined this
// session, so without this an owner that restarts (or that created the org
// before any contributor added repos) never auto-pulls grouped repos. Each
// call is idempotent (mounted Set dedups) and fail-soft per org.
if (typeof svc._autoMountOrgRepos === 'function') {
for (const org of await orgs.list()) {
svc._autoMountOrgRepos(org.name).catch(() => {})
}
}
// SSR routes are app-owned. hyper-svc gives us http.Server; we add routes.
if (opts.http) {
registerRoutes(svc)
registerGitRoutes(svc)
}
await svc.listen()
// Child services — started AFTER the svc is listening because they need
// the swarm to be active. Only started when swarm is enabled.
let deployServer = null
let syncServer = null
// Child services require swarm to be useful. Individually optional via
// opts.deploy and opts.sync (default: true when swarm is enabled).
if (svc.swarm && opts.deploy !== false) {
const releaseDir = opts.releaseDir || join(opts.storage, 'releases')
try {
deployServer = await startDeployService({
releaseDir,
swarm: svc.swarm.swarm,
corestore: corestore.namespace('deploy'),
identity
})
svc._deploy = deployServer
// ReleaseManager writes per-repo/version/channel structure *into*
// the same release dir the DeployServer watches — publishing a
// release triggers hyper-deploy.rescan() so peers see changes.
// overlay: registers each artifact by content hash on every index
// rebuild, so a peer can actually FETCH what _index.json advertises.
// Without it the index names files no one can pull (§10 of the
// release-artifacts spec).
svc._releases = new ReleaseManager({
releaseDir,
deploy: deployServer,
overlay: overlayV2,
eventLog: releaseEventLog
})
// Re-point the overlay's serve maps at our release files. Without this a
// restarted daemon serves NOTHING it did not publish since boot: those maps
// are in-memory, and _writeIndex only (re)registers NEW-or-CHANGED entries,
// so an artifact whose hash has not moved is never re-registered. Measured
// 2026-08-08: tabertron held favela-android's APK on disk and advertised its
// hash, while stortron got "no holder" for it AND for a 436-byte manifest.
// Cheap (one index scan, no reads) and non-fatal.
try {
const seeded = await svc._releases.seedOverlayServeMap()
if (seeded > 0) console.log('[releases] overlay serve map seeded: ' + seeded + ' artifact(s)')
} catch (err) {
console.error('[releases] overlay serve-map seed failed:', err && err.message)
}
// Track in-flight fire-and-forget builds (auto-triggered by push)
// so graceful shutdown can await them instead of orphaning handles.
svc._pendingBuilds = new Set()
} catch (err) {
// eslint-disable-next-line no-console
console.error('[pear-git-v3] deploy service failed to start:', err.message)
}
} else {
// Without swarm we still want the RM so CLI / tests can publish
// locally. It just skips the rescan() callback when deploy is absent.
const releaseDir = opts.releaseDir || join(opts.storage, 'releases')
svc._releases = new ReleaseManager({ releaseDir, deploy: null, eventLog: releaseEventLog })
svc._pendingBuilds = new Set()
}
// vdrive seeder — announce every repo drive that CI has published, so the
// pear:// links the build log prints actually resolve. Inert when no repo has
// opted in (`vdrive:` in ci.yaml): the release dirs simply contain no .vdrive.
// Durable root for deriving each repo's vdrive key. corestore is NOT a mounted
// module, so this cannot be reached via svc.modules — expose it explicitly, the
// same way _sync/_releases/_vdriveSeeder are.
svc._vdriveRootKey = corestore.primaryKey || null
svc._vdriveSeeder = new VdriveSeeder({
releaseDir: svc._releases ? svc._releases.releaseDir : null,
wire: (repo, opened) => svc.wireVdrive(repo, opened),
unwire: (repo) => svc.unwireVdrive(repo),
log: {
info: (m) => console.error(m),
warn: (m) => console.error(m)
}
})
// Fire-and-forget: a drive that will not open must never delay or fail boot.
;(async () => {
try { await svc._vdriveSeeder.start() } catch (err) {
console.error('[vdrive] seeder boot failed:', err && err.message)
}
})()
// Start sync service — with swarm for P2P replication, or without for local browsing only.
//
// Note: hyper-sync uses raw-byte git protocol which is incompatible with
// protomux-based corestore replication on the same connection. We pass
// `hyperswarmOpts` (not the shared swarm) so the sync service wrapper
// creates its OWN Hyperswarm. See lib/services/sync.js for the full
// rationale — this matches pear-git v1's documented pattern.
if (opts.sync !== false) {
const reposDir = opts.reposDir || join(opts.storage, 'git-repos')
try {
syncServer = await startSyncService({
reposDir,
// Tombstone gate — prune retired names after scan(), before serve().
isRetired: retiredRepos.isRetired,
hyperswarmOpts: opts.swarm ? (opts.swarm.swarm || {}) : null,
// Propagate the operator's intent. `swarm: false` must mean NO NETWORK
// for git sync too — previously this service built its own live
// Hyperswarm regardless, so a daemon (or test) started with the swarm
// disabled still joined the DHT and pulled from real peers.
offline: opts.swarm === false,
keypair: identity,
// Push ACL over punch:// (issue #37, item 2). hyper-sync calls this
// only for write services (git-receive-pack); reads stay open. The
// peer's swarm noise key resolves to an IdentityRegistry user via
// the noisekey index; push requires admin/write.
authorize: async ({ remotePublicKey }) => {
const noiseKey = remotePublicKey ? b4a.toString(remotePublicKey, 'hex') : null
return canPushGit(svc.identityRegistry, { transport: 'swarm', noiseKey })
}
})
svc._sync = syncServer
// Surface the absolute reposDir + the host user so SSR pages can
// render ssh://user@host/<reposDir>/<repo>.git clone URLs. Hostname
// is resolved client-side from window.location so proxies work.
svc._reposDir = reposDir
svc._hostUser = os.userInfo().username
// Pre-warm the /repos landing-page enrich cache in the background so the
// first human visitor after a (re)start doesn't pay the cold git cost
// (~15s across a large install). Fire-and-forget; recomputes only repos
// whose pack mtime changed since the persisted cache was written.
;(async () => {
try {
await enrichReposForWarm(syncServer.status(), syncServer.repos)
} catch (err) {
console.error('[prewarm] repo-activity warm failed:', err.message)
}
})()
// One-time backfill of the issue-author index from existing history so
// the public profile "recent issues" list isn't empty for pre-deploy
// issues. Marker-guarded (runs once per storage); background — opening
// every tracker is a one-time cost. Re-run by deleting the marker.
;(async () => {
try {
const marker = join(opts.storage, '.issue-index-backfilled')
if (!fs.existsSync(marker)) {
const r = await backfillIssueIndex(svc)
fs.writeFileSync(marker, JSON.stringify({ at: Date.now(), ...r }))
console.log('[backfill] issue-index:', JSON.stringify(r))
}
} catch (err) {
console.error('[backfill] issue-index failed:', err.message)
}
})()
// Repo-event log integration. Two halves:
//
// 1. On the primary, subscribe to sync.on('serve') and append a
// "created" entry the FIRST time we see a name this boot.
// Idempotency guard: seed the `loggedNames` set from the
// existing log so repeated startPearGit boots don't append
// duplicate "created" entries for the same repos.
// 2. On a peer with an origin configured (when opts.origin.{serverKey,
// repoEventLogKey} are both set), boot the RepoSync which
// opens the origin's log by key, streams entries, and pulls
// each repo through a throttle.
if (repoEventLog.writable) {
const loggedNames = new Set()
// Names for which a RepoEventLog entry has already conveyed the tracker
// bootstrap key. A "created" entry appended before the repo's tracker
// key was registered ships issueBootstrapKey:null; unless we re-emit a
// keyed entry once the key exists, a mirror that replicated the null
// entry binds no tracker and forks a split-brain island forever
// (2026-07-05 hyper-proto-peer). Seed from entries that already carry a
// key so we only re-emit for repos that never conveyed one.
const keyedNames = new Set()
try {
for await (const e of repoEventLog.entries({ live: false, start: 0 })) {
if (e && e.name) {
loggedNames.add(e.name)
if (e.issueBootstrapKey) keyedNames.add(e.name)
}
}
} catch {}
syncServer.on('serve', (name) => {
if (!name) return
// Fire-and-forget: logging/hook failures must not affect serving.
;(async () => {
// (Re)install the managed post-receive hook on EVERY serve — it's
// idempotent and writes the CURRENT http port, so a hook baked
// with a now-dead port (the 5098 bug, which silently dropped every
// file:// push) self-heals on the next boot. The hook POSTs to
// ref-update-observed, which drives gip publish + the RepoEventLog
// op:1 append → mirrors pull. Previously this sat behind the
// loggedNames guard, so an already-served repo never got its hook
// refreshed after a port change.
try { installPostReceiveHook(syncServer, name, svc) } catch (err) {
// eslint-disable-next-line no-console
console.error('[hook] install failed for', name, '-', err.message)
}
// Append/re-emit the "created" RepoEventLog entry that conveys the
// repo's tracker bootstrap key. See maybeAppendCreatedEntry for the
// null-key → keyed re-emit rationale (2026-07-05 hyper-proto-peer
// split-brain). Extracted so the logic is unit-testable.
try {
await maybeAppendCreatedEntry({
name, syncServer, issueRegistry, repoEventLog, loggedNames, keyedNames
})
} catch (err) {
// eslint-disable-next-line no-console
console.error('[repo-event-log] create append failed for', name, '-', err.message)
}
})()
})
// Eager hook regen on boot. The on('serve') listener above is attached
// AFTER startSyncService already ran sync.serve() for existing repos, so
// those boot-serves never reached it — leaving a hook baked with a now-
// dead port (the 5098 bug) un-rewritten until the repo is next accessed.
// svc is already listening (svc.listen ran before this sync setup), so
// the port is known here. Walk the repos once and (re)write every managed
// hook to the LIVE port. Fire-and-forget; idempotent.
;(async () => {
let fixed = 0
try {
for (const d of fs.readdirSync(reposDir)) {
if (!d.endsWith('.git')) continue
try { installPostReceiveHook(syncServer, d.replace(/\.git$/, ''), svc); fixed++ } catch {}
}
} catch {}
if (fixed > 0) {
// eslint-disable-next-line no-console
console.log('[hook] (re)installed post-receive hook on ' + fixed + ' repo(s) at boot (live port)')
}
})()
}
// Observable resync: hyper-sync emits pull lifecycle events per
// repo. Log them so `pgh app log pear-git-v3 -f` shows what resync
// is actually doing (was invisible before — user had to guess).
syncServer.on('pull', (remoteName, repoName, status) => {
console.log('[sync] pull ' + remoteName + '/' + repoName + ' → ' + status)
// Third bare-creation path (#138): hyper-sync's pull loop inits a bare
// for a repo it does not have yet. Marked HERE rather than inside
// hyper-sync, which has no business knowing about a forge-level
// visibility marker — it already emits this event, so the forge can own
// the policy. Safe for the same reason as the other two sites: an
// unexported repo never replicates, so anything pulled was already
// public at the source. Idempotent, so re-pulls cost nothing.
if (status === 'complete') ensureExportMarker(gipReposDir, repoName)
})
// ── State reconciler (the `repo scan`-style object sweep) ──
// Heals any repo whose git OBJECTS lag its refs — joined-after-cursor, a
// failed pull, a crash mid-clone — by punch:// PACK-pulling the missing
// objects (via RefReconciler's overlay+punch ensureObjects). Per-repo
// reconcile is EVENT-DRIVEN (RefReconciler subscribes to each tracker's
// `update`), so runtime ref changes self-heal; this is the boot + slow-
// interval SCAN over ALL known repos as the safety net. Cursor-
// independent — unlike the reverse puller's log replay. pear-git #3.
{
const runReconcile = () => {
Promise.resolve(svc._refReconcilers && svc._refReconcilers.reconcileAll())
// Report skipped too. Once the sweep started asking needsPass()
// (0.1.220) a healthy daemon skips nearly everything, so gating the
// line on `scanned` alone would make a working sweep look like a
// stopped one — and hide the number that says the skip is working.
.then((r) => {
if (!r) return
const skipped = r.skipped || 0
if (!r.scanned && !skipped) return
console.log('[reconcile] swept ' + r.scanned + ' repo(s)' +
(skipped ? ', skipped ' + skipped + ' quiet' : ''))
})
.catch((err) => console.error('[reconcile] sweep failed:', err && err.message))
}
// Boot sweep — delayed so the reverse pullers register their remotes
// first (the ensureObjects PACK pull needs a sync remote to pull from).
const bt = setTimeout(runReconcile, 30000); if (bt && bt.unref) bt.unref()
const iv = setInterval(runReconcile, 5 * 60 * 1000); if (iv && iv.unref) iv.unref()
svc._reconcileTimers = [bt, iv]
svc._reconcileNow = runReconcile
}
// Before announcing on the swarm, make sure every scanned repo has
// an issue tracker. Repos dropped into the repos dir out-of-band
// (manual git init, external clone) never went through the
// HTTP create-repo flow that registers a tracker — so without
// this, the 'serve' emit's repoEventLog append would carry a null
// issueBootstrapKey and mirrors couldn't auto-bind.
if (syncServer.repos) {
for (const name of syncServer.repos.keys()) {
try { await issueRegistry.getForRepo(name) } catch (err) {
// eslint-disable-next-line no-console
console.error('[pear-git-v3] ensure tracker for', name, '-', err && err.message)
}
}
}
// Backfill OrgEventLog for orgs that predate it. For every org in the
// orgs view, if the log has no entry yet, append op=0 carrying the
// org's RECORD autobase bootstrap key. Mirrors stream the log and
// mountOrg() each key — and since org issues live IN that record
// autobase (linked-core #42), the issues replicate with it. No
// separate per-org issue tracker to mint or convey.
if (orgEventLog.writable) {
// Track the LAST-announced orgBootstrapKey PER ORG (not just the name).
// A migrated org's key changes (single-orgs era: orgBootstrapKey=null →
// per-org #37: the org-record autobase key). Skipping by name alone left
// those stale null entries in the log forever, so mirrors read null and
// never mountOrg → 0 orgs on the mirror. Re-announce on KEY CHANGE.
const loggedOrgKeys = new Map() // name -> orgBootstrapKey hex | null
try {
for await (const e of orgEventLog.entries({ live: false, start: 0 })) {
if (e && e.name) {
loggedOrgKeys.set(e.name, e.orgBootstrapKey ? b4a.toString(e.orgBootstrapKey, 'hex') : null)
}
}
} catch {}
try {
const allOrgs = await orgs.list()
for (const org of allOrgs) {
if (!org || !org.name) continue
// Org issues live IN the org RECORD autobase (linked-core #42),
// so convey only its bootstrap key — mirrors mountOrg() it and
// get the issues for free. No separate issue tracker key.
const orgRoom = orgs.getOrgRoom(org.name)
const orgKeyHex = (orgRoom && orgRoom.keyHex) || null
// Skip ONLY if the log already carries the CURRENT key. A new org
// OR a migrated org (key changed null→per-org) gets (re-)announced.
if (loggedOrgKeys.get(org.name) === orgKeyHex) continue
const orgKey = orgKeyHex ? Buffer.from(orgKeyHex, 'hex') : null
try {
await orgEventLog.append({
op: 0, // created
name: org.name,
issueBootstrapKey: null,
orgBootstrapKey: orgKey
})
loggedOrgKeys.set(org.name, orgKeyHex)
} catch (err) {
// eslint-disable-next-line no-console
console.error('[org-event-log] backfill append failed for', org.name, '-', err.message)
}
}
} catch (err) {
// orgs.list() shouldn't fail at this point (orgs is ready), but
// boot must not block on it. Log and move on.
// eslint-disable-next-line no-console
console.error('[pear-git-v3] org pre-mint/backfill walk failed:', err && err.message)
}
// Stash the seen-set (names) on svc so the create-org handler skips
// duplicate appends after this initial pass.
svc._orgEventLogSeen = new Set(loggedOrgKeys.keys())
// In-band schema-version marker (#52 Phase 4.4, Keet's "version in the
// oplog"): stamp the canonical schema version into the replicated log
// so mirrors learn it by replication and an under-versioned node can
// refuse to serve. Idempotent; origin-only (we're inside the
// orgEventLog.writable guard).
try {
if (await announceSchemaVersion(orgEventLog, TARGET_VERSION)) {
console.log('[schema-guard] announced schema v' + TARGET_VERSION + ' to the org log')
}
} catch (err) {
console.error('[schema-guard] version announce failed:', err && err.message)
}
}
// Serving is part of what a pear-git daemon IS. startSyncService
// only scans on boot — we call serve() here, after the
// repoEventLog's on('serve') hook is attached, so the initial emits
// land in the log. _serveOne is idempotent, and the repos-dir
// watcher in startSyncService handles repos added after boot.
await syncServer.serve().catch((err) => {
// eslint-disable-next-line no-console
console.error('[pear-git-v3] initial serve failed:', err && err.message)
})
} catch (err) {
// eslint-disable-next-line no-console
console.error('[pear-git-v3] sync service failed to start:', err.message)
}
}
// RepoSync — opt-in via opts.origin (formerly opts.mirror). Activates
// when both serverKey (for pulling repos) and repoEventLogKey (for
// discovering them) are supplied. Needs the sync service running.
//
// opts.mirror still accepted as a backward-compat alias. Legacy
// fields: `primaryServerKey` on mirror; canonical: `serverKey` on origin.
const originCfg = opts.origin || opts.mirror || null
svc.originCfg = originCfg // exposed so follow-repo handler can guard "already a mirror"
const originServerKey = originCfg && (originCfg.serverKey || originCfg.primaryServerKey)
const originLogKey = originCfg && originCfg.repoEventLogKey
// Identity gate for every swarm-RPC connection to the origin.
//
// `serverKey` names a TOPIC, not a daemon: SwarmTransport creates its
// Hyperswarm with no keyPair, and a mirror joins its ORIGIN's topic, so a
// mirror is a server on the very topic used to reach the origin. Measured
// 2026-08-08 on stortron: connectSwarm({serverKey: <tabertron>}) was answered
// by STORTRON ITSELF 6/6 — it fetched its OWN release index and then failed
// every artifact with "no holder", and resync-from-origin hung twice at 300 s.
// A wrong ANSWER reads as a data problem, which is why it went unnoticed.
//
// `status` is anonymous-callable on every pgv3 version and returns the
// daemon's identity, so this works against an origin of any age.
const verifyIsOrigin = (wantKeyHex) => async (candidate) => {
const st = await candidate.call('status', {})
return !!st && st.identity === wantKeyHex
}
// ── Release-artifact replication ──────────────────────────────
// Release artifacts do NOT ride the autobase/sync path: they are plain files
// under <storage>/releases, and each daemon's deploy topic is derived from
// its OWN identity, so two backends never replicate them. Measured
// 2026-08-07: tabertron 16 repos/558 versions vs stortron 2/4, neither a
// superset, and the ACTIVE public backend was the near-empty one.
//
// Started here because this is the first point where the origin's serverKey
// (from the deployment bundle — never derived from local identity) is in
// scope alongside the release manager and the V2 overlay.
//
// EVENT-DRIVEN, no interval. The old 15-minute poll (gated behind
// PGV3_RELEASE_PULL=1) is gone: reconciliation now fires when the origin
// appends to its ReleaseEventLog, i.e. when the release index actually
// changed, and is SCOPED to the repo the event names. See
// services/release-puller.js startReleasePull.
if (svc._releases && svc.modules.overlayV2 && !svc._releasePull) {
try {
const { ReleasePuller, startReleasePull } = await import('./services/release-puller.js')
const { DeletedReleases } = await import('./services/deleted-releases.js')
// Replicated tombstone set. Created EMPTY here because the origin's
// release log is not opened until followReleaseLog() below; the same
// object is folded there and mutated live, so the puller sees tombstones
// as they arrive. Without it the puller re-requests versions the origin
// has deleted and retries forever -- see services/deleted-releases.js.
const deletedReleases = new DeletedReleases()
svc._deletedReleases = deletedReleases
const puller = new ReleasePuller({
releaseDir: svc._releases._releaseDir,
overlay: svc.modules.overlayV2,
deploy: svc._deploy || null,
releases: svc._releases,
deleted: deletedReleases,
log: (l) => console.log(l)
})
svc._releasePull = startReleasePull({
puller,
originServerKey,
connect: async ({ serverKey }) => {
const { Client } = await import('hyper-svc')
return Client.connectSwarm({
serverKey,
topicSalt: 'pear-git-v3',
timeout: 15000,
verify: verifyIsOrigin(serverKey)
})
},
log: (l) => console.log(l)
})
} catch (err) {
console.error('[release-pull] failed to start:', err.message)
}
}
// Follow the ORIGIN's release-event log — the trigger half of the above.
//
// Separate core from our own `releaseEventLog` (which is OUR writer): this one
// is opened BY KEY, so it is a read-only replica and rides the corestore
// replication hyper-svc already performs on every swarm connection — exactly
// how the origin's RepoEventLog and OrgEventLog are followed.
//
// The key comes from the deployment bundle ONLY. An earlier draft asked the
// origin over swarm-RPC when the bundle lacked it, and that hung
// basic.test.js: connectSwarm left its Hyperswarm running on timeout, so a
// mirror with an unreachable origin leaked a swarm that pinned the event loop
// open. That leak is fixed in hyper-svc 0.1.9, but this stays bundle-only on
// purpose — boot must not depend on a peer answering.
//
// A bundle predating this log therefore yields no follow until the operator
// runs the existing `resync-from-origin` (which refreshes the bundle with new
// key fields, below) and restarts — the identical lifecycle orgEventLog.coreKey
// already has, and the reason that refresh path exists.
//
// Cursor semantics: records the last SEEN seq, persisted per origin. Warm boot
// resumes at cursor+1 so mutations that landed while this mirror was down fire
// a scoped reconcile on start. A missing/corrupt cursor replays from 0, which
// is safe (entries are wake-ups) and matches RepoPuller's cold-boot fallback.
//
// NOTE the log only carries mutations from the moment it exists — the 559
// versions published before it was introduced generate no entries. Historical
// backfill stays a deliberate one-time `pull-releases` by an operator, which is
// the desired behaviour: upgrading must not silently start a 286 MiB transfer.
//
// Set by close() below; the follow is a LIVE stream, so it needs an explicit
// stop or teardown waits on it forever.
let releaseFollowStopped = false
const originReleaseLogKey = originCfg && originCfg.releaseEventLogKey
if (!networkDisabled && svc._releasePull && originServerKey && originReleaseLogKey) {
const followReleaseLog = async () => {
const originReleaseLog = new ReleaseEventLog({ corestore, key: originReleaseLogKey })
await originReleaseLog.ready()
svc._originReleaseLog = originReleaseLog
// Let the core learn its real length from peers before subscribing —
// without this a live stream can start at 0 and never see blocks that
// arrived pre-subscription (the same priming RepoPuller._runLoop does).
try { await originReleaseLog.core.update({ wait: true }) } catch {}
// Fold the whole log from seq 0 -- NOT from the saved cursor. The cursor
// only tracks what has been NOTIFIED; a delete that landed before this
// peer's cursor would otherwise be invisible and the version would be
// chased forever. The fold is a pure replay, so re-reading is free.
if (svc._deletedReleases) {
try {
const { foldDeleted } = await import('./services/deleted-releases.js')
await foldDeleted(originReleaseLog, svc._deletedReleases)
if (svc._deletedReleases.size) {
console.log('[release-pull] ' + svc._deletedReleases.size +
' tombstoned version(s) will not be pulled')
}
} catch (err) {
console.error('[release-pull] tombstone fold failed:', err && err.message)
}
}
const cursorPath = join(opts.storage, 'release-event-cursor.json')
let cursor = {}
try { cursor = JSON.parse(fs.readFileSync(cursorPath, 'utf8')) || {} } catch {}
const last = cursor[originServerKey]
const startSeq = typeof last === 'number' && last >= 0 ? last + 1 : 0
console.log('[release-pull] following origin release-log from seq ' + startSeq +
' (' + originReleaseLog.length + ' entries known)')
const saveCursor = (seq) => {
try {
cursor[originServerKey] = seq
const tmp = cursorPath + '.tmp'
fs.writeFileSync(tmp, JSON.stringify(cursor, null, 2))
fs.renameSync(tmp, cursorPath)
} catch {
// Non-fatal: worst case we replay entries next boot, which is safe.
}
}
let seq = startSeq
for await (const entry of originReleaseLog.entries({ live: true, start: startSeq })) {
if (releaseFollowStopped) break
try {
// Apply BEFORE notify: a delete arriving in the same batch must
// already be tombstoned when the reconcile it triggers plans its pull.
if (svc._deletedReleases) svc._deletedReleases.applyEntry(entry, seq)
svc._releasePull.notify(entry)
saveCursor(seq)
} catch (err) {
console.error('[release-pull] notify failed:', err && err.message)
}
seq++
}
}
followReleaseLog().catch((err) => {
console.error('[release-pull] release-log follow failed:', err && err.message)
})
} else if (svc._releasePull && !originReleaseLogKey) {
console.log('[release-pull] no releaseEventLog key in the deployment bundle — ' +
'run `pgh app resync pear-git-v3` then restart to start following release events')
}
const originOrgLogKey = originCfg && originCfg.orgEventLogKey
// Debounced mirror re-announce. The one-shot _doAnnounce (mirror-writer
// coordination) only enrolls trackers OPEN at boot; an org/repo mounted
// LATER (OrgPuller.mountOrg / RepoPuller pull) would never get a writer
// grant until a manual restart — the regression that left a mirror an
// INCOMPLETE writer (e.g. "Not writable" on a repo created after boot).
// Re-firing the announce on each new mount keeps the mirror a COMPLETE
// writer as resources arrive; it's idempotent on the origin (the watcher
// dedupes per entry and addWriter no-ops for an existing writer). Debounced
// to collapse the boot flurry of mounts into one announce. svc._announce-
// MirrorKeys is wired later by setupMirrorWriterCoordination (mirror only),
// so this is a no-op until then and on an origin (which never announces).
let _reannounceTimer = null
function scheduleMirrorReannounce () {
if (_reannounceTimer) return
_reannounceTimer = setTimeout(() => {
_reannounceTimer = null
try { if (svc._announceMirrorKeys) svc._announceMirrorKeys() } catch {}
}, 1500)
}
let repoSync = null
// Mirroring from a remote origin is inherently a NETWORK activity, so it must
// honour `swarm: false`. Without this guard a daemon booted on a copy of a
// mirror's storage (origin-bundle.json comes along with it) starts pulling
// every repo from the real origin even with the swarm disabled — which is
// what made cutover-prod-copy.test.js hang: it clones prod state, so it
// inherited prod's origin config and dutifully began mirroring production.
if (!networkDisabled && originCfg && originServerKey && originLogKey && syncServer) {
try {
const originLog = new RepoEventLog({
corestore,
key: originLogKey
})
await originLog.ready()
repoSync = new RepoPuller({
sync: syncServer,
issueRegistry,
log: originLog,
primaryServerKey: originServerKey,
maxConcurrentClones: originCfg.maxConcurrentClones || 3,
// Persisted cursor — without it, every restart replays the
// entire RepoEventLog from seq 0 (~80 minutes of redundant
// git fetches per restart at v3 production scale).
cursorPath: join(opts.storage, 'repo-puller-cursor.json'),
onMount: scheduleMirrorReannounce,
retired: retiredRepos
})
await repoSync.start()
svc._origin = repoSync
} catch (err) {
// eslint-disable-next-line no-console
console.error('[pear-git-v3] repo-sync failed to start:', err.message)
}
}
// Sister to repoSync: streams the origin's OrgEventLog and binds
// per-org issue-tracker bootstrap keys. Same shape, no git pull.
// Only boots if the bundle / origin config carries orgEventLogKey
// — older bundles won't have it, in which case the static device-
// invite path (mounts.issueOrgs) is the only propagation channel
// for that mirror until it's re-enrolled.
let orgSync = null
if (originCfg && originOrgLogKey) {
try {
const originOrgLog = new OrgEventLog({
corestore,
key: originOrgLogKey
})
await originOrgLog.ready()
orgSync = new OrgPuller({
issueRegistry,
orgs,
supportedVersion: TARGET_VERSION,
onBehind: (info) => { svc._schemaBehind = info },
log: originOrgLog,
onMount: scheduleMirrorReannounce
})
await orgSync.start()
svc._orgSync = orgSync
} catch (err) {
// eslint-disable-next-line no-console
console.error('[pear-git-v3] org-sync failed to start:', err.message)
}
}
// ── FOLLOWER MODE (read-only, non-mirror) ───────────────────────
//
// Distinct from the mirror flow above: a follower has its own identity
// and selectively replicates ONE remote resource (repo or org) without
// writer enrollment. Backed by:
// - FollowRegistry (lib/follow-registry.js) — in-memory trackers
// in namespace
// `follow:<srv>:<scope>:<name>`
// - FollowsRegistry (lib/follows-registry.js) — persistent index
// in <storage>/follows.json
// - RepoFollower (lib/services/repo-follower.js)
// — per-resource pull
// engine, one
// instance per follow
try {
const followRegistry = new FollowRegistry(corestore)
const followsRegistry = new FollowsRegistry(
join(opts.storage, 'follows.json')
)
const activeRepoFollowers = new Map() // key `<srv>|<name>` → RepoFollower
async function startRepoFollow (payload) {
if (!payload || payload.kind !== 'repo') {
throw new Error('startRepoFollow requires a repo-kind payload')
}
const srv = String(payload.sourceServerKey).toLowerCase()
const key = srv + '|' + payload.name
// Idempotent: already following this (source, name).
if (activeRepoFollowers.has(key)) {
return { alreadyFollowing: true }
}
// Persist FIRST so a crash mid-start still surfaces the follow
// on next boot (idempotent — start is clone-if-missing).
const payloadHex = Buffer.from(JSON.stringify(payload)).toString('hex')
followsRegistry.addRepo(srv, payload.name, payloadHex)
const follower = new RepoFollower({
corestore,
swarm: svc.swarm,
sync: syncServer,
followRegistry,
sourceServerKey: srv,
repoName: payload.name,
issueBootstrapKey: payload.issueBootstrapKey,
repoEventLogKey: payload.repoEventLogKey,
topicSalt: payload.topicSalt,
cursorPath: join(opts.storage, 'repo-follower-cursors.json')
})
activeRepoFollowers.set(key, follower)
await follower.start()
return { started: true }
}
async function stopRepoFollow (sourceServerKey, name) {
const srv = String(sourceServerKey).toLowerCase()
const key = srv + '|' + name
const follower = activeRepoFollowers.get(key)
if (follower) {
try { await follower.close() } catch {}
activeRepoFollowers.delete(key)
}
// Per never-delete-storage rule: keep tracker cores + git-repo
// bytes on disk. Just drop the index entry so we don't restore
// on next boot.
const removed = followsRegistry.removeRepo(srv, name)
// FollowRegistry's tracker cache holds the open instance —
// close it too so we don't leak file handles.
try { await followRegistry.closeRepo(srv, name) } catch {}
return removed
}
function listRepoFollows () {
const persisted = followsRegistry.listRepos()
return persisted.map((entry) => {
const key = entry.sourceServerKey + '|' + entry.name
const follower = activeRepoFollowers.get(key)
const status = follower ? follower.status() : null
return {
sourceServerKey: entry.sourceServerKey,
name: entry.name,
addedAt: entry.addedAt,
active: !!follower,
status
}
})
}
/**
* Best-effort: dial the source's daemon over swarm-RPC, call
* `repo-follow-payload`, return decoded payload.
*
* For Phase 1b this is intentionally simple — the swarm-RPC client
* implementation lives behind hyper-svc's swarm transport. If
* unimplemented or unreachable, surface a clean error so the user
* can paste the binary payload instead.
*/
async function fetchPayloadFromUrl (url) {
const parsed = parseFollowUrl(url) // throws EBADURL if malformed
// Phase 1b deliberately ships the URL helper without an active
// swarm fetch — operator path is `pgh repo follow <b64-payload>`.
// Phase 1c adds the swarm-RPC dial. The error message says so.
const err = new Error(
'URL-form follow not yet implemented in daemon (' + parsed.kind + ':' + parsed.name + '). ' +
'Use the b64 payload form: `pgh repo follow <payload>`. ' +
'Get the payload via `pgh repo info <repo> --follow-payload` or by calling ' +
'repo-follow-payload over UDS on the source daemon.'
)
err.code = 'EURL_FOLLOW_NYI'
throw err
}
svc._follows = {
registry: followRegistry,
persisted: followsRegistry,
activeRepoFollowers,
startRepoFollow,
stopRepoFollow,
listRepoFollows,
fetchPayloadFromUrl
}
// ── BOOT-TIME RESTORE ──
// Each persisted repo follow respawns a RepoFollower. Best-effort —
// any single failure is logged, others continue. Boot is not
// blocked: restores fire-and-forget after the svc is listening.
const persistedRepoFollows = followsRegistry.listRepos()
if (persistedRepoFollows.length > 0) {
// eslint-disable-next-line no-console
console.log('[follow] restoring', persistedRepoFollows.length, 'repo follow(s)')
for (const entry of persistedRepoFollows) {
try {
const payload = JSON.parse(Buffer.from(entry.payloadHex, 'hex').toString('utf8'))
startRepoFollow(payload).catch((err) => {
// eslint-disable-next-line no-console
console.warn('[follow] restore failed for', entry.name, '-', err.message)
})
} catch (err) {
// eslint-disable-next-line no-console
console.warn('[follow] could not decode persisted follow:', err.message)
}
}
}
} catch (err) {
// eslint-disable-next-line no-console
console.error('[pear-git-v3] follower mode init failed:', err.message)
svc._follows = null
}
// Remember the auto-import flag on svc so the post-TOFU auth hook
// respects the same setting (tests and careful deployments opt out).
svc._autoImportSSH = opts.autoImportSSH !== false
svc._sshAllowRoot = !!opts.allowRoot
// Trust-map migration: retired. Pre-F.6 deployments that still need
// trust→registry data migration can invoke the one-off helper at
// `lib/identity-migrate.js` via a UDS script. See spec/identity-ops.md
// for the migration procedure.
// Local-operator gate for the debug/repair handlers below (org #12).
// hyper-svc dispatches /rpc/<method> anonymously with no allowlist, and the
// daemon is internet-reachable via 7rnx.net/git/rpc/, so every handler
// without a gate is world-callable — over swarm-rpc too, where ctx.identity
// is a non-null { trust: 'network' } stub rather than an authenticated user.
// These handlers expose daemon internals or mutate autobase state and were
// only ever meant for the local operator; `tracker-repair`, `heal-repos`,
// `repo-resync` and `resync-from-origin` already assert exactly this.
function assertLocalOnly (ctx, method) {
if (!ctx || ctx.transport !== 'uds') {
const err = new Error(method + ' is UDS-only')
err.code = 'EAUTH'
err.status = 403
throw err
}
}
// Debug handler — dumps mirror-writer entries the daemon sees in its
// identity-autobase view. Useful for diagnosing cross-machine
// replication of register-mirror-writers ops.
svc.handle('dump-mirror-writers', async (ctx, params = {}) => {
assertLocalOnly(ctx, 'dump-mirror-writers')
const base = identityRegistry._base
if (params.forceUpdate && base && typeof base.update === 'function') {
try { await base.update({ wait: true }) } catch {}
}
// One-off promotion path: used to upgrade an existing writer
// (added with role 'writer' pre-fix) to 'indexer' so their stalled
// appends can linearize. After the one-shot fix runs, the normal
// device-invite flow (now role 'indexer') keeps new peers healthy.
if (params.promoteWriterKey) {
try {
await identityRegistry.addWriter(params.promoteWriterKey, {
role: 'indexer',
addedBy: svc.identity.publicKeyHex
})
return { promoted: params.promoteWriterKey }
} catch (err) {
return { error: err.message }
}
}
const entries = []
const writers = []
const writerCores = []
try {
for await (const e of identityRegistry.listMirrorWriters()) entries.push(e)
if (identityRegistry._view) {
for await (const e of identityRegistry._view.createReadStream({
gte: 'writer:', lt: 'writer:\xff'
})) {
const keyHex = e.key.slice('writer:'.length)
writers.push({ key: keyHex, ...e.value })
// What does THIS peer's corestore hold for this writer's core?
try {
const core = corestore.get({ key: b4a.from(keyHex, 'hex') })
await core.ready()
writerCores.push({ key: keyHex, length: core.length, writable: core.writable })
await core.close()
} catch (err) {
writerCores.push({ key: keyHex, error: err.message })
}
}
}
} catch (err) {
return { error: err.message }
}
// Autobase internal state — used to diagnose linearizer stalls where
// writer-core blocks replicate but apply doesn't run on them. ack()
// short-circuits on _acking / _interrupting / _appending /
// fastForwarding / !isLocalIndexer, so capturing all five shows
// exactly which gate is closed.
const autobaseState = {}
try {
autobaseState.acking = !!base._acking
autobaseState.interrupting = !!base._interrupting
autobaseState.appending = base._appending !== null && base._appending !== undefined
autobaseState.fastForwarding = typeof base.isFastForwarding === 'function'
? !!base.isFastForwarding()
: null
autobaseState.systemCoreLength = base.core ? base.core.length : null
autobaseState.systemCoreSignedLength = base.core && base.core.core && base.core.core.state
? base.core.core.state.length : null
autobaseState.viewVersion = base.view ? base.view.version : null
const applyState = base._applyState || null
if (applyState) {
autobaseState.applyState = {
opened: !!applyState.opened,
isLocalIndexer: typeof applyState.isLocalIndexer === 'function'
? !!applyState.isLocalIndexer() : null,
isLocalPendingIndexer: typeof applyState.isLocalPendingIndexer === 'function'
? !!applyState.isLocalPendingIndexer() : null
}
} else {
autobaseState.applyState = null
}
const linearizer = base.linearizer || null
if (linearizer) {
autobaseState.linearizer = {
indexerCount: linearizer.indexers ? linearizer.indexers.length : null,
indexers: linearizer.indexers ? linearizer.indexers.map(ix => ({
key: ix.core && ix.core.key ? b4a.toString(ix.core.key, 'hex').slice(0, 16) : null,
length: ix.core ? ix.core.length : null,
indexed: typeof ix.indexed === 'number' ? ix.indexed : null
})) : null,
tail: typeof linearizer.tail === 'number' ? linearizer.tail : null,
tip: typeof linearizer.tip === 'number' ? linearizer.tip : null
}
} else {
autobaseState.linearizer = null
}
autobaseState.activeWriters = base.activeWriters && typeof base.activeWriters.size === 'number'
? base.activeWriters.size : null
autobaseState.ackIntervalMs = base._ackInterval || null
autobaseState.ackTimerArmed = !!base._ackTimer
} catch (err) {
autobaseState.error = err.message
}
return {
identityViewLength: base && base.view ? base.view.version : null,
identityWritable: !!identityRegistry.writable,
identityKey: base && base.key ? b4a.toString(base.key, 'hex') : null,
identityLocalKey: base && base.local ? b4a.toString(base.local.key, 'hex') : null,
swarmConnections: svc.swarm ? svc.swarm.connections : null,
swarmTopic: svc.swarm && svc.swarm.topic ? b4a.toString(svc.swarm.topic, 'hex') : null,
autobaseState,
writerCores,
writersInView: writers,
entries
}
})
// Debug: probe a specific core by key and report its state. Used to
// verify whether a peer's writer-core / autobase-system-core has
// actually replicated into our corestore. If length stays 0 after
// ready() + a wait window, blocks aren't flowing despite swarm conn.
svc.handle('probe-core', async (ctx, params = {}) => {
assertLocalOnly(ctx, 'probe-core')
if (!params.key || typeof params.key !== 'string' || params.key.length !== 64) {
throw new Error('probe-core requires { key: 64-char hex }')
}
const waitMs = Math.min(parseInt(params.waitMs, 10) || 5000, 30000)
const core = corestore.get({ key: b4a.from(params.key, 'hex') })
await core.ready()
const initial = {
length: core.length,
writable: core.writable,
fork: core.fork,
peers: core.peers ? core.peers.length : null
}
// attempt to fetch any peer state
try {
if (typeof core.update === 'function') {
await core.update({ wait: true, timeout: waitMs })
}
} catch {}
const afterUpdate = {
length: core.length,
writable: core.writable,
fork: core.fork,
peers: core.peers ? core.peers.length : null
}
// Try to download block 0 to confirm we can actually receive blocks.
let block0 = null
let block0Err = null
if (core.length > 0) {
try {
const buf = await core.get(0, { wait: true, timeout: waitMs })
block0 = { received: !!buf, byteLength: buf ? buf.byteLength : 0 }
} catch (err) {
block0Err = err.message
}
}
await core.close()
return { key: params.key, initial, afterUpdate, block0, block0Err }
})
// Debug: report swarm peer pubkeys so we can verify tabertron is
// actually peered to stortron specifically (vs unrelated DHT
// walkers on the same topic).
svc.handle('debug-swarm-peers', async (ctx) => {
assertLocalOnly(ctx, 'debug-swarm-peers')
if (!svc.swarm || !svc.swarm.swarm) return { peers: [], note: 'no swarm' }
const peers = []
for (const conn of svc.swarm.swarm.connections) {
peers.push({
remotePublicKey: conn.remotePublicKey ? b4a.toString(conn.remotePublicKey, 'hex') : null,
remoteAddress: conn.rawStream && conn.rawStream.remoteHost
? conn.rawStream.remoteHost + ':' + conn.rawStream.remotePort
: null,
opened: !!conn.opened,
destroyed: !!conn.destroyed
})
}
return { peers, total: peers.length }
})
// Debug: introspect IdentityRegistry's actual storage layers — system
// core (autobase internal), the identity-view hyperbee (user-facing
// queries), and the linearizer's view of writers.
svc.handle('identity-introspect', async (ctx) => {
assertLocalOnly(ctx, 'identity-introspect')
const base = identityRegistry._base
const out = {}
try {
out.systemCore = {
key: base.core ? base.core.key.toString('hex').slice(0, 16) : null,
length: base.core ? base.core.length : null,
signedLength: base.core && base.core.core ? base.core.core.state.length : null
}
out.bee = {
type: identityRegistry._view ? identityRegistry._view.constructor.name : null,
coreKey: identityRegistry._view && identityRegistry._view.core
? identityRegistry._view.core.key.toString('hex').slice(0, 16)
: null,
coreLength: identityRegistry._view && identityRegistry._view.core
? identityRegistry._view.core.length
: null,
version: identityRegistry._view ? identityRegistry._view.version : null
}
// count users via a stream over view
let userCount = 0
try {
for await (const _ of identityRegistry._view.createReadStream({
gte: 'user:', lt: 'user:\xff'
})) userCount++
} catch {}
out.userCount = userCount
out.linearizer = {
indexerCount: base.linearizer && base.linearizer.indexers
? base.linearizer.indexers.length
: null,
activeWriters: base.activeWriters ? base.activeWriters.size : null
}
// try a force update + check views again
await base.update({ wait: true }).catch(() => {})
out.afterUpdate = {
systemCoreLength: base.core ? base.core.length : null,
viewVersion: identityRegistry._view ? identityRegistry._view.version : null,
beeCoreLength: identityRegistry._view && identityRegistry._view.core
? identityRegistry._view.core.length : null
}
} catch (err) {
out.error = err.message
}
return out
})
// Debug: try autobase fast-forward step-by-step with errors surfaced.
// Reproduces FastForward._upgrade logic from autobase/lib/fast-forward.js
// to see which gate is failing in our scenario.
svc.handle('identity-ff-debug', async (ctx) => {
assertLocalOnly(ctx, 'identity-ff-debug')
const base = identityRegistry._base
const result = { steps: [], finalErr: null }
try {
const rootKey = base.core.key
result.steps.push({ step: 'rootKey', value: rootKey ? rootKey.toString('hex').slice(0, 24) : null })
const core = corestore.get({ key: rootKey, active: true, encryption: null })
await core.ready()
result.steps.push({
step: 'core.ready',
length: core.length,
manifestVersion: core.manifest && core.manifest.version,
manifestLinked: core.manifest && core.manifest.linked
? core.manifest.linked.map(k => k.toString('hex').slice(0, 16))
: null,
manifestSigners: core.manifest && core.manifest.signers
? core.manifest.signers.length
: null
})
if (core.length === 0) {
result.finalErr = 'core.length is 0 — no blocks to fast-forward to'
await core.close()
return result
}
// Try reading last block
try {
const block = await core.get(core.length - 1, { timeout: 5000 })
result.steps.push({ step: 'lastBlockFetch', byteLength: block ? block.byteLength : 0 })
} catch (err) {
result.steps.push({ step: 'lastBlockFetch', error: err.message })
}
// Open SystemView at checkout
const SystemView = await import('autobase/lib/system.js').then(m => m.default).catch(() => null)
if (!SystemView) {
result.finalErr = 'could not import SystemView'
await core.close()
return result
}
const system = new SystemView(core, { checkout: core.length })
await system.ready()
result.steps.push({
step: 'systemView.ready',
encryptionLength: system.encryptionLength,
indexers: system.indexers
? system.indexers.map(i => i.key ? i.key.toString('hex').slice(0, 16) : '?')
: null,
entropy: system.entropy ? system.entropy.toString('hex').slice(0, 16) : null
})
// Manifest version >= 2 check — this is the silent-fail gate
if (core.manifest.version >= 2) {
const linked = core.manifest.linked
if (system.encryptionLength === 0) {
result.finalErr = 'GATE FAILED: system.encryptionLength === 0'
} else if (linked === null) {
result.finalErr = 'GATE FAILED: manifest.linked is null'
} else if (!linked.length) {
result.finalErr = 'GATE FAILED: manifest.linked is empty'
} else {
result.steps.push({ step: 'manifestVersion2GatePassed', linkedLength: linked.length })
}
}
await core.close()
} catch (err) {
result.finalErr = err.message
result.errStack = err.stack
}
return result
})
// Debug: trigger autobase repair (force fast-forward) on the identity
// registry. Used to test peer-driven recovery of a fresh-after-corruption
// autobase. Reports view length and length of replicated system core
// before and after, so we can see whether peer state is reachable.
svc.handle('identity-repair', async (ctx) => {
assertLocalOnly(ctx, 'identity-repair')
const base = identityRegistry._base
const before = {
viewLength: base && base.view ? base.view.length : null,
coreLength: base && base.core ? base.core.length : null,
coreSignedLength: base && base.core && base.core.core ? base.core.core.state.length : null,
writable: !!(base && base.writable),
isFastForwarding: typeof base.isFastForwarding === 'function' ? base.isFastForwarding() : null
}
let repairError = null
try {
if (typeof base.repair === 'function') {
await base.repair()
} else {
repairError = 'base.repair() not available'
}
} catch (err) {
repairError = err.message
}
const after = {
viewLength: base && base.view ? base.view.length : null,
coreLength: base && base.core ? base.core.length : null,
coreSignedLength: base && base.core && base.core.core ? base.core.core.state.length : null,
writable: !!(base && base.writable),
isFastForwarding: typeof base.isFastForwarding === 'function' ? base.isFastForwarding() : null
}
return { before, after, repairError }
})
// Debug/recovery: force-fast-forward repair on a PER-REPO tracker autobase.
// This is keet's canonical recovery for the manifest-migration null-view crash
// (a standalone genesis peer whose tracker can't incrementally reconcile a
// group indexer-migration). Runs in the LIVE daemon so peer state is reachable
// — base.repair() (forceFastForward) fast-forwards onto the group's signed
// head, tolerating the migrated null view (the FF path does `if(!ref) continue`)
// where the incremental drain crashes. Non-destructive. UDS-only. Mirror of
// identity-repair above. #3 Increment 2c.
svc.handle('tracker-repair', async (ctx, params = {}) => {
if (ctx.transport !== 'uds') {
const err = new Error('tracker-repair is UDS-only')
err.code = 'EAUTH'
throw err
}
const repoName = params.repo
if (!repoName) return { error: 'pass { repo }' }
const opened = await issueRegistry.openTrackerForRepair(repoName)
if (opened.error) return { error: opened.error }
const { tracker, baseErr } = opened
const base = tracker.base
const snap = () => {
const s = {}
try { s.viewLen = base && base.view ? base.view.length : null } catch { s.viewLen = null }
try { s.signedLength = base.signedLength } catch { s.signedLength = null }
try { s.indexedLength = base.indexedLength } catch { s.indexedLength = null }
try { s.writable = !!(base && base.writable) } catch { s.writable = null }
try { const sys = base._applyState && base._applyState.system; s.indexers = sys && sys.indexers ? sys.indexers.length : null } catch { s.indexers = null }
try { s.views = base.system.views.map(v => v.key.toString('hex').slice(0, 8)).join(',') } catch { s.views = null }
return s
}
const before = snap()
let repairError = null
try {
if (typeof base.repair === 'function') await base.repair()
else repairError = 'base.repair() not available'
} catch (err) { repairError = err.message }
try { await base.update() } catch {}
const after = snap()
let issues = null
try { issues = (await tracker.list({ status: 'all' })).length } catch (e) { issues = 'ERR:' + (e && e.message) }
return {
repo: repoName,
beforeBaseError: baseErr ? (baseErr.message || String(baseErr)) : null,
before,
after,
repairError,
issues
}
})
// forceResync — clean RE-JOIN of a stuck tracker into a FRESH corestore.
// A standalone peer whose tracker can't reconcile a group manifest migration
// has a poisoned local writer in the MAIN corestore. A fresh namespace does
// NOT shed it (autobase's local key is primaryKey-derived, namespace-
// independent — proven in force-resync-participation.test.js); only a fresh
// corestore does. forceResync re-joins in a per-tracker side corestore (clean
// local writer) that replicates the group's healthy state as a READ-ONLY
// replica. Because that side store is OUTSIDE the main corestore.replicate on
// each connection, we mount it (future connections) + replicate on existing
// connections + join its room topic here. Write participation is restored by
// admitting the returned localKey (`pgh repo grant-writer`) on the tracker
// owner. UDS-only.
svc.handle('repo-resync', async (ctx, params = {}) => {
if (ctx.transport !== 'uds') {
const err = new Error('repo-resync is UDS-only')
err.code = 'EAUTH'
throw err
}
const repoName = params.repo
if (!repoName) return { error: 'pass { repo }' }
let tracker
try {
tracker = await issueRegistry.forceResync(repoName)
} catch (err) {
return { error: 'forceResync failed: ' + (err && err.message) }
}
const gen = (issueRegistry._index.resyncGen && issueRegistry._index.resyncGen[repoName]) || 0
// Wire the side store into swarm replication (mount + existing conns + topic).
svc.wireSideStoreTracker(repoName, tracker, gen)
try { await tracker.base.update() } catch {}
// AUTO-ADMISSION via the DOORBELL (P2P, no url): ring it with a vouch signed
// by this peer's OLD writer key (captured by forceResync; still in the group's
// writer set). The owner's apply verifies the vouch + addWriters — no HTTP, no
// serverKey. Falls back to manual repo-add-writer if no voucher was captured.
let doorbell = 'skipped (no voucher key — manual repo-add-writer)'
if (tracker._voucherKeyPair) {
const started = svc.ringDoorbellUntilAdmitted(repoName, tracker, tracker._voucherKeyPair)
doorbell = started
? 'ringing until admitted (re-ring loop; owner admits on apply)'
: 'skipped (voucher unusable — manual repo-add-writer)'
}
let issues = null
try { issues = (await tracker.list({ status: 'all' })).length } catch (e) { issues = 'ERR:' + (e && e.message) }
return {
ok: true,
repo: repoName,
resyncGen: gen,
localKey: tracker.localKeyHex,
writable: !!(tracker.base && tracker.base.writable),
issues,
doorbell,
note: 'read-only until admitted; the doorbell auto-admits via the owner (P2P). ' +
'If it stays read-only, fall back to `pgh repo grant-writer ' + repoName + '` on the owner.'
}
})
// ── Mirror-writer coordination (multi-autobase bootstrap) ──
// See docs/mirror-writer-coordination.md. Primary watches
// identityRegistry for register-mirror-writers entries and issues
// canonical in-band addWriter ops on orgs / trackers. Mirror
// announces its auto-gen local keys once ready.
setupMirrorWriterCoordination({
svc,
identityRegistry,
identityHex: identity.publicKeyHex,
orgs,
issueRegistry,
originCfg,
networkDisabled,
// Wiring for bidirectional repo discovery (#16): when a peer announces
// its own origin coordinates (`__server__` / `__repoeventlog__`), the
// watcher boots a reverse RepoPuller toward it so repos created on that
// peer are discovered + cloned + served here too.
syncServer,
corestore,
storage: opts.storage,
originServerKey
})
// Boot: activate the swarm rendezvous for every org/repo room this
// daemon already tracks, so peers can find us for those specific
// rooms (Keet activateRoom). New rooms created/joined after boot are
// activated by their handlers. Best-effort — never block boot.
try {
for (const { name } of orgs.listOrgKeys()) {
const room = orgs.getOrgRoom(name)
if (room) svc.activateRoom(room.discoveryKey)
}
// Per-repo issue trackers are SEPARATE rooms — activate each. Org issue
// rooms ARE the org RECORD rooms (linked-core #42), already activated by
// the listOrgKeys loop above, so no separate org-tracker activation.
if (issueRegistry && typeof issueRegistry.listAllKnownRepos === 'function') {
for (const repoName of issueRegistry.listAllKnownRepos()) {
try {
const t = await issueRegistry.getForRepo(repoName)
if (!t) continue
// A forceResync side-store tracker (its own corestore, OUTSIDE the main
// corestore.replicate) needs full replication wiring re-attached at
// boot — not just a topic join — so the persisted heal keeps
// replicating after a restart. Plain trackers just activate the topic.
if (t._sideStore) {
const gen = (issueRegistry._index.sideStore && issueRegistry._index.sideStore[repoName]) || 0
svc.wireSideStoreTracker(repoName, t, gen)
} else {
svc.activateRoom(t.discoveryKey)
}
} catch {}
}
}
} catch (err) {
// eslint-disable-next-line no-console
console.error('[pear-git-v3] boot room activation failed:', err && err.message)
}
// Boot: re-register outstanding blind-pairing invites (persisted) so an
// already-shared invite survives a daemon restart. Needs the shared swarm +
// the org/repo room mounted. Best-effort — never block boot.
if (svc.swarm && svc.swarm.swarm && svc._pendingPairings) {
for (const [name, descriptor] of svc._pendingPairings.list('orgs')) {
try { await orgs.resumeOrgPairing(name, descriptor, svc.swarm.swarm) } catch (err) {
// eslint-disable-next-line no-console
console.error('[pending-pairings] resume org', name, '-', err && err.message)
}
}
for (const [name, descriptor] of svc._pendingPairings.list('repos')) {
try { await issueRegistry.resumeRepoPairing(name, descriptor, svc.swarm.swarm) } catch (err) {
// eslint-disable-next-line no-console
console.error('[pending-pairings] resume repo', name, '-', err && err.message)
}
}
}
// ── Ref-update observed (called by the post-receive hook) ─────
//
// Every bare repo we serve gets a post-receive hook that POSTs
// here on git push. We append an op:1 entry to RepoEventLog —
// mirrors replicate the autobase, their RepoSync _enqueue sees
// the entry and triggers a targeted pull. Autobase → keet-room
// style propagation: no polling, no per-repo cold connection for
// "have there been updates". Actual git bytes still flow via
// git-upload-pack on the existing punch:// topic.
//
// Accept any transport (hook posts via local HTTP). No body auth
// because a rogue POST only causes a redundant mirror fetch — no
// write to the repo itself, no new refs unless git-receive-pack
// on origin already accepted them.
// Shared gip producer: project ONE ref change into the P2P layer — objects →
// overlay (git bundle), refs + bundle-chain → the per-repo autobase. Used by
// BOTH the post-receive hook path (ref-update-observed, file:// pushes) AND
// the in-process HTTP receive-pack observer (handlers/repos.js), so an HTTP
// push reaches gip too — previously only the hook drove gip, leaving HTTP
// pushes on the legacy repoEventLog path only. Writer-only (self-guards on
// tracker.writable); idempotent on replay via the FF guard. The bare repo
// stays canonical — this is the additive P2P projection.
const GIP_ZERO_OID = '0000000000000000000000000000000000000000'
// SECURITY (pear-git org #12): ref state is READ FROM THIS DAEMON'S BARE
// REPO — never taken from the caller. `ref-update-observed` dispatches
// anonymously over HTTP (and over swarm-rpc), and is internet-reachable via
// 7rnx.net/git/rpc/, so honouring a body-supplied newSha let any caller
// repoint — or, with the zero oid, DELETE — a ref inside the shared,
// replicated per-repo autobase. `tracker.writable` gates the autobase-writer
// concept only; it is not an authorization check on the caller.
//
// The caller now chooses only WHICH repo/ref to re-read. A forged call is at
// most a redundant, idempotent re-check against disk. This is the same
// disk → view direction that services/ref-publish-reconciler.js already runs
// as a boot pass; here it is narrowed to the one ref the push touched.
//
// Callers may still pass oldSha/newSha (handlers/repos.js does) — they are
// ignored. That path fires after receive-pack has written the refs, so the
// disk read yields the identical result.
//
// Scope note: listDiskRefs covers refs/heads/ + refs/tags/ only, matching
// what RefReconciler materializes on mirrors. Out-of-band refs are
// deliberately not propagated.
async function gipPublishRef ({ name, ref }) {
if (!name || !ref || !svc._reposDir) return
if (!isValidRepoName(name)) return
const repoPath = join(svc._reposDir, name + '.git')
const tracker = await issueRegistry.getForRepo(name)
if (!tracker.writable) return // only writer nodes publish
const userId = svc.identity.publicKeyHex
const disk = await listDiskRefs(repoPath)
const diskOid = disk.get(ref) || null
// getRef, not listRefs: both do one base.update(), but listRefs also
// streams the entire ref range — needless on a per-push path (#21).
const view = await tracker.getRef(ref)
const announced = (view && view.hash) || null
if (!diskOid) {
// Gone from disk ⇒ a real delete. Only a ref the view actually carries
// can be dropped, so an unknown ref name is a no-op rather than a delete.
if (!announced || announced === GIP_ZERO_OID) return
await tracker.deleteRef(ref, userId, announced)
} else if (announced !== diskOid) {
await publishRefUpdate({
repoPath,
repoName: name,
ref,
oldOid: announced || GIP_ZERO_OID,
newOid: diskOid,
tracker,
overlay: overlayV2,
spoolDir: join(opts.storage, 'gip-bundles'),
userId
})
}
// keep this writer's own bare repo's reconciler running (no-op if already
// present; harmless on the owner where bare == view)
if (svc._refReconcilers) svc._refReconcilers.ensureFor(name).catch(() => {})
}
svc._gipPublishRef = gipPublishRef
// Set a repo's DEFAULT BRANCH, fleet-wide.
//
// HEAD used to be per-backend (a symbolic ref, unmodelable in the ref→hash
// view), so each node repaired it with a master→main→first heuristic and
// nodes disagreed — 4 of 142 repos did, and the public saw whichever backend
// the load balancer hit. RefReconciler now replicates it via meta:head, but
// it SEEDS from whichever writable node reconciles first, which is a race
// when nodes already disagree. This makes it a deliberate choice instead.
//
// Admin-gated: the default branch is what the repo page renders and what
// `git clone` checks out.
svc.handle('repo-set-head', async (ctx, params = {}) => {
await requireAdmin(ctx, svc.identityRegistry)
const name = params.name
if (!name) throw new Error('repo-set-head requires { name }')
if (!isValidRepoName(name)) throw new Error('repo-set-head: invalid repo name')
if (!params.ref) throw new Error('repo-set-head requires { ref } (branch name)')
// Accept either `main` or `refs/heads/main`.
const target = String(params.ref).startsWith('refs/heads/')
? String(params.ref)
: 'refs/heads/' + String(params.ref)
const repoPath = svc._reposDir ? join(svc._reposDir, name + '.git') : null
if (!repoPath || !fs.existsSync(repoPath)) throw new Error('unknown repo: ' + name)
// The branch MUST exist here, or we would write a HEAD that gives every
// node a dangling default and makes `git clone` find no branch at all.
const disk = await listDiskRefs(repoPath)
if (!disk.has(target)) {
throw new Error('no such branch: ' + target + ' (have: ' +
Array.from(disk.keys()).filter(r => r.startsWith('refs/heads/')).join(', ') + ')')
}
const tracker = await issueRegistry.getForRepo(name)
if (!tracker || typeof tracker.setMeta !== 'function') {
throw new Error('repo tracker does not support meta (peer running older code)')
}
if (!tracker.writable) {
throw new Error('this node is not a writer for ' + name + ' — run this on the tracker owner')
}
// Replicate it, then apply locally so the operator sees the change now
// rather than on the next reconcile pass.
await tracker.setMeta('head', target)
try {
await execFile('git', ['--git-dir', repoPath, 'symbolic-ref', 'HEAD', target], { encoding: 'utf8' })
} catch (err) {
return { ok: true, name, head: target, replicated: true, localApplyError: err.message }
}
invalidateRepoActivity(name)
return { ok: true, name, head: target, replicated: true }
})
svc.handle('ref-update-observed', async (ctx, params = {}) => {
if (!params || !params.name) throw new Error('ref-update-observed requires { name }')
// #12: validate BEFORE the name reaches the filesystem, the activity cache
// or issueRegistry.getForRepo — the last of which would otherwise
// MATERIALIZE a per-repo tracker autobase for any attacker-chosen name.
if (!isValidRepoName(params.name)) {
throw new Error('ref-update-observed: invalid repo name')
}
// And require the repo to actually exist here. The hook only ever fires for
// a repo this daemon serves, so this costs a stat on the real path and
// stops junk names entering the replicated event log.
const repoPath = svc._reposDir ? join(svc._reposDir, params.name + '.git') : null
if (repoPath && !fs.existsSync(repoPath)) {
return { ok: false, reason: 'unknown repo' }
}
// Drop this repo's cached /repos activity so the landing page reflects the
// push. Precise invalidation: loose-object pushes don't always bump the
// pack mtime the enrich cache keys on, so we can't rely on that alone.
invalidateRepoActivity(params.name)
// gip-overlay producer (shared helper). Fire-and-forget so it never blocks
// or breaks ref-update-observed; runs even where repoEventLog isn't
// writable (it self-guards on the per-repo tracker.writable). Ref state is
// re-read from disk inside — params.oldSha/newSha are untrusted and unused.
if (params.ref && svc._reposDir) {
gipPublishRef({ name: params.name, ref: params.ref })
.catch((err) => {
// eslint-disable-next-line no-console
console.error('[ref-update-observed] gip publish failed:', err.message)
})
}
if (!repoEventLog || !repoEventLog.writable) {
return { ok: false, reason: 'repoEventLog not writable on this peer' }
}
// `head` is replicated to peers, so it is derived from disk too rather
// than echoed from the request body.
let head = ''
if (params.ref && repoPath) {
const disk = await listDiskRefs(repoPath)
head = disk.get(params.ref) || ''
}
try {
await repoEventLog.append({
op: 1, // ref-update
name: params.name,
head,
issueBootstrapKey: null
})
} catch (err) {
return { ok: false, reason: err.message }
}
// Auto-build trigger. handlers/repos.js fires autoBuildFromPush on
// HTTP git-receive-pack close, which covers HTTP pushes only. Local
// file:// pushes (via the `pear` remote pointing at the bare repo)
// don't go through the HTTP handler — only the post-receive hook
// fires, calling here. Without this trigger they'd update the bare
// repo + repoEventLog but never run ci.yaml. Synthesizes a single-
// ref before/after pair. Fire-and-forget — the ref-update-observed
// response should not block on the build.
//
// #12: the before/after pair is derived from disk, not from the body.
// buildAndPublish builds whatever the ref currently points at (it ignores
// the sha it is handed), so a forged body could never choose WHICH code
// ran — but it could force an unbounded number of CI runs, anonymously.
// Deduping on the disk sha closes that: once a sha has been built here,
// repeat calls for it are no-ops. That also settles the double-build this
// note used to flag: an HTTP push reaches both this handler (via
// post-receive) and the receive-pack-close path in handlers/repos.js.
if (svc._sync && svc._releases && params.ref && head) {
const buildKey = params.name + ' ' + params.ref
const alreadyBuilt = svc._lastBuiltRef.get(buildKey) || ''
svc._lastBuiltRef.set(buildKey, head)
import('./ci-orchestrator.js').then(async ({ autoBuildFromPush }) => {
try {
const before = new Map([[params.ref, alreadyBuilt]])
const after = new Map([[params.ref, head]])
await autoBuildFromPush({
vdriveSeeder: svc._vdriveSeeder,
// Durable root for the drive key: derived per repo, so losing a
// drive store is a rebuild, not a dead pear:// link (#145).
vdriveRootKey: svc._vdriveRootKey,
sync: svc._sync,
releaseManager: svc._releases,
pendingBuilds: svc._pendingBuilds,
repo: params.name,
before,
after
})
} catch (err) {
// eslint-disable-next-line no-console
console.error('[ref-update-observed] auto-build failed:', err.message)
}
}).catch(() => {})
}
return { ok: true, name: params.name, ref: params.ref, newSha: head }
}, { anonymous: true })
// ── Heal (git fsck → refetch → force-reclone per broken repo) ──
//
// Iterates every bare repo in the daemon's reposDir. Runs git fsck
// on each (parallel). For any that report missing/unreadable
// objects, calls syncServer.pull() — hyper-sync's pull() includes
// the fsck-detected --refetch fallback and the force-reclone last
// resort. Healthy repos are skipped. Returns per-repo status so
// the CLI can stream progress.
//
// UDS-only — trusted local caller (pgh app heal).
svc.handle('heal-repos', async (ctx) => {
if (ctx.transport !== 'uds') {
const err = new Error('heal-repos is UDS-only')
err.code = 'EAUTH'
throw err
}
if (!syncServer) return { error: 'sync service not active' }
const reposDir = svc._reposDir
if (!reposDir || !fs.existsSync(reposDir)) return { error: 'no repos dir' }
const dirs = (await readdir(reposDir)).filter(d => d.endsWith('.git'))
if (dirs.length === 0) return { reposDir, results: [] }
if (!syncServer.remotes.has('primary') && originServerKey) {
syncServer.addRemote('primary', originServerKey)
}
// Phase 1: fsck all (parallel — cheap, local, disk-bound)
const fsckResults = await Promise.all(dirs.map(async (d) => {
const repoPath = join(reposDir, d)
const broken = typeof syncServer._fsckMissingObjects === 'function'
? await syncServer._fsckMissingObjects(repoPath)
: false
return { repo: d.replace(/\.git$/, ''), repoPath, broken }
}))
// Phase 2: heal broken ones (serial through hyper-sync's pull,
// which has the fsck→refetch→force-reclone tier).
const results = []
for (const { repo, repoPath, broken } of fsckResults) {
if (!broken) {
results.push({ repo, status: 'healthy' })
continue
}
try {
const pullRes = await syncServer.pull('primary', repo)
const stillBroken = typeof syncServer._fsckMissingObjects === 'function'
? await syncServer._fsckMissingObjects(repoPath)
: false
const failedFromPull = pullRes && pullRes.failed && pullRes.failed.length > 0
? pullRes.failed[0].error
: null
if (stillBroken || failedFromPull) {
results.push({ repo, status: 'failed', error: failedFromPull || 'still broken after heal' })
} else {
results.push({ repo, status: 'healed' })
}
} catch (err) {
results.push({ repo, status: 'failed', error: err.message })
}
}
return {
reposDir,
total: results.length,
healthy: results.filter(r => r.status === 'healthy').length,
healed: results.filter(r => r.status === 'healed').length,
failed: results.filter(r => r.status === 'failed').length,
results
}
})
// ── Resync from origin (git-pull-style) ────────────────────────
//
// Mirrors call this to catch up on tracker bootstrap keys their
// origin knows about but they don't. Use case: primary ran a
// migration (or created repos) AFTER the mirror's device-invite
// bundle was generated, so the mirror's origin-bundle only has
// the old tracker set. Resync connects to origin via swarm-RPC,
// calls the origin's `dump-keys`, binds any missing tracker keys
// locally, and opens the trackers so replication picks them up.
//
// UDS-only; caller is trusted (local operator).
svc.handle('resync-from-origin', async (ctx, params = {}) => {
if (ctx.transport !== 'uds') {
const err = new Error('resync-from-origin is UDS-only')
err.code = 'EAUTH'
throw err
}
const sourceKey = params.sourceServerKey || originServerKey
if (!sourceKey) {
return { error: 'no origin configured; pass { sourceServerKey }' }
}
if (typeof sourceKey !== 'string' || sourceKey.length !== 64) {
return { error: 'sourceServerKey must be 64-char hex' }
}
const { Client } = await import('hyper-svc')
let client
try {
client = await Client.connectSwarm({
serverKey: sourceKey,
topicSalt: 'pear-git-v3',
timeout: params.timeout || 15000,
// Without this a mirror can resync FROM ITSELF — binding its own ~160
// trackers and pulling its own repos, which is what made this hang.
verify: verifyIsOrigin(sourceKey)
})
} catch (err) {
return { error: 'could not reach origin: ' + err.message }
}
try {
// origin's dump-keys is UDS-only on their side. That restriction is
// intentional for HTTP callers; for swarm-RPC we need a public-ish
// peer-facing key dump. Workaround: primary should eventually ship
// a `peer-dump-keys` handler that's swarm-OK; today we try
// `dump-keys` over swarm and accept that this requires the origin
// to allow swarm callers (they do — swarm ctx is distinct from
// http).
const keys = await client.call('dump-keys', {})
// Persist the refreshed keys to <storage>/origin-bundle.json FIRST, before
// any of the tracker-binding / git-pull work below.
//
// This used to run LAST, and that ordering lost the write outright: the
// bundle costs ~1 ms and depends on nothing but `keys`, while the loop
// below opens an autobase per repo (~160 on a real mirror) and can run for
// many minutes. Measured 2026-08-08 on stortron: `dump-keys` answered in
// 18 ms, then two separate 300 s waits both gave up inside that loop, so
// the bundle — the entire point of the resync for a peer acquiring a NEW
// key — was never written. Any timeout, crash, or wedge in the heavy phase
// discarded a result that was already fully in hand.
//
// Identity (linked-device.json) and swarm (existing topic) are NOT touched
// — bundle-only refresh. A restart is what picks the new fields up.
let bundleWritten = false
try {
const bundlePath = join(opts.storage, 'origin-bundle.json')
let existing = {}
try { existing = JSON.parse(fs.readFileSync(bundlePath, 'utf8')) || {} } catch {}
const refreshed = {
...existing,
sourceServerKey: sourceKey,
orgs: (keys.orgs && keys.orgs.bootstrapKey)
? { bootstrapKey: keys.orgs.bootstrapKey }
: (existing.orgs || {}),
// Spread `existing.identity` FIRST so sibling fields survive. Writing
// `{ bootstrapKey }` alone silently DROPPED every other key on this
// object — which would have deleted `fastForwardTo` (the origin's
// `_system` core key) on the next resync, sending the mirror back to
// deriving its own system state with nothing logged.
//
// Prefer a freshly-reported fastForwardTo when the origin sends one,
// otherwise keep what we already have.
identity: {
...(existing.identity || {}),
...((keys.identity && keys.identity.bootstrapKey)
? { bootstrapKey: keys.identity.bootstrapKey }
: {}),
...((keys.identity && keys.identity.fastForwardTo)
? { fastForwardTo: keys.identity.fastForwardTo }
: {})
},
issueRegistry: keys.issues || existing.issueRegistry || { repos: {}, orgs: {} },
repoEventLog: (keys.repoEventLog && keys.repoEventLog.coreKey)
? { coreKey: keys.repoEventLog.coreKey }
: (existing.repoEventLog || {}),
orgEventLog: (keys.orgEventLog && keys.orgEventLog.coreKey)
? { coreKey: keys.orgEventLog.coreKey }
: (existing.orgEventLog || {}),
// Same lifecycle as orgEventLog: an older bundle has no release log
// key, so this refresh is how an already-paired mirror acquires it
// (then a restart begins following release events).
releaseEventLog: (keys.releaseEventLog && keys.releaseEventLog.coreKey)
? { coreKey: keys.releaseEventLog.coreKey }
: (existing.releaseEventLog || {})
}
// Atomic write: tmp + rename
const tmp = bundlePath + '.tmp'
fs.writeFileSync(tmp, JSON.stringify(refreshed, null, 2))
fs.renameSync(tmp, bundlePath)
bundleWritten = true
console.log('[resync] bundle refreshed at ' + bundlePath +
' (releaseEventLog ' + ((refreshed.releaseEventLog || {}).coreKey || 'absent') + ')')
} catch (err) {
// eslint-disable-next-line no-console
console.error('[resync] bundle write failed:', err.message)
}
const reposAdded = []
const reposAlready = []
// Mismatches: this peer's tracker root differs from origin's.
// Reported, but NOT runtime-fixed. bind*BootstrapKey is a no-op
// when a tracker is already open (runtime rebind is unsafe — it
// crashes autobase apply because the corestore-derived local
// writer carries blocks under the old apply context). Recovery
// for an actually-mismatched live mirror is `pgh device join`
// (re-pair with a fresh corestore), not resync. The 0.1.57
// strict-bootstrap mode prevents new mirrors from minting
// sibling autobases in the first place; mismatches here only
// fire on legacy / pre-0.1.57 mirrors.
const reposMismatched = []
const orgsAdded = []
const orgsAlready = []
const orgsMismatched = []
const mine = await issueRegistry.keys()
for (const [repoName, keyHex] of Object.entries(keys.issues?.repos || {})) {
const myKey = mine.repos && mine.repos[repoName]
if (myKey === keyHex) {
reposAlready.push(repoName)
continue
}
// Set the bootstrap key. Takes effect on next getForRepo IFF
// the tracker isn't already open. If it IS open with the wrong
// root, this is a no-op and we record the mismatch.
issueRegistry.bindRepoBootstrapKey(repoName, keyHex)
await issueRegistry.getForRepo(repoName)
if (myKey) reposMismatched.push({ name: repoName, mine: myKey, origin: keyHex })
else reposAdded.push(repoName)
}
// Org issues live IN the org RECORD autobase (linked-core #42), so
// there is no separate org issue tracker to bind on resync. Org
// records — and thus their issues — replicate via mountOrg from
// OrgEventLog (OrgPuller), not from a per-org issue bootstrap key.
// orgsAdded/orgsAlready/orgsMismatched stay empty (kept for the
// response shape).
// Re-announce mirror-writer keys after any newly-opened tracker.
// The boot-time announce is a one-shot; a tracker freshly opened
// here (added — not mismatched, since mismatches never actually
// rebound) wouldn't otherwise be advertised to the primary,
// leaving us not-writable on the freshly-bound canonical autobase.
if ((reposAdded.length || orgsAdded.length) &&
typeof svc._announceMirrorKeys === 'function') {
try { await svc._announceMirrorKeys() } catch (err) {
// eslint-disable-next-line no-console
console.error('[resync] re-announce failed:', err.message)
}
}
if (reposMismatched.length || orgsMismatched.length) {
// eslint-disable-next-line no-console
console.warn('[resync] mismatched trackers — runtime rebind is unsafe; recovery requires `pgh device join` from a healthy peer:',
{ reposMismatched: reposMismatched.length, orgsMismatched: orgsMismatched.length })
}
// Git-repo resync: ask origin for its repo list, git-clone any
// missing locally via syncServer.pull('primary', name) — same
// path RepoSync uses on RepoEventLog 'created' entries. Wrapped
// in a try so a unreachable origin doesn't block the (already
// successful) tracker-key portion above.
const gitAdded = []
const gitAlready = []
const gitFailed = []
let gitError = null
if (syncServer) {
try {
if (!syncServer.remotes.has('primary')) {
syncServer.addRemote('primary', sourceKey)
}
const remoteList = await client.call('list-repos', {})
const remoteRepos = (remoteList && remoteList.repos) || []
// syncServer.repos is a Map<name, info>. Use status().repos
// which is the array shape we want (same one list-repos returns).
const localStatus = syncServer.status ? syncServer.status() : { repos: [] }
const localNames = new Set((localStatus.repos || []).map(r => r.name))
// pull is clone-if-missing OR fetch-if-present (per
// hyper-sync). Always call it so already-local repos get
// their latest commits + new repos get cloned. No "skip if
// local" — that meant stortron stayed on day-old commits
// for any repo cloned earlier, even after resync.
for (const r of remoteRepos) {
const wasLocal = localNames.has(r.name)
try {
// syncServer.pull() doesn't THROW on per-repo failures —
// it returns { synced: [...], failed: [{name, error}] }.
// Old code only caught exceptions; failed pulls reported
// success. Inspect result.failed to surface real errors.
const res = await syncServer.pull('primary', r.name)
if (res && res.failed && res.failed.length > 0) {
gitFailed.push({ name: r.name, error: res.failed[0].error })
continue
}
if (wasLocal) gitAlready.push(r.name)
else gitAdded.push(r.name)
} catch (err) {
gitFailed.push({ name: r.name, error: err.message })
}
}
// pull writes to disk but doesn't update syncServer.repos
// (it's the in-memory cache used by list-repos / status). Always
// rescan after pulls so list-repos + SSR /repos sparklines /
// last-activity reflect freshly-fetched commits without a
// daemon restart.
if ((gitAdded.length > 0 || gitAlready.length > 0) && typeof syncServer.scan === 'function') {
try { await syncServer.scan() } catch {}
}
} catch (err) {
gitError = err.message
}
}
// Hot-boot OrgPuller if it wasn't running at startup (old bundle
// had no orgEventLogKey) and the refreshed keys now provide one.
// Same shape as the at-boot wiring further up. RepoPuller hot-boot
// is intentionally NOT done here — it requires syncServer remote
// bookkeeping that's harder to mutate post-init; restart suffices.
let orgPullerStarted = false
if (originCfg && !orgSync && keys.orgEventLog && keys.orgEventLog.coreKey) {
try {
const originOrgLog = new OrgEventLog({
corestore,
key: keys.orgEventLog.coreKey
})
await originOrgLog.ready()
orgSync = new OrgPuller({ issueRegistry, orgs, log: originOrgLog, supportedVersion: TARGET_VERSION, onBehind: (info) => { svc._schemaBehind = info } })
await orgSync.start()
svc._orgSync = orgSync
orgPullerStarted = true
} catch (err) {
// eslint-disable-next-line no-console
console.error('[resync] hot-boot OrgPuller failed:', err.message)
}
}
return {
source: sourceKey,
reposAdded,
reposAlready,
reposMismatched,
orgsAdded,
orgsAlready,
orgsMismatched,
gitAdded,
gitAlready,
gitFailed,
gitError,
bundleWritten,
orgPullerStarted
}
} finally {
try { await client.close() } catch {}
}
})
// Auto-import SSH keys for the owner, if an owner already exists.
// No-op on first boot when no owner is set yet — the auth-verify
// handler will re-run this immediately after TOFU grants owner.
if (svc._autoImportSSH) {
try {
const { autoImportSSHKeys } = await import('./auth/ssh-identity.js')
const r = await autoImportSSHKeys(svc, { allowRoot: svc._sshAllowRoot })
if (r.imported > 0) {
// eslint-disable-next-line no-console
console.log(`[pear-git-v3] auto-imported ${r.imported} SSH key${r.imported === 1 ? '' : 's'} for owner ${r.ownerPrincipal.slice(0, 12)}…`)
}
} catch (err) {
// eslint-disable-next-line no-console
console.error('[pear-git-v3] SSH auto-import failed:', err.message)
}
}
return {
svc,
corestore,
identity: svc.identity,
orgs,
issueRegistry,
overlayV2,
identityRegistry,
sessions,
challenges,
deploy: deployServer,
sync: syncServer,
repoEventLog,
orgEventLog,
repoSync,
async close () {
// Force the debounced /repos enrich-cache write out before anything
// else can hang the teardown. The persist timer is unref'd (it must not
// hold the process open), so without this every entry computed in the
// last second before exit is lost and re-run against git on the next
// boot — the "slow on the first load after a restart" path. Rebuildable
// derived data, so a failure here is not worth failing shutdown.
try { await flushRepoActivityCache() } catch {}
// Same reasoning for the /repos ordering key: its persist timer is
// unref'd, so a restart would otherwise order the first page on
// mtimes until the boot reconcile walk catches up.
try { await flushRefActivity() } catch {}
// Drain any in-flight auto-build jobs first so they don't write to
// the release dir or spawn subprocesses after the rest of the daemon
// tears down. buildAndPublish handles its own errors so allSettled
// never rejects here.
if (svc._pendingBuilds && svc._pendingBuilds.size > 0) {
try { await Promise.allSettled([...svc._pendingBuilds]) } catch {}
}
try { await refReconcilers.closeAll() } catch {}
try { if (repoSync) await repoSync.close() } catch {}
// Reverse RepoPullers booted for bidirectional discovery (#16).
if (svc._reversePullers instanceof Map) {
for (const p of svc._reversePullers.values()) {
try { if (p) await p.close() } catch {}
}
}
try { if (orgSync) await orgSync.close() } catch {}
// Release replication: stop the debounced reconciler, then the live log
// stream (destroying the read stream lets the follow loop unwind).
releaseFollowStopped = true
try { if (svc._releasePull) svc._releasePull.stop() } catch {}
try { if (svc._originReleaseLog) await svc._originReleaseLog.close() } catch {}
try { if (releaseEventLog) await releaseEventLog.close() } catch {}
try {
if (syncServer && syncServer._reposWatcher) {
if (syncServer._reposWatcher.cancelRescan) syncServer._reposWatcher.cancelRescan()
syncServer._reposWatcher.close()
}
} catch {}
try { if (syncServer) await syncServer.close() } catch {}
// hyper-sync does not own its injected swarm — tear it down here.
try { if (syncServer && syncServer._dedicatedSwarm) await syncServer._dedicatedSwarm.destroy() } catch {}
try { if (deployServer) await deployServer.close() } catch {}
try { await svc.close() } catch {}
try { sessions.close() } catch {}
try { challenges.destroy() } catch {}
try { await repoEventLog.close() } catch {}
try { await orgEventLog.close() } catch {}
try { await overlayV2.close() } catch {}
try { await issueRegistry.close() } catch {}
try { await orgs.close() } catch {}
try { if (typeof identityRegistry.close === 'function') await identityRegistry.close() } catch {}
try { await identityDb.close() } catch {}
try { await corestore.close() } catch {}
}
}
}
/**
* Expand a deploymentBundle (from `pgh device invite --mirror`) into the
* canonical startPearGit opts shape (mounts/mirror/swarm). Explicit opts
* on the caller still win — this only fills in fields the caller left
* unset. Bundle shape matches pgh/lib/device-linker.js and
* pear-git-v3/lib/device-linker.js:
*
* {
* orgs: { bootstrapKey },
* identity: { bootstrapKey },
* issueRegistry: { repos: {name: key}, orgs: {name: key} },
* repoEventLog: { coreKey },
* swarm: { topic },
* sourceServerKey
* }
*/
function expandDeploymentBundle (opts) {
const b = opts.deploymentBundle
if (!b || typeof b !== 'object') return opts
const mounts = { ...(opts.mounts || {}) }
if (b.orgs && b.orgs.bootstrapKey && !(mounts.orgs && mounts.orgs.bootstrapKey)) {
mounts.orgs = { bootstrapKey: b.orgs.bootstrapKey }
}
if (b.identity && b.identity.bootstrapKey && !(mounts.identity && mounts.identity.bootstrapKey)) {
// fastForwardTo = the ORIGIN's `_system` core key. Keet ships exactly this
// in every room invitation (keet-core/index.js:1991 `fastForwardTo:
// e.base.getSystemKey()`) and the joiner hands it to Autobase as `{ key }`
// (tasks/pairing/keys.js:46 → index.js:1643). Without it a joining mirror
// must DERIVE its own system state; a mirror that ends up with an empty
// indexer set derives a DIFFERENT `_system` core (the manifest is a
// function of the indexer set, autobase/lib/store.js:541-578), after which
// the origin's signed checkpoints are for a core it does not hold and it
// can never converge. Optional — older bundles lack the field and behave
// exactly as before.
mounts.identity = {
bootstrapKey: b.identity.bootstrapKey,
fastForwardTo: b.identity.fastForwardTo || null
}
}
if (b.issueRegistry) {
if (b.issueRegistry.repos && !mounts.issueRepos) {
mounts.issueRepos = { ...b.issueRegistry.repos }
}
if (b.issueRegistry.orgs && !mounts.issueOrgs) {
mounts.issueOrgs = { ...b.issueRegistry.orgs }
}
}
const origin = { ...(opts.origin || opts.mirror || {}) }
if (b.repoEventLog && b.repoEventLog.coreKey && !origin.repoEventLogKey) {
origin.repoEventLogKey = b.repoEventLog.coreKey
}
if (b.orgEventLog && b.orgEventLog.coreKey && !origin.orgEventLogKey) {
origin.orgEventLogKey = b.orgEventLog.coreKey
}
if (b.releaseEventLog && b.releaseEventLog.coreKey && !origin.releaseEventLogKey) {
origin.releaseEventLogKey = b.releaseEventLog.coreKey
}
if (b.sourceServerKey && !origin.serverKey && !origin.primaryServerKey) {
origin.serverKey = b.sourceServerKey
}
const swarm = opts.swarm === false
? opts.swarm
: { ...(opts.swarm && typeof opts.swarm === 'object' ? opts.swarm : {}) }
if (swarm !== false && b.swarm && b.swarm.topic && !swarm.topic) {
swarm.topic = b.swarm.topic
}
return { ...opts, mounts, origin, swarm }
}
/**
* Multi-autobase writer coordination via IdentityRegistry control plane.
*
* Canonical autobase pattern for enrolling a new writer (upstream README):
* an existing writer appends `{ addWriter: key }`, apply calls
* `host.addWriter(key)`. pear-git-v3 has several autobases (orgs, per-repo
* issue trackers) whose local-writer keys auto-generate per corestore
* namespace — the primary can't pre-compute a mirror's keys. We need a
* channel for the mirror to TELL the primary its keys post-boot.
*
* IdentityRegistry is that channel: the mirror is already a writer on
* it via the device-invite pre-auth (that mechanism still works for
* identity because identity uses its own identity.keyPair — the only
* autobase that does; no collision since only one uses that kp).
*
* Flow:
* - Mirror boot: collects `orgs.localKeyHex` (+ any open tracker keys)
* and appends `register-mirror-writers` to IdentityRegistry.
* - Primary (on EVERY peer, but only the primary's addWriter call
* lands): watch identityRegistry view for new `mirror-writers:*`
* entries. For each, append the canonical `{ addWriter: key }` op
* on the matching autobase. Apply calls host.addWriter.
* - Mirror peer also runs the watcher but its addWriter append is
* rejected (not a writer on orgs) → silent no-op, harmless.
*/
function setupMirrorWriterCoordination ({
svc, identityRegistry, identityHex, orgs, issueRegistry, originCfg,
syncServer = null, corestore = null, storage = null, originServerKey = null,
// Passed in, NOT read from a captured `opts`: this is a separate top-level
// function, so startPearGit's locals are not in scope here.
networkDisabled = false
}) {
const processedMirrors = new Set()
// Reverse RepoPullers this daemon has booted toward announcing peers,
// keyed by that peer's serverKey. Guards against re-booting on every
// re-announce (each carries a fresh recordedAt) — see maybeBootReverse-
// Puller (#16, bidirectional repo discovery).
const reversePullers = new Map()
if (!(svc._reversePullers instanceof Map)) svc._reversePullers = new Map()
const ownServerKey = svc && svc.identity ? svc.identity.publicKeyHex : null
const forwardOrigin = originServerKey
? String(originServerKey).toLowerCase()
: null
// Boot a reverse RepoPuller toward a peer that announced its own origin
// coordinates. This is the missing half of mirror replication: the peer
// (a mirror) publishes `__server__` (its serverKey) + `__repoeventlog__`
// (its RepoEventLog key) through the shared IdentityRegistry autobase; we
// open that log by key and pull every repo it created, so a repo made on
// EITHER daemon converges onto both. Idempotent + guarded:
// - needs syncServer + corestore wired (primary with a sync service)
// - skips our own key and the forward origin we already pull from
// - one puller per serverKey (survives re-announcements)
async function maybeBootReversePuller (serverKeyRaw, logKeyRaw, opts = {}) {
// NOTE: `opts` here is this function's OWN parameter, not the daemon's —
// use the captured networkDisabled flag. Reverse pulling clones repos from
// a peer, so it is network activity and must honour `swarm: false`.
if (networkDisabled) return
if (!syncServer || !corestore) return
if (!serverKeyRaw || !logKeyRaw) return
const serverKey = String(serverKeyRaw).toLowerCase()
const logKey = String(logKeyRaw).toLowerCase()
if (serverKey.length !== 64 || logKey.length !== 64) return
if (ownServerKey && serverKey === String(ownServerKey).toLowerCase()) return
if (forwardOrigin && serverKey === forwardOrigin) return // already pulling
if (reversePullers.has(serverKey)) return
reversePullers.set(serverKey, null) // reserve slot before any await
try {
const peerLog = new RepoEventLog({ corestore, key: logKey })
await peerLog.ready()
const remoteName = 'peer-' + serverKey.slice(0, 12)
const puller = new RepoPuller({
sync: syncServer,
issueRegistry,
log: peerLog,
primaryServerKey: serverKey,
remoteName,
// Org-scoped callers pass a filter admitting only repos in orgs this
// daemon joined; mirror callers pass none (full replica).
repoFilter: opts.repoFilter || null,
// svc._retiredRepos, NOT `retiredRepos` — this is a separate top-level
// function; that binding lives in startPearGit's scope. Referencing it
// here threw `retiredRepos is not defined` and killed every reverse
// puller boot (#16) on 0.1.178.
retired: svc._retiredRepos || null,
// Per-peer cursor so multiple reverse pullers don't race on one file.
cursorPath: storage
? join(storage, 'repo-puller-cursor-' + serverKey.slice(0, 16) + '.json')
: null
// No onMount re-announce here: the primary does not announce mirror-
// writer keys, and push/writer authority is a separate opt-in concern.
})
await puller.start()
reversePullers.set(serverKey, puller)
if (svc._reversePullers instanceof Map) svc._reversePullers.set(serverKey, puller)
// eslint-disable-next-line no-console
console.log('[mirror-writer] booted reverse repo-puller toward peer',
serverKey.slice(0, 12), '(bidirectional discovery)')
} catch (err) {
reversePullers.delete(serverKey) // allow a retry on the next announce
// eslint-disable-next-line no-console
console.error('[mirror-writer] reverse puller boot failed for',
serverKey.slice(0, 12), '-', err && err.message)
}
}
// Expose the idempotent reverse-puller boot for the org-scoped path
// (org-grouped-repos): an org peer boots a FILTERED reverse puller toward
// each co-member that announced its coords via the org autobase. Same
// per-serverKey dedup, so a full-mirror puller (if one exists for that peer)
// always wins and stays unfiltered.
svc._bootReversePuller = maybeBootReversePuller
async function processNewEntries () {
try {
for await (const entry of identityRegistry.listMirrorWriters()) {
// Boot a reverse puller from ANY peer's announced origin coordinates,
// even our own re-announce is filtered inside maybeBootReversePuller.
// Do this BEFORE the self/dup guards below so re-announcements (fresh
// recordedAt but same coordinates) still reach the idempotent boot.
const km = entry.keys || {}
if (km.__server__ && km.__repoeventlog__ && entry.devicePubkey !== identityHex) {
maybeBootReversePuller(km.__server__, km.__repoeventlog__)
}
const sig = entry.devicePubkey + ':' + entry.recordedAt
if (processedMirrors.has(sig)) continue
processedMirrors.add(sig)
if (entry.devicePubkey === identityHex) continue
// Per-entry counters for the summary at end of fan-out. One
// line per success was noisy at scale (~97 trackers) AND a
// mid-fan-out snapshot looked like partial enrollment, hiding
// that the watcher was healthy. The summary is unambiguous.
const dev = entry.devicePubkey.slice(0, 12)
let okCount = 0
let skipCount = 0
const errs = []
for (const [ns, writerKey] of Object.entries(entry.keys || {})) {
try {
if (ns.startsWith('__')) {
// Meta namespaces (e.g. `__server__` / `__repoeventlog__`, #16)
// are origin coordinates for reverse-puller boot, not autobase
// writer keys. Handled above by maybeBootReversePuller; skip
// silently here so they don't inflate the "unknown" skip count.
continue
} else if (ns.startsWith('org:')) {
// A-passive (pear-git #3): enroll the mirror as a PLAIN WRITER
// (indexer:false), NOT an indexer. Auto-indexing every mirror
// grows the indexer set unboundedly → manifest migrations that
// crash intermittent standalone peers (the ashatron ollama break).
// A writer can still push/replicate; only a bounded, designated
// set should co-sign. AND: never re-add an existing writer — a
// re-add with a changed role DEMOTES it (verified 2026-07-19),
// which is itself a migration. So SKIP if already a writer; this
// freezes existing indexer sets (zero migration) while stopping
// the growth. #3.
const orgName = ns.slice('org:'.length)
const oroom = orgs.getOrgRoom(orgName)
let already = false
try { const ws = oroom ? await oroom.listWriters() : []; already = ws.some(w => w.writerKey === writerKey) } catch {}
if (already) { skipCount++; continue }
await orgs.addWriter(orgName, 'mirror-' + dev, {
writerKey, role: 'writer', addedBy: identityHex
})
okCount++
} else if (ns.startsWith('issues:repo:') && issueRegistry) {
const repoName = ns.slice('issues:repo:'.length)
const tracker = await issueRegistry.getForRepo(repoName)
// A-passive: plain writer, and skip if already a writer (see above).
let already = false
try { const ws = await tracker.listWriters(); already = ws.some(w => w.writerKey === writerKey) } catch {}
if (already) { skipCount++; continue }
await tracker.addWriter('mirror-' + dev, {
writerKey, role: 'writer', addedBy: identityHex
})
okCount++
} else {
skipCount++
}
} catch (err) {
errs.push({ ns, err: err && err.message ? err.message : String(err) })
if (process.env.PGV3_DEBUG_MIRROR_WRITER) {
// eslint-disable-next-line no-console
console.log('[mirror-writer] addWriter on', ns, 'skipped:', err.message)
}
}
}
// eslint-disable-next-line no-console
console.log('[mirror-writer] enrolled', dev,
'on', okCount + ' namespaces' +
(skipCount > 0 ? ' (' + skipCount + ' unknown skipped)' : '') +
(errs.length > 0 ? ' (' + errs.length + ' errors)' : ''))
}
} catch (err) {
// eslint-disable-next-line no-console
console.error('[mirror-writer] watcher error:', err.message)
}
}
// Watch identityRegistry for new entries. Autobase 'update' fires
// whenever the view advances (local append or remote replication).
if (identityRegistry._base && typeof identityRegistry._base.on === 'function') {
identityRegistry._base.on('update', processNewEntries)
}
// Initial pass (picks up anything already in the view).
processNewEntries()
// Mirror-side announcement. originCfg set ⇒ this peer is a mirror.
// Announce localKeyHex per namespace once identity autobase is
// writable (the pre-auth ADD_WRITER for identity propagates via
// replication and may not land on the writable flag until after
// initial swarm sync).
//
// Includes orgs + every per-repo and per-org issue tracker that's
// open at announce time. The primary's watcher (above) calls
// tracker.addWriter(<our localKeyHex>) on each, making us writable
// on every shared tracker. Without this, the mirror gets
// "Not writable" on issue create/comment.
// See test/issues-mirror-writer.test.js for the proof.
//
// Re-entrant: extracted into _doAnnounce so the resync handler can
// re-trigger after rebinding a tracker post-boot. The first call is
// gated by `announced`; subsequent re-announces (post-rebind) bypass
// the gate by passing `force: true`.
// Failure handling here is load-bearing. `registerMirrorWriters` APPENDS to
// the identity autobase, and every append calls Autobase._bump, which is
// debounceify'd around _advance — so a failed announce that retries
// immediately drives _advance in a pure-microtask cycle that never hands the
// event loop back.
//
// Measured on stortron (mirror, 145 repos, swarm disabled): the heartbeat died
// at t=45s with _advance at 1,684, then _advance ran past 800,000 while
// _queueBump stayed flat at 427 — re-entry, not scheduling. Caller stacks
// captured at _advance #200,000 and #800,000 were identical:
// _doAnnounce -> registerMirrorWriters -> _appendBatch -> _bump -> _advance
// The daemon never printed its banner, never accept()ed a connection, and left
// exited git children unreaped (SIGCHLD also needs the macrotask phase).
//
// The trigger is `announce failed: Not writable` — a mirror whose writer key
// the origin has not enrolled yet. That is the NORMAL state on first boot after
// re-keying, and permanent when the swarm is off. Clearing `announced` in the
// catch reopened the gate with no delay and no in-flight guard, so every retry
// re-opened all 145 tracker autobases and appended again.
//
// Three guards, all required:
// - `announcing` — an in-flight attempt is never re-entered
// - backoff — a failed attempt is not retried immediately
// - setTimeout — the retry is a MACROTASK, so the loop is handed back.
// `await` alone cannot do this (see 0.1.180, 0.1.214).
const ANNOUNCE_BACKOFF_MIN = 2000
const ANNOUNCE_BACKOFF_MAX = 60000
let announced = false
let announcing = false
let announceFailedAt = 0
let announceBackoffMs = ANNOUNCE_BACKOFF_MIN
let announceRetryTimer = null
function scheduleAnnounceRetry () {
if (announceRetryTimer) return
announceRetryTimer = setTimeout(() => {
announceRetryTimer = null
_doAnnounce({ force: false }).catch(() => {})
}, announceBackoffMs)
if (announceRetryTimer && typeof announceRetryTimer.unref === 'function') {
announceRetryTimer.unref()
}
}
async function _doAnnounce ({ force = false } = {}) {
if (announced && !force) return
if (announcing) return // an attempt is already in flight — never overlap
if (announceFailedAt && (Date.now() - announceFailedAt) < announceBackoffMs) {
scheduleAnnounceRetry() // still cooling down; ensure a retry is pending
return
}
announcing = true
announced = true
try {
const keys = {}
// Per-org RECORD autobases this mirror tracks (#37). Announce our
// local writer key on each as `org:<name>` so the primary's watcher
// enrolls us (HA symmetric writer). Orgs mounted after this one-shot
// announce are picked up on the next re-announce / mirror restart —
// same limitation as trackers.
if (typeof orgs.listOrgKeys === 'function') {
for (const { name } of orgs.listOrgKeys()) {
const room = orgs.getOrgRoom(name)
if (room && room.localKeyHex) keys['org:' + name] = room.localKeyHex
}
}
if (issueRegistry) {
if (typeof issueRegistry.listAllKnownRepos === 'function') {
for (const repoName of issueRegistry.listAllKnownRepos()) {
try { await issueRegistry.getForRepo(repoName) } catch {}
}
}
// Per-repo issue trackers stay SEPARATE autobases — announce our
// local writer key on each so the primary enrolls us. Org issues
// live in the org RECORD autobase (announced as `org:<name>` above,
// linked-core #42), so there is no separate `issues:org:` key.
for (const repoName of issueRegistry.listRepos()) {
try {
const tracker = await issueRegistry.getForRepo(repoName)
if (tracker && tracker.localKeyHex) {
keys['issues:repo:' + repoName] = tracker.localKeyHex
}
} catch {}
}
}
// Also announce our OWN origin coordinates so the primary can boot a
// reverse RepoPuller toward us — this is what makes repo creation/
// discovery BIDIRECTIONAL (#16). A repo created on this mirror lands
// in our own RepoEventLog; the primary opens that log by key and pulls
// the repo, exactly mirroring the forward path. Both values are 64-char
// hex, so they satisfy registerMirrorWriters' keys validation; the
// primary's watcher treats the `__server__` / `__repoeventlog__`
// namespaces specially (boots a reverse puller) and never enrolls them
// as autobase writers.
if (svc.identity && svc.identity.publicKeyHex) {
keys.__server__ = svc.identity.publicKeyHex
}
if (svc.bootstrapKeys && svc.bootstrapKeys.repoEventLog) {
keys.__repoeventlog__ = svc.bootstrapKeys.repoEventLog
}
await identityRegistry.registerMirrorWriters(identityHex, keys)
// Success: clear the failure clock so a later re-announce starts fresh.
announceFailedAt = 0
announceBackoffMs = ANNOUNCE_BACKOFF_MIN
// eslint-disable-next-line no-console
console.log('[mirror-writer] announced local keys to primary:',
Object.keys(keys).join(', '), force ? '(re-announce)' : '')
} catch (err) {
if (!force) announced = false
// Back off before the next attempt, and make that attempt a MACROTASK.
// Without this, "Not writable" retried at microtask speed and each retry
// appended to the identity autobase — the wedge described above.
announceFailedAt = Date.now()
// eslint-disable-next-line no-console
console.error('[mirror-writer] announce failed:', err.message,
'— retrying in ' + Math.round(announceBackoffMs / 1000) + 's')
announceBackoffMs = Math.min(announceBackoffMs * 2, ANNOUNCE_BACKOFF_MAX)
scheduleAnnounceRetry()
} finally {
announcing = false
}
}
if (originCfg) {
identityRegistry.onWritable(() => { _doAnnounce({ force: false }) })
}
// Expose to caller so resync-from-origin can re-announce after rebinds.
// No-op gracefully when not a mirror (originCfg unset → nothing to
// announce, but identityRegistry isn't writable on primary anyway).
svc._announceMirrorKeys = () => _doAnnounce({ force: true })
}
/**
* Install a post-receive hook on the bare repo for `name` in
* syncServer's reposDir. The hook fires on every `git push` and
* POSTs ref-update info to this daemon's HTTP /rpc endpoint. Idempotent
* — overwrites any existing pear-git-v3 hook; leaves foreign hooks
* in place under .post-receive.bak on first install.
*/
function installPostReceiveHook (syncServer, name, svc) {
const info = syncServer.info ? syncServer.info(name) : null
const repoPath = info && info.path ? info.path : join(syncServer._reposDir, name + '.git')
const hookDir = join(repoPath, 'hooks')
const hookPath = join(hookDir, 'post-receive')
// HTTP URL the hook posts to. Read the listening port from svc.http
// (set after `await svc.listen()`). Configured-port fallback for when
// the address() probe is null. PEAR_GIT_HOOK_URL env always wins.
// No hardcoded port default — a mismatch silently breaks ref-observe
// (post-receive POSTs into the void) and leaves mirrors out of sync.
const httpServer = svc && svc.http && svc.http.server
const listeningPort = httpServer && httpServer.address && httpServer.address()
? httpServer.address().port
: null
const configuredPort = svc && svc.http && typeof svc.http._port === 'number' && svc.http._port > 0
? svc.http._port
: null
const port = listeningPort || configuredPort
if (!process.env.PEAR_GIT_HOOK_URL && !port) {
throw new Error('installPostReceiveHook: cannot determine HTTP port — svc.http not listening, no PEAR_GIT_HOOK_URL set')
}
const httpUrl = process.env.PEAR_GIT_HOOK_URL || ('http://127.0.0.1:' + port)
const marker = '# managed-by=pear-git-v3-post-receive'
const script = '#!/bin/sh\n' + marker + '\n' +
'REPO=$(basename "$(pwd)" .git)\n' +
'URL=' + JSON.stringify(httpUrl) + '/rpc/ref-update-observed\n' +
'while read oldsha newsha refname; do\n' +
' [ -z "$refname" ] && continue\n' +
' curl -s -m 5 -X POST "$URL" \\\n' +
' -H "Content-Type: application/json" \\\n' +
' --data "{\\"name\\":\\"$REPO\\",\\"ref\\":\\"$refname\\",\\"oldSha\\":\\"$oldsha\\",\\"newSha\\":\\"$newsha\\"}" \\\n' +
' >/dev/null 2>&1 || true\n' +
'done\n'
fs.mkdirSync(hookDir, { recursive: true })
// Preserve a pre-existing foreign hook on first install only.
if (fs.existsSync(hookPath)) {
let existing = ''
try { existing = fs.readFileSync(hookPath, 'utf8') } catch {}
if (!existing.includes(marker)) {
const backup = hookPath + '.bak'
if (!fs.existsSync(backup)) {
try { fs.renameSync(hookPath, backup) } catch {}
}
}
}
fs.writeFileSync(hookPath, script, { mode: 0o755 })
}