/**
* CI orchestrator — turns a git ref into a published release.
*
* Flow for a single build:
*
* 1. Export the ref's tree from the bare repo into a temp checkout dir
* via `git archive | tar -x`.
* 2. Load + validate `.pear-git/ci.yaml` (skip build if missing).
* 3. Pick the channel for this ref (tag → production, branch → experimental, …).
* 4. Derive the version name (tag verbatim or `<branch>-<shortSha>`).
* 5. Run steps sequentially — abort on first failure.
* 6. Collect artifacts named by the yaml + a build.log aggregating stdout.
* 7. Publish via ReleaseManager (which also triggers hyper-deploy rescan).
*
* Idempotent: if a version already exists for this ref, the orchestrator
* errors cleanly rather than silently overwriting. Operators can delete
* the existing version first if they really want to republish.
*/
import process from '#node:process'
import fs from '#node:fs/promises'
import path from '#node:path'
import os from '#node:os'
import { spawn } from '#node:child_process'
import { loadCIConfig, pickChannel, deriveVersion, runSteps, collectArtifacts } from './ci-runner.js'
import { publishVersionToDrive, deriveVdrivePrimaryKey } from './vdrive-publish.js'
// Releases kept per repo after each CI publish (step 9). PGV3_KEEP_RELEASES=0 disables.
const keepEnv = parseInt(process.env.PGV3_KEEP_RELEASES, 10)
const DEFAULT_KEEP_RELEASES = Number.isFinite(keepEnv) && keepEnv >= 0 ? keepEnv : 2
export async function buildAndPublish ({
sync,
releaseManager,
repo,
ref,
isTag = false,
version: forcedVersion,
channel: forcedChannel,
vdriveSeeder,
vdriveRootKey = null,
keepReleases = DEFAULT_KEEP_RELEASES,
onLog
} = {}) {
if (!sync) throw new Error('buildAndPublish requires sync service')
if (!releaseManager) throw new Error('buildAndPublish requires release manager')
if (!repo || !ref) throw new Error('buildAndPublish requires { repo, ref }')
const info = sync.info(repo)
if (!info) throw new Error('repo not found: ' + repo)
const repoDir = info.path
// 1. Resolve ref → SHA so versioning is stable even if the ref moves.
// Also compute commit count from the bare repo — CI steps run in a
// `git archive | tar -x` extraction which has no .git/, so any
// in-step `git rev-list` returns empty. Expose both via env vars
// instead so steps can derive their own version strings.
const sha = await gitRevParse(repoDir, ref)
let commitCount = ''
try { commitCount = await gitRevListCount(repoDir, sha) } catch {}
// 2. Export tree into a fresh temp dir
const checkoutDir = await fs.mkdtemp(path.join(os.tmpdir(), 'pear-git-ci-' + repo + '-'))
try {
log(onLog, `[ci] exporting ${repo}@${sha.slice(0, 7)} to ${checkoutDir}`)
await gitExport(repoDir, sha, checkoutDir)
// 3. Load config
const config = loadCIConfig(checkoutDir)
if (!config) {
return { ok: false, skipped: true, reason: 'no .pear-git/ci.yaml — skipping build' }
}
// 4. Pick channel + derive version
const channel = forcedChannel ?? pickChannel(config, ref, { isTag })
const version = forcedVersion ?? deriveVersion({ ref, sha, isTag })
log(onLog, `[ci] building ${repo} version=${version} channel=${channel || '(none)'}`)
// 5. Run steps. Inject git metadata so steps can fabricate semver
// versions or build identifiers without re-running git.
const stepEnv = {
GIT_COMMIT_SHA: sha,
GIT_COMMIT_SHORT: sha.slice(0, 8),
GIT_COMMIT_COUNT: commitCount,
GIT_REF: ref,
GIT_REF_KIND: isTag ? 'tag' : 'branch'
}
const runResult = await runSteps(checkoutDir, config, { repoName: repo, channel, version, env: stepEnv })
for (const s of runResult.steps) {
log(onLog, `[ci] step ${s.name}: ${s.status} (${s.duration}ms)`)
}
// Aggregate step output into a build.log artifact regardless of success
const buildLog = runResult.steps.map(s =>
`=== ${s.name} (${s.status}, ${s.duration}ms) ===\n${s.output}${s.error ? '\n[error] ' + s.error : ''}`
).join('\n\n')
if (!runResult.ok) {
return {
ok: false,
version,
channel,
sha,
steps: runResult.steps,
buildLog,
reason: 'build failed — see steps[].error'
}
}
// 6. Collect artifacts (+ always include build.log)
const artifacts = await collectArtifacts(checkoutDir, config)
artifacts.push({ path: 'build.log', data: Buffer.from(buildLog, 'utf8') })
// 7. Publish
log(onLog, `[ci] publishing ${artifacts.length} artifact${artifacts.length === 1 ? '' : 's'} to ${version}${channel ? ' / ' + channel : ''}`)
const published = await releaseManager.publish({
repo,
version,
channel,
artifacts,
meta: {
fromCommit: sha,
fromRef: ref,
refKind: isTag ? 'tag' : 'branch',
buildMs: runResult.totalMs,
builtBy: 'ci-orchestrator'
}
})
// 8. Optional: publish the same artifacts into the repo's virtual
// Hyperdrive, giving the repo a durable pear:// link. Non-fatal —
// a release must not fail because its drive did.
let vdrive = null
if (config.vdrive && config.vdrive.enabled && releaseManager.releaseDir) {
try {
// The seeder holds the drive's store with an exclusive lock for as long
// as it is announced, and a second opener gets "File descriptor could
// not be locked". So the SECOND publish for a repo cannot write its
// drive unless the seeder lets go first. Drop, publish, re-announce.
// The drive is unserved for that window, which is the cost of writing it.
if (vdriveSeeder) await vdriveSeeder.dropRepo(repo)
vdrive = await publishVersionToDrive({
releaseDir: releaseManager.releaseDir,
repo,
version,
channel,
vdrive: config.vdrive,
// Derived from the daemon's long-lived corestore, so losing the drive
// store is a rebuild rather than a permanently dead link (#145).
primaryKey: vdriveRootKey ? deriveVdrivePrimaryKey(vdriveRootKey, repo) : null,
onLog
})
if (vdrive.skipped) log(onLog, `[ci] vdrive skipped: ${vdrive.reason}`)
} catch (err) {
vdrive = { ok: false, error: err.message }
log(onLog, `[ci] vdrive failed (release unaffected): ${err.message}`)
}
// Re-announce unconditionally. On success this is what makes a freshly
// published drive reachable — the boot scan cannot have seen a drive that
// did not exist yet. On FAILURE it puts back what the drop above took
// away, so a bad publish does not also stop serving the previous release.
if (vdriveSeeder) {
try {
const seeded = await vdriveSeeder.announceRepo(repo)
if (seeded) log(onLog, `[ci] vdrive seeding ${repo} -> ${seeded.link}`)
} catch (err) {
log(onLog, `[ci] vdrive announce failed (release unaffected): ${err.message}`)
}
}
// Step 7 wrote _index.json and hashed this version. The vdrive step above
// then REWROTE manifest.json (annotateManifest) and added locations.json,
// so the index now describes a version directory that no longer exists.
// Re-index and re-announce, or a mirror fetches every artifact, fails on
// the manifest hash, and the version reads `incomplete` there for ever —
// rendering "no files", never re-served, channel pin stranded on the
// previous build. Measured in prod 2026-09-05: nine seconds between the
// index write and the manifest rewrite (#18).
//
// Runs even when the vdrive FAILED: annotateManifest is best-effort and
// may have partially written, and a redundant reindex is cheap (the hash
// cache makes an unchanged artifact a stat). Non-fatal, like the rest of
// step 8 — a release must not fail because its index refresh did.
try {
await releaseManager.reannounce(repo, version, channel)
} catch (err) {
log(onLog, `[ci] release re-index failed (release published, mirrors may lag): ${err.message}`)
}
}
// 9. Collect this repo's older releases: newest `keepReleases` stay, and
// channel-pinned versions always stay (ReleaseManager.gc). gc-releases
// existed but nothing ran it — tabertron's release dir held 453 stale
// versions (11 GB) on 2026-10-11. Non-fatal; 0 disables.
// gc's keepN counts UNPINNED versions, and this build just pinned its
// channel — so with a channel, keepN = keepReleases - 1 leaves exactly
// keepReleases: the new build + the previous one(s).
if (keepReleases > 0) {
try {
const gc = await releaseManager.gc({ repo, keepN: channel ? keepReleases - 1 : keepReleases })
if (gc.deleted.length) log(onLog, `[ci] collected ${gc.deleted.length} older release(s) of ${repo} (keep ${keepReleases})`)
} catch (err) {
log(onLog, `[ci] release gc failed (release unaffected): ${err.message}`)
}
}
return {
ok: true,
version,
channel,
sha,
steps: runResult.steps,
buildLog,
manifest: published.manifest,
vdrive
}
} finally {
// Clean up the temp checkout — artifacts have already been read
await fs.rm(checkoutDir, { recursive: true, force: true }).catch(() => {})
}
}
function log (onLog, line) {
if (typeof onLog === 'function') onLog(line)
// eslint-disable-next-line no-console
else if (process.env.DEBUG_CI) console.log(line)
}
/**
* Snapshot the SHA of every ref in the bare repo. Format: Map<refname, sha>.
* Used by autoBuildFromPush to diff before/after a `git-receive-pack`.
*/
export async function snapshotRefs (repoDir) {
return new Promise((resolve, reject) => {
const proc = spawn('git', ['-C', repoDir, 'for-each-ref', '--format=%(refname) %(objectname)'], {
stdio: ['ignore', 'pipe', 'pipe']
})
const chunks = []
const errChunks = []
proc.stdout.on('data', c => chunks.push(c))
proc.stderr.on('data', c => errChunks.push(c))
proc.on('error', reject)
proc.on('exit', (code) => {
if (code !== 0) return reject(new Error('for-each-ref failed: ' + Buffer.concat(errChunks).toString('utf-8').trim()))
const out = new Map()
for (const line of Buffer.concat(chunks).toString('utf-8').split('\n')) {
if (!line.trim()) continue
const space = line.indexOf(' ')
if (space < 0) continue
out.set(line.slice(0, space), line.slice(space + 1).trim())
}
resolve(out)
})
})
}
/**
* Given a before/after ref snapshot pair, run buildAndPublish for every
* ref whose SHA changed. Fire-and-forget — each build runs in the
* background; errors are logged to stderr but do not affect the HTTP
* response for the triggering git push.
*
* Only refs under refs/heads/ (branches) and refs/tags/ (tags) are
* considered — refs/stash, refs/notes, etc. are ignored.
*/
export async function autoBuildFromPush ({ sync, releaseManager, repo, before, after, onLog, pendingBuilds, vdriveSeeder, vdriveRootKey = null } = {}) {
if (!sync || !releaseManager) return { triggered: [] }
const triggered = []
// Determine changed refs
for (const [refname, newSha] of after) {
const oldSha = before.get(refname)
if (oldSha === newSha) continue
if (!refname.startsWith('refs/heads/') && !refname.startsWith('refs/tags/')) continue
const isTag = refname.startsWith('refs/tags/')
const ref = refname.replace(/^refs\/(heads|tags)\//, '')
triggered.push({ ref, isTag, oldSha: oldSha || null, newSha })
}
// Fire each build fire-and-forget so the caller (HTTP push handler) can
// return the 200 response immediately. Failures are logged, not thrown.
//
// If pendingBuilds is provided (a Set), track the promise there so
// graceful shutdown can `Promise.allSettled([...pendingBuilds])` before
// closing subsystems, preventing orphaned file handles.
for (const t of triggered) {
const p = buildAndPublish({
sync,
releaseManager,
vdriveSeeder,
vdriveRootKey,
repo,
ref: t.ref,
isTag: t.isTag,
onLog: onLog || ((line) => console.log('[auto-build][' + repo + '] ' + line))
}).then(result => {
if (result.skipped) {
log(onLog, `[auto-build][${repo}] ${t.ref}: ${result.reason}`)
} else if (result.ok) {
log(onLog, `[auto-build][${repo}] ${t.ref}: published ${result.version}${result.channel ? ' → ' + result.channel : ''}`)
} else {
log(onLog, `[auto-build][${repo}] ${t.ref}: BUILD FAILED — ${result.reason || 'unknown'}`)
}
}).catch(err => {
// eslint-disable-next-line no-console
console.error(`[auto-build][${repo}] ${t.ref}: orchestrator error — ${err.message}`)
}).finally(() => {
if (pendingBuilds) pendingBuilds.delete(p)
})
if (pendingBuilds) pendingBuilds.add(p)
}
return { triggered }
}
function gitRevParse (repoDir, ref) {
return new Promise((resolve, reject) => {
const proc = spawn('git', ['-C', repoDir, 'rev-parse', ref], {
stdio: ['ignore', 'pipe', 'pipe']
})
const chunks = []
const errChunks = []
proc.stdout.on('data', c => chunks.push(c))
proc.stderr.on('data', c => errChunks.push(c))
proc.on('error', reject)
proc.on('exit', (code) => {
if (code === 0) {
resolve(Buffer.concat(chunks).toString('utf-8').trim())
} else {
reject(new Error('git rev-parse failed: ' + Buffer.concat(errChunks).toString('utf-8').trim()))
}
})
})
}
function gitRevListCount (repoDir, sha) {
return new Promise((resolve, reject) => {
const proc = spawn('git', ['-C', repoDir, 'rev-list', '--count', sha], {
stdio: ['ignore', 'pipe', 'pipe']
})
const chunks = []
const errChunks = []
proc.stdout.on('data', c => chunks.push(c))
proc.stderr.on('data', c => errChunks.push(c))
proc.on('error', reject)
proc.on('exit', (code) => {
if (code === 0) {
resolve(Buffer.concat(chunks).toString('utf-8').trim())
} else {
reject(new Error('git rev-list --count failed: ' + Buffer.concat(errChunks).toString('utf-8').trim()))
}
})
})
}
function gitExport (repoDir, sha, outDir) {
return new Promise((resolve, reject) => {
// Use `git archive | tar -x` so we don't need the destination to be
// a full clone. Works with bare repos (which pear-git-v3 uses).
const archive = spawn('git', ['-C', repoDir, 'archive', '--format=tar', sha], {
stdio: ['ignore', 'pipe', 'pipe']
})
// Pipe archive.stdout → tar.stdin explicitly. Passing a stream as a child's
// stdio entry (stdio: [archive.stdout, ...]) is a Node-only idiom that
// bare-subprocess does NOT support — under Bare tar would get no input
// ("does not look like a tar archive"). Explicit .pipe works on both.
const tar = spawn('tar', ['-x', '-C', outDir], {
stdio: ['pipe', 'inherit', 'pipe']
})
archive.stdout.pipe(tar.stdin)
const errChunks = []
archive.stderr.on('data', c => errChunks.push(c))
tar.stderr.on('data', c => errChunks.push(c))
archive.on('error', reject)
tar.on('error', reject)
tar.on('exit', (code) => {
if (code === 0) resolve()
else reject(new Error('git archive | tar failed: ' + Buffer.concat(errChunks).toString('utf-8').trim()))
})
})
}