/**
* Publish a released version into the repo's virtual Hyperdrive.
*
* Opt-in per repo via `.pear-git/ci.yaml`:
*
* vdrive: true
* # or
* vdrive:
* channels: [production] # only these channels go into the drive
*
* ReleaseManager has already written the artifacts as plain files under
* <releaseDir>/<repo>/versions/<version>/. That directory IS the byte store —
* the drive keeps a 32-byte pointer per block and reads the bytes back from
* those files on demand, so a drive costs roughly 0.1% of what it publishes.
*
* ONE drive per repo, republished per version — not a drive per version. The
* pear:// link stays valid across releases and resolves to the newest, which
* is what makes it usable as a distribution link. Per-version pinning is still
* available through the verlink form that `pear install` prints.
*
* Failures are NON-FATAL: a release must not fail because its drive did.
*/
import fsp from '#node:fs/promises'
import path from '#node:path'
import crypto from 'hypercore-crypto'
import { VDRIVE_LOCATIONS_FILE } from './release-manager.js'
export function vdriveStoreDir (releaseDir, repo) {
return path.join(releaseDir, repo, '.vdrive')
}
// Namespace so this can never collide with another derivation off the same root.
const VDRIVE_NS = 'pear-git-v3/vdrive/1'
/**
* Derive a drive's corestore primaryKey from durable state.
*
* The drive key derives from the store's primaryKey, and that primaryKey used to
* be RANDOM — minted by corestore on first open and persisted only inside
* `<releaseDir>/<repo>/.vdrive`. So losing that directory destroyed the pear://
* link permanently: nothing recorded the secret and nothing could regenerate it.
* (Upstream `pear touch` has the same shape — a random name nobody records.)
*
* Deriving it instead from the daemon's long-lived corestore primaryKey plus the
* repo name makes the link a pure function of state that already survives —
* backed up, replicated, and far more durable than a per-repo scratch store.
* Losing the drive store becomes a rebuild rather than data loss.
*
* Per-machine by design. Two daemons have different roots, so this does NOT make
* the link reproducible across the fleet — that is the multisig question (#145),
* deliberately deferred. What it buys is recoverability, which is the part that
* was actually urgent.
*/
// corestore's own wording when a store's persisted seed differs from the
// primaryKey it was handed (corestore/index.js `_open`). Matched on the message
// because it throws a plain Error with no code.
function isKeyMismatch (err) {
return /another corestore is stored here/i.test((err && err.message) || '')
}
export function deriveVdrivePrimaryKey (rootPrimaryKey, repo) {
if (!rootPrimaryKey || rootPrimaryKey.byteLength !== 32) {
throw new Error('deriveVdrivePrimaryKey requires a 32-byte root primaryKey')
}
if (!repo) throw new Error('deriveVdrivePrimaryKey requires a repo name')
return crypto.hash([
Buffer.from(VDRIVE_NS),
rootPrimaryKey,
Buffer.from(repo)
])
}
/**
* Record the drive in the version's manifest.json.
*
* The link is otherwise unrecoverable: the manifest never carried it, the
* deploy log never printed it, and once the seeder holds the store the only
* trace is a daemon log line. Writing it here makes `pgh deploy log` and any
* manifest reader able to answer "what is this release's pear:// link".
*
* Non-fatal: a manifest we cannot annotate must not fail a published release.
*/
async function annotateManifest (releaseDir, repo, version, vdrive) {
const manifestPath = path.join(releaseDir, repo, 'versions', version, 'manifest.json')
try {
const raw = JSON.parse(await fsp.readFile(manifestPath, 'utf8'))
raw.vdrive = {
link: vdrive.link,
key: vdrive.key,
blocks: vdrive.blocks,
locations: vdrive.locations
}
await fsp.writeFile(manifestPath, JSON.stringify(raw, null, 2))
return true
} catch {
return false
}
}
/**
* Copy the drive's locations index into the version dir, so it replicates.
*
* A vdrive stores `index -> contentHash`; serving needs `contentHash -> (file,
* offset, length)`. That index lives beside the drive, at
* `<releaseDir>/<repo>/.vdrive/locations.json` — outside the version dir, so it
* is not part of what replicates with a release. The consequence measured on the
* fleet: stortron holds every `ifsrenderer-linux` version and the key (which
* rides in manifest.json), yet cannot resolve a single block, so seeding a drive
* it could serve for ~281KB would cost a second full copy of every byte.
*
* A COPY, not a move. The index is per-drive state and the origin needs it in
* place: `publishTree` reads it back to seed dedup on the next republish, and
* `openDrive` reads it to serve. The version-dir copy is a replication vehicle,
* not the working copy.
*
* The index is cumulative — publishTree loads the prior one, adds this version's
* chunks, and saves the union — so each copy is a snapshot of the whole drive at
* that point, and the NEWEST version's copy is sufficient on its own. Paths are
* already relative to `<releaseDir>/<repo>`, which the version dir sits under, so
* the bytes are portable exactly as written.
*
* Non-fatal: a release must not fail because its index could not be copied.
*/
async function copyLocations (releaseDir, repo, version) {
const from = path.join(vdriveStoreDir(releaseDir, repo), VDRIVE_LOCATIONS_FILE)
const to = path.join(releaseDir, repo, 'versions', version, VDRIVE_LOCATIONS_FILE)
try {
await fsp.copyFile(from, to)
return true
} catch {
return false
}
}
async function dirSize (dir) {
let total = 0
for (const e of await fsp.readdir(dir, { withFileTypes: true })) {
const full = path.join(dir, e.name)
if (e.isDirectory()) total += await dirSize(full)
else if (e.isFile()) total += (await fsp.stat(full)).size
}
return total
}
/**
* Byte-wise sort of drive paths.
*
* publishTree writes files in exactly the order it is given, and that order
* lands in the drive's merkle tree — measured on the real path against a
* 6-file release: identical bytes in reversed order produced tree dacbc54b…
* instead of 0997c75b…, same 1844 blocks. So the order must come from the
* names, never from the filesystem.
*
* Byte-wise rather than locale-aware, matching hyper-overlay's tree format
* ("Entries MUST be sorted byte-wise by name"). localeCompare would order
* 'Build.log' after 'aaa.txt' on some locales and before it on others, which
* is the same class of bug one layer up.
*/
export function sortDriveNames (names) {
return [...names].sort((a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b)))
}
// Exported for test/vdrive-walk-order.test.js — the ordering this produces is
// baked into the drive's merkle tree, so it needs a direct test.
export { walk as walkForTest }
async function walk (dir, base = dir, out = []) {
await collect(dir, base, out)
// readdir order is filesystem-defined (hash order on ext4, and unstable
// across filesystems and directories), so it must not reach publishTree.
return sortDriveNames(out)
}
async function collect (dir, base, out) {
for (const entry of await fsp.readdir(dir, { withFileTypes: true })) {
const full = path.join(dir, entry.name)
if (entry.isDirectory()) await collect(full, base, out)
else if (entry.isFile()) out.push('/' + path.relative(base, full).split(path.sep).join('/'))
}
return out
}
/**
* @param {object} opts
* @param {string} opts.releaseDir ReleaseManager's release root
* @param {string} opts.repo
* @param {string} opts.version the version just published
* @param {string} [opts.channel]
* @param {object} opts.vdrive normalised config from loadCIConfig
* @param {function} [opts.chunk] injectable chunker (tests)
* @param {function} [opts.onLog]
*/
export async function publishVersionToDrive ({
releaseDir,
repo,
version,
channel,
vdrive,
chunk,
primaryKey,
onLog
} = {}) {
if (!vdrive || !vdrive.enabled) return { ok: false, skipped: true, reason: 'not enabled' }
if (!releaseDir || !repo || !version) {
throw new Error('publishVersionToDrive requires { releaseDir, repo, version }')
}
if (vdrive.channels && vdrive.channels.length > 0) {
if (!channel || !vdrive.channels.includes(channel)) {
return {
ok: false,
skipped: true,
reason: 'channel not selected for vdrive: ' + (channel || '(none)')
}
}
}
// Lazily resolved so a missing or incompatible module degrades to "no drive"
// instead of taking the release path down with it.
let publishTree
try {
const mod = await import('hyper-vdrive/publish')
publishTree = mod.publishTree || (mod.default && mod.default.publishTree)
if (typeof publishTree !== 'function') throw new Error('publishTree not exported')
} catch (err) {
return { ok: false, skipped: true, reason: 'hyper-vdrive unavailable: ' + err.message }
}
const chunker = chunk || (await import('hyper-overlay/chunker')).chunk
const versionDir = path.join(releaseDir, repo, 'versions', version)
const names = await walk(versionDir)
if (names.length === 0) return { ok: false, skipped: true, reason: 'no artifacts to publish' }
let result
try {
result = await publishTree({
storeDir: vdriveStoreDir(releaseDir, repo),
// locations are recorded relative to the repo's release root, so a seeder
// can resolve them later without re-chunking anything
root: path.join(releaseDir, repo),
files: names.map((name) => ({ name, file: path.join(versionDir, name) })),
chunk: chunker,
// `unsafe` is corestore's required acknowledgement for supplying a
// primaryKey at all. It is exactly what we want here: the key is the point.
corestoreOpts: primaryKey ? { primaryKey, unsafe: true } : undefined
})
} catch (err) {
// ONLY the key-mismatch case becomes a skip. corestore refuses to open a
// store whose persisted seed differs from the primaryKey it was handed
// ("Another corestore is stored here") rather than silently rekeying it,
// which would change the pear:// link without saying so. That is permanent
// and needs a deliberate operator action, so it is reported, not thrown.
//
// Everything else must keep propagating — in particular the seeder's
// exclusive store lock ("File descriptor could not be locked"), which is
// TRANSIENT and is what drives the orchestrator's drop/publish/re-announce
// sequence. Swallowing that into a skip would hide a condition the caller
// resolves by retrying. See test/vdrive-seeder.test.js.
if (!isKeyMismatch(err)) throw err
return {
ok: false,
skipped: true,
reason:
'drive store does not match the derived key (' + err.message + '). ' +
'Move ' + vdriveStoreDir(releaseDir, repo) + ' aside to mint a new drive.'
}
}
const annotated = await annotateManifest(releaseDir, repo, version, result)
const locationsCopied = await copyLocations(releaseDir, repo, version)
if (onLog) {
// Report the pointer/literal split and the store size. Without them a large
// .vdrive is ambiguous — it reads the same whether the redirect failed (all
// literal) or the store merely needs compacting, and those want opposite
// responses. Once the seeder holds the lock you cannot go back and measure.
let storeBytes = null
try {
storeBytes = await dirSize(vdriveStoreDir(releaseDir, repo))
} catch {}
onLog(
`[ci] vdrive ${repo} ${version}: ${result.files} files, ${result.blocks} blocks ` +
`(${result.pointerBlocks} pointer, ${result.literalBlocks} literal), ` +
`${result.locations} locations` +
(storeBytes === null ? '' : `, store ${(storeBytes / 1048576).toFixed(1)}MB`) +
(result.compacted === false ? ' [compaction failed]' : '') +
` -> ${result.link}`
)
}
return { ok: true, ...result, annotated, locationsCopied }
}