| … | … | @@ -165,6 +165,25 @@ export async function buildAndPublish ({ |
| 165 | 165 | log(onLog, `[ci] vdrive announce failed (release unaffected): ${err.message}`) |
| 166 | 166 | } |
| 167 | 167 | } |
| 168 | ||
| 169 | // Step 7 wrote _index.json and hashed this version. The vdrive step above | |
| 170 | // then REWROTE manifest.json (annotateManifest) and added locations.json, | |
| 171 | // so the index now describes a version directory that no longer exists. | |
| 172 | // Re-index and re-announce, or a mirror fetches every artifact, fails on | |
| 173 | // the manifest hash, and the version reads `incomplete` there for ever — | |
| 174 | // rendering "no files", never re-served, channel pin stranded on the | |
| 175 | // previous build. Measured in prod 2026-09-05: nine seconds between the | |
| 176 | // index write and the manifest rewrite (#18). | |
| 177 | // | |
| 178 | // Runs even when the vdrive FAILED: annotateManifest is best-effort and | |
| 179 | // may have partially written, and a redundant reindex is cheap (the hash | |
| 180 | // cache makes an unchanged artifact a stat). Non-fatal, like the rest of | |
| 181 | // step 8 — a release must not fail because its index refresh did. | |
| 182 | try { | |
| 183 | await releaseManager.reannounce(repo, version, channel) | |
| 184 | } catch (err) { | |
| 185 | log(onLog, `[ci] release re-index failed (release published, mirrors may lag): ${err.message}`) | |
| 186 | } | |
| 168 | 187 | } |
| 169 | 188 | |
| 170 | 189 | return { |
| … | … | @@ -30,6 +30,14 @@ import { |
| 30 | 30 | } from './services/release-event-log.js' |
| 31 | 31 | |
| 32 | 32 | const MANIFEST_FILE = 'manifest.json' |
| 33 | // Written into the version dir by the vdrive step (vdrive-publish.js | |
| 34 | // copyLocations) so the drive's contentHash -> (file, offset, length) map | |
| 35 | // replicates with the release. Named here because ReleaseManager owns what a | |
| 36 | // version directory may contain, and _writeIndex has to know to index it. | |
| 37 | export const VDRIVE_LOCATIONS_FILE = 'locations.json' | |
| 38 | // Files a version carries that are NOT listed in manifest.artifacts. Anything | |
| 39 | // omitted here is invisible to _writeIndex and therefore never replicates. | |
| 40 | const RESERVED_VERSION_FILES = [MANIFEST_FILE, VDRIVE_LOCATIONS_FILE] | |
| 33 | 41 | // Overlay namespace for release artifacts (the attachment spool shares this overlay). |
| 34 | 42 | export const RELEASE_OVERLAY_PREFIX = 'release:' |
| 35 | 43 | // Well-known discovery path at the release-dir root. See _writeIndex(). |
| … | … | @@ -204,6 +212,30 @@ export class ReleaseManager { |
| 204 | 212 | }, null, 2)) |
| 205 | 213 | } |
| 206 | 214 | |
| 215 | /** | |
| 216 | * Re-index a published version whose files changed out of band, and wake the | |
| 217 | * mirrors. | |
| 218 | * | |
| 219 | * Publishing is not the last thing that writes into a version directory. The | |
| 220 | * vdrive step runs afterwards: it rewrites manifest.json to carry the | |
| 221 | * pear:// link and drops locations.json beside it. Both happen AFTER publish() | |
| 222 | * hashed the version into _index.json, so without this the index advertises a | |
| 223 | * content hash for bytes that no longer exist — a mirror fetches every | |
| 224 | * artifact, fails on the manifest, and the version reads `incomplete` on the | |
| 225 | * far side for ever. Measured in prod: manifest 665 B indexed, 880 B on disk, | |
| 226 | * nine seconds apart (#18). | |
| 227 | * | |
| 228 | * Idempotent, and safe to call when nothing changed: _writeIndex keeps a | |
| 229 | * hash cache keyed on (path, size, mtime), so unchanged artifacts cost a | |
| 230 | * stat rather than a re-hash. The event is a wake-up the mirror reacts to by | |
| 231 | * re-reading the index, so a duplicate is harmless. | |
| 232 | */ | |
| 233 | async reannounce (repo, version, channel) { | |
| 234 | if (!repo || !version) throw new Error('reannounce requires { repo, version }') | |
| 235 | await this._rescan({ op: OP_PUBLISHED, repo, version, channel: channel || '' }) | |
| 236 | return { repo, version, channel: channel || '' } | |
| 237 | } | |
| 238 | ||
| 207 | 239 | async promote (repo, version, toChannel) { |
| 208 | 240 | if (!repo || !version || !toChannel) { |
| 209 | 241 | throw new Error('promote requires { repo, version, toChannel }') |
| … | … | @@ -391,7 +423,14 @@ export class ReleaseManager { |
| 391 | 423 | // it never enters that peer's own index and never gets re-served — |
| 392 | 424 | // replication would be a star with the publisher at the centre. |
| 393 | 425 | // It is a reserved artifact path, hence listed explicitly here. |
| 394 | const versionFiles = [...(v.artifacts || []).map(a => a.path), MANIFEST_FILE] | |
| 426 | // | |
| 427 | // locations.json is reserved for the same reason and was NOT listed, | |
| 428 | // which is why it never replicated despite being copied into the | |
| 429 | // version dir expressly so that it would (#18). A mirror then held every | |
| 430 | // byte and the vdrive key yet could not resolve a single block. Missing | |
| 431 | // files are skipped by the stat() below, so a release with no vdrive is | |
| 432 | // unaffected. | |
| 433 | const versionFiles = [...(v.artifacts || []).map(a => a.path), ...RESERVED_VERSION_FILES] | |
| 395 | 434 | for (const apath of versionFiles) { |
| 396 | 435 | const rel = r.repo + '/versions/' + v.version + '/' + apath |
| 397 | 436 | const abs = join(this._releaseDir, rel) |
| … | … | @@ -25,6 +25,8 @@ import fsp from '#node:fs/promises' |
| 25 | 25 | import path from '#node:path' |
| 26 | 26 | import crypto from 'hypercore-crypto' |
| 27 | 27 | |
| 28 | import { VDRIVE_LOCATIONS_FILE } from './release-manager.js' | |
| 29 | ||
| 28 | 30 | export function vdriveStoreDir (releaseDir, repo) { |
| 29 | 31 | return path.join(releaseDir, repo, '.vdrive') |
| 30 | 32 | } |
| … | … | @@ -122,8 +124,8 @@ async function annotateManifest (releaseDir, repo, version, vdrive) { |
| 122 | 124 | * Non-fatal: a release must not fail because its index could not be copied. |
| 123 | 125 | */ |
| 124 | 126 | async function copyLocations (releaseDir, repo, version) { |
| 125 | const from = path.join(vdriveStoreDir(releaseDir, repo), 'locations.json') | |
| 126 | const to = path.join(releaseDir, repo, 'versions', version, 'locations.json') | |
| 127 | const from = path.join(vdriveStoreDir(releaseDir, repo), VDRIVE_LOCATIONS_FILE) | |
| 128 | const to = path.join(releaseDir, repo, 'versions', version, VDRIVE_LOCATIONS_FILE) | |
| 127 | 129 | try { |
| 128 | 130 | await fsp.copyFile(from, to) |
| 129 | 131 | return true |
| … | … | @@ -1,6 +1,6 @@ |
| 1 | 1 | { |
| 2 | 2 | "name": "pear-git-v3", |
| 3 | "version": "0.1.251", | |
| 3 | "version": "0.1.252", | |
| 4 | 4 | "description": "P2P git forge — multi-writer issue/org/release/peer-mirror daemon. Built bottom-up after the failed v2.", |
| 5 | 5 | "type": "module", |
| 6 | 6 | "main": "index.js", |
| … | … | @@ -0,0 +1,96 @@ |
| 1 | /** | |
| 2 | * The release index must describe the version directory as it ACTUALLY IS | |
| 3 | * after the vdrive step, not as it was before. | |
| 4 | * | |
| 5 | * ci-orchestrator step 7 publishes via ReleaseManager, which writes _index.json | |
| 6 | * and hashes manifest.json. Step 8 then publishes the vdrive, and | |
| 7 | * vdrive-publish.js:annotateManifest() appends the `vdrive` block to that same | |
| 8 | * manifest while copyLocations() drops locations.json beside it. Nothing | |
| 9 | * re-indexed, so the index advertised a content hash for bytes that no longer | |
| 10 | * existed. | |
| 11 | * | |
| 12 | * Measured in prod 2026-09-05 (ifsrenderer-linux linux-port-52962c7): | |
| 13 | * | |
| 14 | * 14:50:30.106 _index.json written -> manifest.json @ 665 bytes | |
| 15 | * 14:50:39.166 manifest.json rewritten -> 880 bytes | |
| 16 | * | |
| 17 | * The mirror fetched 5 of 6 files, failed on the manifest, and without it the | |
| 18 | * version reads `incomplete` on the far side: rendered "— no files", never | |
| 19 | * re-served, channel pin stranded on the previous build. See pear-git-v3 #18. | |
| 20 | */ | |
| 21 | import test from 'brittle' | |
| 22 | import { mkdtemp, readFile, writeFile, stat } from '#node:fs/promises' | |
| 23 | import { join } from '#node:path' | |
| 24 | import { tmpdir } from '#node:os' | |
| 25 | import { hashChunk } from 'hyper-overlay/chunker' | |
| 26 | ||
| 27 | import { ReleaseManager, VDRIVE_LOCATIONS_FILE } from '../lib/release-manager.js' | |
| 28 | ||
| 29 | const REPO = 'demo-repo' | |
| 30 | const VERSION = 'linux-port-deadbee' | |
| 31 | ||
| 32 | async function published () { | |
| 33 | const dir = await mkdtemp(join(tmpdir(), 'pgv3-relidx-')) | |
| 34 | const rm = new ReleaseManager({ releaseDir: dir }) | |
| 35 | await rm.publish({ | |
| 36 | repo: REPO, | |
| 37 | version: VERSION, | |
| 38 | channel: 'experimental', | |
| 39 | artifacts: [{ path: 'dist/app.AppImage', data: Buffer.from('an artifact') }] | |
| 40 | }) | |
| 41 | return { dir, rm } | |
| 42 | } | |
| 43 | ||
| 44 | /** What the vdrive step does to a version directory AFTER it was published. */ | |
| 45 | async function runVdriveStep (dir) { | |
| 46 | const vdir = join(dir, REPO, 'versions', VERSION) | |
| 47 | const manifestPath = join(vdir, 'manifest.json') | |
| 48 | const raw = JSON.parse(await readFile(manifestPath, 'utf8')) | |
| 49 | raw.vdrive = { link: 'pear://xyzzy', key: 'ab'.repeat(32), blocks: 1839, locations: 1217 } | |
| 50 | await writeFile(manifestPath, JSON.stringify(raw, null, 2)) | |
| 51 | await writeFile(join(vdir, VDRIVE_LOCATIONS_FILE), JSON.stringify({ 0: 'x' })) | |
| 52 | } | |
| 53 | ||
| 54 | const indexed = async (dir) => | |
| 55 | JSON.parse(await readFile(join(dir, '_index.json'), 'utf8')).files || [] | |
| 56 | ||
| 57 | test('the index describes the manifest as it is on disk after the vdrive step', async (t) => { | |
| 58 | const { dir, rm } = await published() | |
| 59 | await runVdriveStep(dir) | |
| 60 | ||
| 61 | await rm.reannounce(REPO, VERSION, 'experimental') | |
| 62 | ||
| 63 | const rel = REPO + '/versions/' + VERSION + '/manifest.json' | |
| 64 | const entry = (await indexed(dir)).find(f => f.path === rel) | |
| 65 | t.ok(entry, 'manifest.json is indexed') | |
| 66 | ||
| 67 | const abs = join(dir, rel) | |
| 68 | t.is(entry.size, (await stat(abs)).size, | |
| 69 | 'indexed size matches disk — a stale size means a mirror fetches a hash nothing holds') | |
| 70 | t.is(entry.hash, hashChunk(await readFile(abs)), 'indexed hash matches disk') | |
| 71 | }) | |
| 72 | ||
| 73 | test('locations.json is indexed, so the vdrive index can actually replicate', async (t) => { | |
| 74 | const { dir, rm } = await published() | |
| 75 | await runVdriveStep(dir) | |
| 76 | ||
| 77 | await rm.reannounce(REPO, VERSION, 'experimental') | |
| 78 | ||
| 79 | const rel = REPO + '/versions/' + VERSION + '/' + VDRIVE_LOCATIONS_FILE | |
| 80 | const entry = (await indexed(dir)).find(f => f.path === rel) | |
| 81 | ||
| 82 | // Copied into the version dir explicitly "so it replicates". Without an index | |
| 83 | // entry a mirror holds every byte and the vdrive key yet cannot resolve a | |
| 84 | // single block — the exact failure that copy exists to prevent. | |
| 85 | t.ok(entry, 'locations.json is in the index') | |
| 86 | if (entry) t.is(entry.hash, hashChunk(await readFile(join(dir, rel))), 'and its hash matches') | |
| 87 | }) | |
| 88 | ||
| 89 | test('a version with no vdrive is unaffected — the sidecar is simply absent', async (t) => { | |
| 90 | const { dir, rm } = await published() | |
| 91 | ||
| 92 | const files = await indexed(dir) | |
| 93 | const rel = REPO + '/versions/' + VERSION + '/' + VDRIVE_LOCATIONS_FILE | |
| 94 | t.absent(files.find(f => f.path === rel), 'no phantom entry for a file that does not exist') | |
| 95 | t.ok(files.find(f => f.path.endsWith('/manifest.json')), 'manifest still indexed') | |
| 96 | }) |