| … | … | @@ -391,15 +391,26 @@ export class ReleaseManager { |
| 391 | 391 | } |
| 392 | 392 | } |
| 393 | 393 | |
| 394 | const index = { schema: 1, generatedAt: Date.now(), files, channels } | |
| 395 | await writeFile(join(this._releaseDir, INDEX_FILE), JSON.stringify(index, null, 2)) | |
| 396 | ||
| 397 | 394 | // Make every artifact fetchable BY CONTENT HASH over the V2 overlay. The |
| 398 | 395 | // overlay is auto-attached to every hyper-svc swarm connection, so this is |
| 399 | 396 | // what lets a peer actually pull the bytes the index advertises — without |
| 400 | 397 | // it the index names files nobody can fetch. Namespaced so release |
| 401 | 398 | // artifacts cannot collide with the attachment spool sharing this overlay. |
| 402 | 399 | // |
| 400 | // BEFORE the index is written, and that ordering is load-bearing. `_rescan` | |
| 401 | // appends the release-event that wakes every mirror immediately after this | |
| 402 | // returns, and a mirror reacts in milliseconds — so anything the index names | |
| 403 | // but the overlay has not been told about yet is answered "no holder". The | |
| 404 | // window is real work, not a scheduling hiccup: registerFile chunks and | |
| 405 | // hashes the file, which for a 49 MB AppImage is far longer than a mirror | |
| 406 | // needs to read the index and start asking. | |
| 407 | // | |
| 408 | // Measured on stortron 2026-08-17 (ifsrenderer-linux linux-port-14855c7): | |
| 409 | // the artifact registered FIRST arrived, and the three after it — including | |
| 410 | // a 508-byte manifest.json — all failed with "no holder", stranding the | |
| 411 | // version and leaving the channel pinned to the previous release. Size was | |
| 412 | // irrelevant; position in this loop was everything. | |
| 413 | // | |
| 403 | 414 | // Only NEW-or-CHANGED entries are registered. _writeIndex runs on every |
| 404 | 415 | // mutation, and a mutation happens on every push-triggered build; a forge |
| 405 | 416 | // with 558 versions would otherwise issue ~560 overlay index writes per |
| … | … | @@ -422,6 +433,9 @@ export class ReleaseManager { |
| 422 | 433 | this.lastRegistered = registered |
| 423 | 434 | } |
| 424 | 435 | |
| 436 | const index = { schema: 1, generatedAt: Date.now(), files, channels } | |
| 437 | await writeFile(join(this._releaseDir, INDEX_FILE), JSON.stringify(index, null, 2)) | |
| 438 | ||
| 425 | 439 | return index |
| 426 | 440 | } |
| 427 | 441 |
| … | … | @@ -328,7 +328,18 @@ export class ReleasePuller { |
| 328 | 328 | const d = delta() |
| 329 | 329 | for (const e of d.errors) this._log('[release-pull] error: ' + e) |
| 330 | 330 | |
| 331 | return { ok: true, plan, delta: d, stats: { ...this.stats } } | |
| 331 | // PARTIAL is not success. This returned a bare { ok: true } however many | |
| 332 | // individual files failed, and reconcile() re-queues only on `!ok` — so a | |
| 333 | // pull that fetched one file of four reset the failure counter, drained the | |
| 334 | // pending set, and stranded that version forever. The release-event cursor | |
| 335 | // is already past it, so nothing else would ever mention that repo again. | |
| 336 | // | |
| 337 | // Observed on stortron: ifsrenderer-linux linux-port-14855c7 held 1 of 4 | |
| 338 | // files for ~12 hours with its channel still pinned to the previous release, | |
| 339 | // while the bytes were fetchable the whole time. `ok` stays true because the | |
| 340 | // pull itself ran — this is not a transport failure and the summary below is | |
| 341 | // still worth logging — but `partial` tells the caller to come back. | |
| 342 | return { ok: true, partial: d.errors.length > 0, plan, delta: d, stats: { ...this.stats } } | |
| 332 | 343 | } |
| 333 | 344 | } |
| 334 | 345 | |
| … | … | @@ -411,7 +422,8 @@ export function startReleasePull (opts = {}) { |
| 411 | 422 | log('[release-pull] origin ' + originServerKey.slice(0, 12) + |
| 412 | 423 | (pullOpts.repos && pullOpts.repos.length ? ' [' + pullOpts.repos.join(',') + ']' : '') + |
| 413 | 424 | ' → fetched ' + s.fetched + ', replaced ' + s.replaced + |
| 414 | ', skipped ' + s.skipped + ', errors ' + (s.errors ? s.errors.length : 0)) | |
| 425 | ', skipped ' + s.skipped + ', errors ' + (s.errors ? s.errors.length : 0) + | |
| 426 | (res.partial ? ' — PARTIAL, will retry' : '')) | |
| 415 | 427 | } else if (!res.ok) { |
| 416 | 428 | log('[release-pull] failed: ' + res.reason) |
| 417 | 429 | } |
| … | … | @@ -434,7 +446,16 @@ export function startReleasePull (opts = {}) { |
| 434 | 446 | pending.clear() |
| 435 | 447 | again = false |
| 436 | 448 | const res = await pullNow({ repos }) |
| 437 | if (!res.ok) { | |
| 449 | // A PARTIAL pull re-queues too. It is not a transport failure — the pull ran | |
| 450 | // and fetched what it could — but some file the index named was not | |
| 451 | // fetchable, most often because the origin had not finished registering it | |
| 452 | // with the overlay when we asked. Treating that as success is what stranded | |
| 453 | // versions permanently: `pending` was already cleared, `failures` reset, and | |
| 454 | // the release-event cursor had moved past. Re-queuing routes it through the | |
| 455 | // same capped backoff as any other failure, so a briefly-unservable artifact | |
| 456 | // costs a few retries and an unrecoverable one settles at one attempt per | |
| 457 | // MAX_RETRY_MS rather than hammering the origin. | |
| 458 | if (!res.ok || res.partial) { | |
| 438 | 459 | // ANY failure re-queues. This used to restore the work for exactly one |
| 439 | 460 | // reason ('a pull is already in progress') and drop it for every other — |
| 440 | 461 | // including the 15s `swarm-rpc: connection timeout` that fires when the |
| … | … | @@ -1,6 +1,6 @@ |
| 1 | 1 | { |
| 2 | 2 | "name": "pear-git-v3", |
| 3 | "version": "0.1.223", | |
| 3 | "version": "0.1.224", | |
| 4 | 4 | "description": "P2P git forge — multi-writer issue/org/release/peer-mirror daemon. Built bottom-up after the failed v2.", |
| 5 | 5 | "type": "module", |
| 6 | 6 | "main": "index.js", |
| … | … | @@ -0,0 +1,147 @@ |
| 1 | // The index must not advertise bytes this daemon cannot yet serve. | |
| 2 | // | |
| 3 | // `_writeIndex` wrote `_index.json` FIRST and registered the artifacts with the | |
| 4 | // overlay afterwards, sequentially and awaited. `_rescan` then appends the | |
| 5 | // release-event that wakes every mirror. So the publish order was: | |
| 6 | // | |
| 7 | // write _index.json → register file 1 → register file 2 → … → event | |
| 8 | // ^ ^ | |
| 9 | // the mirror is told about all of these ------------- here | |
| 10 | // | |
| 11 | // but a mirror reacts within milliseconds, and registering a large artifact | |
| 12 | // means chunking and hashing it. Any file not yet registered when the mirror | |
| 13 | // asks is answered with "no holder" — the index names bytes nobody can fetch. | |
| 14 | // | |
| 15 | // Measured on stortron 2026-08-17 for ifsrenderer-linux linux-port-14855c7, | |
| 16 | // published 21:53:32, pulled 21:54: | |
| 17 | // | |
| 18 | // fetched dist/ifsrenderer-linux-x86_64.tar.gz (49170279 B) <- first in the list | |
| 19 | // error: no holder for dist/IFSRenderer-x86_64.AppImage <- 49 MB, still chunking | |
| 20 | // error: no holder for build.log <- 8.5 KB | |
| 21 | // error: no holder for manifest.json <- 508 B | |
| 22 | // channel ifsrenderer-linux/experimental NOT pinned to linux-port-14855c7 | |
| 23 | // | |
| 24 | // The file ORDER is the tell: the artifact registered first arrived, everything | |
| 25 | // after it did not. Size was irrelevant — a 508-byte manifest failed while a | |
| 26 | // 49 MB tarball succeeded. Verified afterwards with hyper-overlay's own | |
| 27 | // hashChunk that all four hashes on the origin matched the index exactly, so the | |
| 28 | // bytes and the advertised hashes were correct all along; only the timing was | |
| 29 | // wrong. | |
| 30 | // | |
| 31 | // This is a DIFFERENT window from the one `seedOverlayServeMap` addresses. That | |
| 32 | // one is boot-time: the overlay's hash→path maps are in-memory, so a restart | |
| 33 | // loses them. This one happens at publish time on a daemon that never restarted. | |
| 34 | // | |
| 35 | // The fix is an ordering constraint, so that is what is asserted: every artifact | |
| 36 | // is registered with the overlay BEFORE the index that names it is published. | |
| 37 | // The existing "the event is appended AFTER the index is written" test in | |
| 38 | // release-event-log.test.js pins the other half, giving the full order: | |
| 39 | // | |
| 40 | // register overlay → write _index.json → append event | |
| 41 | ||
| 42 | import test from 'brittle' | |
| 43 | import { mkdtempSync, readFileSync, existsSync } from '#node:fs' | |
| 44 | import { join } from '#node:path' | |
| 45 | import { tmpdir } from '#node:os' | |
| 46 | import b4a from 'b4a' | |
| 47 | ||
| 48 | import { ReleaseManager, RELEASE_OVERLAY_PREFIX } from '../lib/release-manager.js' | |
| 49 | ||
| 50 | const INDEX_FILE = '_index.json' | |
| 51 | ||
| 52 | function tmpStorage (p = 'pgv3-rel-serve-order-') { | |
| 53 | return mkdtempSync(join(tmpdir(), p)) | |
| 54 | } | |
| 55 | ||
| 56 | /** | |
| 57 | * An overlay that records, for each registerFile call, whether the on-disk | |
| 58 | * index ALREADY advertises the path being registered. | |
| 59 | * | |
| 60 | * Reading the index from disk at that instant is what makes this a real | |
| 61 | * ordering assertion rather than a restatement of the implementation: if the | |
| 62 | * index already names the file, a mirror woken now would ask for bytes this | |
| 63 | * overlay has not been told about yet. | |
| 64 | */ | |
| 65 | function recordingOverlay (dir) { | |
| 66 | const calls = [] | |
| 67 | return { | |
| 68 | calls, | |
| 69 | async registerFile (overlayPath, absPath, opts) { | |
| 70 | let advertisedAlready = false | |
| 71 | try { | |
| 72 | const idx = JSON.parse(readFileSync(join(dir, INDEX_FILE), 'utf8')) | |
| 73 | const rel = overlayPath.slice(RELEASE_OVERLAY_PREFIX.length) | |
| 74 | advertisedAlready = (idx.files || []).some((f) => f.path === rel) | |
| 75 | } catch { | |
| 76 | advertisedAlready = false // no index on disk yet — nothing is advertised | |
| 77 | } | |
| 78 | calls.push({ overlayPath, advertisedAlready, contentHash: opts && opts.contentHash }) | |
| 79 | } | |
| 80 | } | |
| 81 | } | |
| 82 | ||
| 83 | test('artifacts are registered with the overlay BEFORE the index names them', async (t) => { | |
| 84 | const dir = tmpStorage() | |
| 85 | const overlay = recordingOverlay(dir) | |
| 86 | const rm = new ReleaseManager({ releaseDir: dir, overlay }) | |
| 87 | ||
| 88 | await rm.publish({ | |
| 89 | repo: 'demo', | |
| 90 | version: 'v1', | |
| 91 | artifacts: [ | |
| 92 | { path: 'dist/big.bin', data: b4a.alloc(4096, 1) }, | |
| 93 | { path: 'dist/small.bin', data: b4a.from('small') }, | |
| 94 | { path: 'build.log', data: b4a.from('log output') } | |
| 95 | ] | |
| 96 | }) | |
| 97 | ||
| 98 | t.ok(overlay.calls.length >= 4, | |
| 99 | 'every artifact plus manifest.json was registered (' + overlay.calls.length + ')') | |
| 100 | ||
| 101 | const premature = overlay.calls.filter((c) => c.advertisedAlready) | |
| 102 | t.is(premature.length, 0, | |
| 103 | 'no file was still being registered while the index already advertised it' + | |
| 104 | (premature.length ? ' — ' + premature.map((c) => c.overlayPath).join(', ') : '')) | |
| 105 | }) | |
| 106 | ||
| 107 | test('the index is complete once written, not filled in behind the mirror', async (t) => { | |
| 108 | const dir = tmpStorage() | |
| 109 | const registered = new Set() | |
| 110 | const overlay = { | |
| 111 | async registerFile (overlayPath) { | |
| 112 | registered.add(overlayPath.slice(RELEASE_OVERLAY_PREFIX.length)) | |
| 113 | } | |
| 114 | } | |
| 115 | const rm = new ReleaseManager({ releaseDir: dir, overlay }) | |
| 116 | ||
| 117 | await rm.publish({ | |
| 118 | repo: 'demo', | |
| 119 | version: 'v1', | |
| 120 | artifacts: [{ path: 'dist/a.bin', data: b4a.from('A') }, { path: 'dist/b.bin', data: b4a.from('B') }] | |
| 121 | }) | |
| 122 | ||
| 123 | t.ok(existsSync(join(dir, INDEX_FILE)), 'an index was written') | |
| 124 | const idx = JSON.parse(readFileSync(join(dir, INDEX_FILE), 'utf8')) | |
| 125 | const missing = (idx.files || []).filter((f) => !registered.has(f.path)).map((f) => f.path) | |
| 126 | ||
| 127 | t.alike(missing, [], | |
| 128 | 'every path the index advertises is one the overlay was already told to serve') | |
| 129 | }) | |
| 130 | ||
| 131 | test('a second publish still registers the NEW version before advertising it', async (t) => { | |
| 132 | // The first publish is the easy case: there is no index on disk at all, so any | |
| 133 | // order passes a naive check. The regression this guards is the second one, | |
| 134 | // where a stale index exists and the new entries are appended to it. | |
| 135 | const dir = tmpStorage() | |
| 136 | const overlay = recordingOverlay(dir) | |
| 137 | const rm = new ReleaseManager({ releaseDir: dir, overlay }) | |
| 138 | ||
| 139 | await rm.publish({ repo: 'demo', version: 'v1', artifacts: [{ path: 'a.bin', data: b4a.from('A') }] }) | |
| 140 | const afterFirst = overlay.calls.length | |
| 141 | await rm.publish({ repo: 'demo', version: 'v2', artifacts: [{ path: 'a.bin', data: b4a.from('B') }] }) | |
| 142 | ||
| 143 | const second = overlay.calls.slice(afterFirst) | |
| 144 | t.ok(second.length >= 2, 'the second publish registered its own files (' + second.length + ')') | |
| 145 | t.is(second.filter((c) => c.advertisedAlready).length, 0, | |
| 146 | 'and none of them were advertised before being registered') | |
| 147 | }) |
| … | … | @@ -17,6 +17,7 @@ |
| 17 | 17 | |
| 18 | 18 | import test from 'brittle' |
| 19 | 19 | import { mkdtemp, rm } from '#node:fs/promises' |
| 20 | import { writeFileSync } from '#node:fs' | |
| 20 | 21 | import { tmpdir } from '#node:os' |
| 21 | 22 | import { join } from '#node:path' |
| 22 | 23 | |
| … | … | @@ -161,6 +162,100 @@ test('per-pull deltas are reported, not lifetime totals', async (t) => { |
| 161 | 162 | t.is(res.stats.fetched, 60, 'the cumulative total is still available') |
| 162 | 163 | }) |
| 163 | 164 | |
| 165 | // ── a pull that fetched SOME of a version is not a success ────────────── | |
| 166 | // | |
| 167 | // pullFrom returned { ok: true } whenever it completed, no matter how many | |
| 168 | // individual files failed — the live log line is literally | |
| 169 | // `fetched 1, replaced 0, skipped 120, errors 3` on the SUCCESS path. reconcile() | |
| 170 | // only re-queues on `!res.ok` and resets `failures = 0` otherwise, so a version | |
| 171 | // that lost the publish/serve race was stranded permanently: nothing retried it, | |
| 172 | // and the release-event cursor had long since moved past. | |
| 173 | // | |
| 174 | // Observed on stortron: ifsrenderer-linux linux-port-14855c7 sat with one of four | |
| 175 | // files present for ~12 hours, its channel still pinned to the previous release, | |
| 176 | // until a pull was triggered by hand. The bytes were fetchable the whole time — | |
| 177 | // only the retry was missing. | |
| 178 | // | |
| 179 | // The origin-side ordering fix (release-index-serve-order.test.js) closes the | |
| 180 | // window; this closes the consequence, so a partial pull heals itself even when | |
| 181 | // the far side is briefly unable to serve. | |
| 182 | ||
| 183 | test('a pull that could not fetch every file reports itself partial', async (t) => { | |
| 184 | const dir = await mkdtemp(join(tmpdir(), 'relpull-partial-')) | |
| 185 | t.teardown(() => rm(dir, { recursive: true, force: true })) | |
| 186 | ||
| 187 | const puller = new ReleasePuller({ | |
| 188 | releaseDir: dir, | |
| 189 | overlay: { fetchFile: async () => null } // null = "no holder" | |
| 190 | }) | |
| 191 | ||
| 192 | const res = await puller.pullFrom({ | |
| 193 | call: async () => ({ | |
| 194 | files: [{ path: 'demo/versions/1/app.bin', hash: 'f'.repeat(64), size: 10, repo: 'demo', version: '1' }] | |
| 195 | }) | |
| 196 | }) | |
| 197 | ||
| 198 | t.ok(res.ok, 'the pull itself ran — this is not a transport failure') | |
| 199 | t.ok(res.partial, 'but it is flagged partial, because a file it was asked for did not land') | |
| 200 | }) | |
| 201 | ||
| 202 | test('a pull where everything landed is not partial', async (t) => { | |
| 203 | // The control. Without it, `partial: true` on every pull would also satisfy | |
| 204 | // the assertion above while making the retry loop spin forever. | |
| 205 | const dir = await mkdtemp(join(tmpdir(), 'relpull-complete-')) | |
| 206 | t.teardown(() => rm(dir, { recursive: true, force: true })) | |
| 207 | ||
| 208 | const puller = new ReleasePuller({ | |
| 209 | releaseDir: dir, | |
| 210 | overlay: { | |
| 211 | fetchFile: async (hash, opts) => { | |
| 212 | writeFileSync(opts.destPath, 'x'.repeat(10)) | |
| 213 | return { size: 10 } | |
| 214 | } | |
| 215 | } | |
| 216 | }) | |
| 217 | ||
| 218 | const res = await puller.pullFrom({ | |
| 219 | call: async () => ({ | |
| 220 | files: [{ path: 'demo/versions/1/app.bin', hash: 'f'.repeat(64), size: 10, repo: 'demo', version: '1' }] | |
| 221 | }) | |
| 222 | }) | |
| 223 | ||
| 224 | t.ok(res.ok, 'the pull ran') | |
| 225 | t.absent(res.partial, 'and is not partial, because every file it was asked for landed') | |
| 226 | }) | |
| 227 | ||
| 228 | test('a partial pull is retried rather than left stranded', async (t) => { | |
| 229 | // The behaviour that was missing. Pre-fix the first (partial) pull reset | |
| 230 | // `failures` to 0, drained `pending`, and nothing ever looked at that repo again. | |
| 231 | const attempts = [] | |
| 232 | let landEverything = false | |
| 233 | ||
| 234 | const pull = startReleasePull({ | |
| 235 | puller: fakePuller(async (client, opts) => { | |
| 236 | attempts.push([...(opts.repos || [])]) | |
| 237 | return landEverything | |
| 238 | ? { ok: true, stats: { errors: [] }, delta: { errors: [] } } | |
| 239 | : { ok: true, partial: true, stats: { errors: ['no holder for x'] }, delta: { errors: ['no holder for x'] } } | |
| 240 | }), | |
| 241 | originServerKey: 'e'.repeat(64), | |
| 242 | debounceMs: 10, | |
| 243 | connect: async () => ({ call: async () => ({ files: [] }), close: async () => {} }), | |
| 244 | log: () => {} | |
| 245 | }) | |
| 246 | t.teardown(() => pull.stop()) | |
| 247 | ||
| 248 | pull.notify({ repo: 'ifsrenderer-linux' }) | |
| 249 | await tick(300) | |
| 250 | ||
| 251 | t.ok(attempts.length >= 2, 'the partial pull was retried (' + attempts.length + ' attempts)') | |
| 252 | t.alike(attempts[1], ['ifsrenderer-linux'], 'carrying the same repo that came back incomplete') | |
| 253 | ||
| 254 | landEverything = true | |
| 255 | await tick(400) | |
| 256 | t.is(pull.pending().length, 0, 'and once everything landed the repo was drained') | |
| 257 | }) | |
| 258 | ||
| 164 | 259 | test('a new error is logged with its text, not just counted', async (t) => { |
| 165 | 260 | const lines = [] |
| 166 | 261 | // A REAL dir: with a bogus releaseDir the fetch loop dies in mkdirSync and |