| … | … | @@ -421,19 +421,48 @@ export class ReleaseManager { |
| 421 | 421 | // an unchanged hash means the overlay already knows about it. |
| 422 | 422 | if (this._overlay && typeof this._overlay.registerFile === 'function') { |
| 423 | 423 | let registered = 0 |
| 424 | const failed = [] | |
| 424 | 425 | for (const f of files) { |
| 425 | 426 | const before = prev.get(f.path) |
| 426 | 427 | if (before && before.hash === f.hash) continue |
| 428 | const overlayPath = RELEASE_OVERLAY_PREFIX + f.path | |
| 429 | const diskPath = join(this._releaseDir, f.path) | |
| 427 | 430 | try { |
| 428 | await this._overlay.registerFile( | |
| 429 | RELEASE_OVERLAY_PREFIX + f.path, | |
| 430 | join(this._releaseDir, f.path), | |
| 431 | { contentHash: f.hash, size: f.size } | |
| 432 | ) | |
| 431 | await this._overlay.registerFile(overlayPath, diskPath, | |
| 432 | { contentHash: f.hash, size: f.size }) | |
| 433 | 433 | registered++ |
| 434 | } catch {} | |
| 434 | } catch (err) { | |
| 435 | // FALL BACK RATHER THAN LOSE THE REGISTRATION. The expensive half of | |
| 436 | // registerFile is the eager chunk-map build, which reads the whole | |
| 437 | // file — 471 MB for a Windows game bundle — and is explicitly | |
| 438 | // optional: without it the file is still servable from the path maps | |
| 439 | // and the serve path chunks lazily on first request. Losing the whole | |
| 440 | // registration because the optional half failed is what leaves the | |
| 441 | // index advertising a file the overlay can never serve. | |
| 442 | try { | |
| 443 | await this._overlay.registerFile(overlayPath, diskPath, | |
| 444 | { contentHash: f.hash, size: f.size, prepare: false }) | |
| 445 | registered++ | |
| 446 | console.warn('[releases] registered ' + f.path + ' WITHOUT a chunk map (' + | |
| 447 | (err && err.message ? err.message : err) + ') — first fetch will chunk lazily') | |
| 448 | } catch (err2) { | |
| 449 | failed.push({ path: f.path, error: err2 && err2.message ? err2.message : String(err2) }) | |
| 450 | } | |
| 451 | } | |
| 435 | 452 | } |
| 436 | 453 | this.lastRegistered = registered |
| 454 | this.lastRegisterFailures = failed | |
| 455 | // NOT SWALLOWED. This used to be a bare `catch {}`, and registerFile is the | |
| 456 | // ONLY thing that writes the overlay's index — the one that decides what | |
| 457 | // this daemon can actually serve. A failure here leaves _index.json (which | |
| 458 | // is written unconditionally, below) advertising a file the overlay does | |
| 459 | // not know about, so every mirror asks for it forever and is told "no | |
| 460 | // holder", with nothing on the publishing side to point at. | |
| 461 | if (failed.length) { | |
| 462 | console.error('[releases] ' + failed.length + ' artifact(s) FAILED to register with the ' + | |
| 463 | 'overlay — the index will advertise bytes this daemon cannot serve:') | |
| 464 | for (const f of failed) console.error(' ' + f.path + ': ' + f.error) | |
| 465 | } | |
| 437 | 466 | } |
| 438 | 467 | |
| 439 | 468 | const index = { schema: 1, generatedAt: Date.now(), files, channels } |
| … | … | @@ -469,10 +498,78 @@ export class ReleaseManager { |
| 469 | 498 | */ |
| 470 | 499 | async seedOverlayServeMap () { |
| 471 | 500 | if (!this._overlay || typeof this._overlay.seedFilePathsFromIndex !== 'function') return 0 |
| 472 | return this._overlay.seedFilePathsFromIndex((overlayPath) => { | |
| 501 | // Collect what the OVERLAY knows while seeding, so the repair below can diff | |
| 502 | // it against what the RELEASE INDEX advertises. Two different indexes: | |
| 503 | // _index.json says what exists, the overlay's FileIndex says what can be | |
| 504 | // served, and only registerFile writes the second one. | |
| 505 | const known = new Set() | |
| 506 | const seeded = await this._overlay.seedFilePathsFromIndex((overlayPath) => { | |
| 473 | 507 | if (!overlayPath || !overlayPath.startsWith(RELEASE_OVERLAY_PREFIX)) return null |
| 508 | known.add(overlayPath) | |
| 474 | 509 | return join(this._releaseDir, overlayPath.slice(RELEASE_OVERLAY_PREFIX.length)) |
| 475 | 510 | }) |
| 511 | await this._repairOverlayRegistrations(known) | |
| 512 | return seeded | |
| 513 | } | |
| 514 | ||
| 515 | /** | |
| 516 | * Register anything the release index advertises that the overlay does not | |
| 517 | * know how to serve. | |
| 518 | * | |
| 519 | * WHY THIS IS NEEDED AT ALL. _writeIndex registers each artifact and THEN | |
| 520 | * writes _index.json — but it wrote the index whether or not the registrations | |
| 521 | * succeeded, and the failure was swallowed. So a single failed registerFile | |
| 522 | * left the two indexes permanently disagreeing: mirrors read _index.json, ask | |
| 523 | * for a hash by content, and are told "no holder" forever. Nothing re-tried, | |
| 524 | * because _writeIndex only registers NEW-or-CHANGED entries and the hash never | |
| 525 | * changes. | |
| 526 | * | |
| 527 | * Measured 2026-08-25 (hyper-tps v0.919.0): tabertron held a 471 MB | |
| 528 | * hypertps-win64.zip on disk, advertised its exact hash in _index.json, and | |
| 529 | * stortron retried "no holder for …dist/hypertps-win64.zip (07af6b4a31a0)" | |
| 530 | * indefinitely — while build.log and manifest.json from the SAME publish | |
| 531 | * replicated fine. Same peer, same connection, same pass: per-content, not | |
| 532 | * per-peer. | |
| 533 | * | |
| 534 | * CHEAP BY CONSTRUCTION: prepare:false, so no file is read and nothing is | |
| 535 | * re-hashed — the content hash is already in the index. It restores the | |
| 536 | * ability to serve; the chunk map is rebuilt lazily on first request. | |
| 537 | */ | |
| 538 | async _repairOverlayRegistrations (known) { | |
| 539 | if (!this._overlay || typeof this._overlay.registerFile !== 'function') return 0 | |
| 540 | let index | |
| 541 | try { | |
| 542 | index = JSON.parse(await readFile(join(this._releaseDir, INDEX_FILE), 'utf8')) | |
| 543 | } catch { | |
| 544 | return 0 // no index yet — nothing published on this daemon | |
| 545 | } | |
| 546 | if (!index || !Array.isArray(index.files)) return 0 | |
| 547 | ||
| 548 | let repaired = 0 | |
| 549 | const failed = [] | |
| 550 | for (const f of index.files) { | |
| 551 | if (!f || !f.path || !f.hash) continue | |
| 552 | const overlayPath = RELEASE_OVERLAY_PREFIX + f.path | |
| 553 | if (known.has(overlayPath)) continue | |
| 554 | try { | |
| 555 | // Returns null (not a throw) when the file is gone — a version collected | |
| 556 | // by gc is expected here, not an error. | |
| 557 | const r = await this._overlay.registerFile(overlayPath, join(this._releaseDir, f.path), | |
| 558 | { contentHash: f.hash, size: f.size, prepare: false }) | |
| 559 | if (r) repaired++ | |
| 560 | } catch (err) { | |
| 561 | failed.push(f.path + ': ' + (err && err.message ? err.message : err)) | |
| 562 | } | |
| 563 | } | |
| 564 | if (repaired) { | |
| 565 | console.log('[releases] overlay repair: registered ' + repaired + | |
| 566 | ' artifact(s) the index advertised but the overlay could not serve') | |
| 567 | } | |
| 568 | if (failed.length) { | |
| 569 | console.error('[releases] overlay repair FAILED for ' + failed.length + ' artifact(s):') | |
| 570 | for (const f of failed) console.error(' ' + f) | |
| 571 | } | |
| 572 | return repaired | |
| 476 | 573 | } |
| 477 | 574 | |
| 478 | 575 | async _rescan (event = null) { |
| … | … | @@ -0,0 +1,133 @@ |
| 1 | // The index and the overlay can disagree, and nothing used to reconcile them. | |
| 2 | // | |
| 3 | // There are THREE ways an artifact ends up advertised-but-unfetchable, and they | |
| 4 | // are distinct. The first two already have fixes: | |
| 5 | // | |
| 6 | // 1. ORDER — _index.json written before registration, so a mirror woken by | |
| 7 | // the release-event asks before the bytes are registered. | |
| 8 | // Fixed by registering first. See release-index-serve-order. | |
| 9 | // 2. RESTART — the overlay's hash→path maps are in-memory, so a reboot forgets | |
| 10 | // how to serve files it still holds. Fixed by seedOverlayServeMap. | |
| 11 | // 3. FAILURE — registerFile THREW, and the call sat in a bare `catch {}` while | |
| 12 | // _index.json was written unconditionally afterwards. This one. | |
| 13 | // | |
| 14 | // (3) is permanent in a way the others are not. _writeIndex only registers | |
| 15 | // NEW-or-CHANGED entries, and a published artifact's hash never changes, so the | |
| 16 | // failed registration is never retried. seedOverlayServeMap cannot help either: | |
| 17 | // it seeds from the OVERLAY's index, which is precisely where the entry is | |
| 18 | // missing. The mirror asks forever and is told "no holder"; the origin logs | |
| 19 | // nothing, because the failure was swallowed. | |
| 20 | // | |
| 21 | // Measured 2026-08-25, hyper-tps v0.919.0: tabertron held a 471 MB | |
| 22 | // hypertps-win64.zip on disk and advertised its exact hash in _index.json, while | |
| 23 | // stortron retried | |
| 24 | // no holder for hyper-tps/versions/v0.919.0/dist/hypertps-win64.zip (07af6b4a31a0) | |
| 25 | // indefinitely — and build.log and manifest.json from the SAME publish | |
| 26 | // replicated fine, same peer, same pass. Per-content, not per-peer. | |
| 27 | // | |
| 28 | // Run: bare test/release-overlay-repair.test.js | |
| 29 | ||
| 30 | import test from 'brittle' | |
| 31 | import { mkdtemp, mkdir, writeFile } from '#node:fs/promises' | |
| 32 | import { tmpdir } from '#node:os' | |
| 33 | import { join } from '#node:path' | |
| 34 | ||
| 35 | import { ReleaseManager, RELEASE_OVERLAY_PREFIX as PREFIX } from '../lib/release-manager.js' | |
| 36 | ||
| 37 | ||
| 38 | // An overlay that knows about SOME paths and records what it is asked to add. | |
| 39 | function stubOverlay (knownPaths = []) { | |
| 40 | const known = new Set(knownPaths) | |
| 41 | const registered = [] | |
| 42 | return { | |
| 43 | known, | |
| 44 | registered, | |
| 45 | async seedFilePathsFromIndex (resolve) { | |
| 46 | let n = 0 | |
| 47 | for (const p of known) { if (resolve(p)) n++ } | |
| 48 | return n | |
| 49 | }, | |
| 50 | async registerFile (overlayPath, diskPath, meta = {}) { | |
| 51 | registered.push({ overlayPath, diskPath, prepare: meta.prepare }) | |
| 52 | known.add(overlayPath) | |
| 53 | return { contentHash: meta.contentHash, size: meta.size } | |
| 54 | } | |
| 55 | } | |
| 56 | } | |
| 57 | ||
| 58 | async function fixture (files) { | |
| 59 | const dir = await mkdtemp(join(tmpdir(), 'rel-repair-')) | |
| 60 | for (const f of files) { | |
| 61 | await mkdir(join(dir, f.path, '..'), { recursive: true }).catch(() => {}) | |
| 62 | await writeFile(join(dir, f.path), f.body || 'x') | |
| 63 | } | |
| 64 | await writeFile(join(dir, '_index.json'), | |
| 65 | JSON.stringify({ schema: 1, files: files.map(f => ({ path: f.path, hash: f.hash, size: 1 })), channels: {} })) | |
| 66 | return dir | |
| 67 | } | |
| 68 | ||
| 69 | test('an artifact the overlay never registered is repaired at seed time', async (t) => { | |
| 70 | const files = [ | |
| 71 | { path: 'app/versions/v1/manifest.json', hash: 'aaa1' }, | |
| 72 | { path: 'app/versions/v1/build.log', hash: 'bbb2' }, | |
| 73 | { path: 'app/versions/v1/dist/big.zip', hash: 'ccc3' } // the one that "failed" | |
| 74 | ] | |
| 75 | const dir = await fixture(files) | |
| 76 | // The overlay knows the two small files and NOT the zip — exactly the live | |
| 77 | // shape: same publish, same pass, one hash missing. | |
| 78 | const overlay = stubOverlay([PREFIX + files[0].path, PREFIX + files[1].path]) | |
| 79 | const rm = new ReleaseManager({ releaseDir: dir, overlay }) | |
| 80 | ||
| 81 | await rm.seedOverlayServeMap() | |
| 82 | ||
| 83 | const repaired = overlay.registered.map(r => r.overlayPath) | |
| 84 | t.is(repaired.length, 1, 'only the missing one is registered') | |
| 85 | t.is(repaired[0], PREFIX + 'app/versions/v1/dist/big.zip') | |
| 86 | t.is(overlay.registered[0].prepare, false, | |
| 87 | 'repaired WITHOUT an eager chunk-map build — no file read, no re-hash') | |
| 88 | }) | |
| 89 | ||
| 90 | test('a fully-registered index repairs nothing', async (t) => { | |
| 91 | const files = [ | |
| 92 | { path: 'app/versions/v1/manifest.json', hash: 'aaa1' }, | |
| 93 | { path: 'app/versions/v1/build.log', hash: 'bbb2' } | |
| 94 | ] | |
| 95 | const dir = await fixture(files) | |
| 96 | const overlay = stubOverlay(files.map(f => PREFIX + f.path)) | |
| 97 | const rm = new ReleaseManager({ releaseDir: dir, overlay }) | |
| 98 | ||
| 99 | await rm.seedOverlayServeMap() | |
| 100 | t.is(overlay.registered.length, 0, 'the common case costs nothing') | |
| 101 | }) | |
| 102 | ||
| 103 | test('an index entry whose file is gone is skipped, not an error', async (t) => { | |
| 104 | // gc collects a version; its files vanish while _index.json still names them | |
| 105 | // until the next write. registerFile returns null rather than throwing. | |
| 106 | const dir = await fixture([{ path: 'app/versions/v1/build.log', hash: 'bbb2' }]) | |
| 107 | await writeFile(join(dir, '_index.json'), JSON.stringify({ | |
| 108 | schema: 1, | |
| 109 | files: [ | |
| 110 | { path: 'app/versions/v1/build.log', hash: 'bbb2', size: 1 }, | |
| 111 | { path: 'app/versions/collected/build.log', hash: 'ddd4', size: 1 } | |
| 112 | ], | |
| 113 | channels: {} | |
| 114 | })) | |
| 115 | const overlay = { | |
| 116 | ...stubOverlay([]), | |
| 117 | async registerFile (overlayPath, diskPath, meta) { | |
| 118 | // Mirror the real behaviour: a missing source is null, not a throw. | |
| 119 | const { existsSync } = await import('#node:fs') | |
| 120 | if (!existsSync(diskPath)) return null | |
| 121 | return { contentHash: meta.contentHash } | |
| 122 | } | |
| 123 | } | |
| 124 | overlay.seedFilePathsFromIndex = async () => 0 | |
| 125 | const rm = new ReleaseManager({ releaseDir: dir, overlay }) | |
| 126 | await t.execution(rm.seedOverlayServeMap(), 'a collected version does not break the repair') | |
| 127 | }) | |
| 128 | ||
| 129 | test('no overlay at all is a no-op', async (t) => { | |
| 130 | const dir = await fixture([{ path: 'app/versions/v1/build.log', hash: 'bbb2' }]) | |
| 131 | const rm = new ReleaseManager({ releaseDir: dir }) | |
| 132 | t.is(await rm.seedOverlayServeMap(), 0) | |
| 133 | }) |