| … | … | @@ -0,0 +1,124 @@ |
| 1 | /** |
| 2 | * seedFilePathsFromIndex — rebuilding the in-memory serve maps at boot. |
| 3 | * |
| 4 | * The maps (`_filePaths`, `_contentHashPaths`) are in-memory and only |
| 5 | * registerFile() fills them, so without this seed a restarted node serves only |
| 6 | * what it wrote since boot. 0.3.9 added the seed. This file covers the seed |
| 7 | * itself, which had NO tests — which is how the bug below survived three |
| 8 | * production sightings (2026-08-08 136 MiB APK, 2026-08-25 471 MB zip, |
| 9 | * 2026-08-29 168 MB APK: bytes on disk, hash advertised, peer told "no holder"). |
| 10 | * |
| 11 | * THE BUG: FileIndex.putFile persists `contentHash: meta.contentHash || null` |
| 12 | * with a `hashing` flag, so a null hash is a first-class stored state. The seed |
| 13 | * set the PATH map unconditionally but the HASH map only `if (f.contentHash)`, |
| 14 | * while counting the entry as seeded either way. Such an entry is therefore |
| 15 | * invisible to fetch-by-oid, reports as healthy, and — because consumers build |
| 16 | * their "already known" set from what the resolver saw — is skipped by the |
| 17 | * repair pass written for exactly this failure. |
| 18 | * |
| 19 | * FIX CONTRACT (deliberately non-breaking): the return stays a NUMBER, but now |
| 20 | * counts only entries actually seeded into the HASH map — i.e. genuinely |
| 21 | * servable by oid. atproto-store logs it as "servable-by-oid after restart", |
| 22 | * a claim that was false before this change and is true after it, with no edit |
| 23 | * there. The resolver additionally receives the index ENTRY, so a consumer can |
| 24 | * tell a fully-seeded entry from a half-seeded one and route the latter to its |
| 25 | * repair pass instead of skipping it. |
| 26 | */ |
| 27 | |
| 28 | import test from 'brittle' |
| 29 | import crypto from 'hypercore-crypto' |
| 30 | import { tmpStore, tmpDir, fs, path } from './helpers.js' |
| 31 | import { HyperOverlay } from '../lib/overlay.js' |
| 32 | |
| 33 | async function makeOverlay (label) { |
| 34 | const ov = new HyperOverlay(tmpStore('seed-' + label), { |
| 35 | namespace: 'overlay-v2', |
| 36 | destDir: tmpDir('seed-' + label + '-dest') |
| 37 | }) |
| 38 | await ov.ready() |
| 39 | return ov |
| 40 | } |
| 41 | |
| 42 | test('seedFilePathsFromIndex: a hashed entry seeds BOTH maps', async (t) => { |
| 43 | const ov = await makeOverlay('hashed') |
| 44 | t.teardown(async () => { try { await ov.close() } catch {} }) |
| 45 | |
| 46 | const dir = tmpDir('seed-hashed-files') |
| 47 | const body = crypto.randomBytes(4096) |
| 48 | const disk = path.join(dir, 'a.bin') |
| 49 | fs.writeFileSync(disk, body) |
| 50 | const hash = crypto.data(body).toString('hex') |
| 51 | |
| 52 | await ov._index.putFile('release:a.bin', { contentHash: hash, size: body.length, mtime: Date.now() }) |
| 53 | |
| 54 | const res = await ov.seedFilePathsFromIndex(() => disk) |
| 55 | |
| 56 | t.is(res, 1, 'one entry seeded (return stays a NUMBER — atproto-store reads it as one)') |
| 57 | t.is(ov._filePaths.get('release:a.bin'), disk, 'path map seeded') |
| 58 | t.is(ov._contentHashPaths.get(hash), disk, 'HASH map seeded — servable by oid') |
| 59 | }) |
| 60 | |
| 61 | test('seedFilePathsFromIndex: a NULL-contentHash entry is reported, not silently half-seeded', async (t) => { |
| 62 | // The exact production shape: the file is on disk and correct, but its |
| 63 | // overlay index entry was written while still `hashing`, so the hash is null. |
| 64 | const ov = await makeOverlay('nullhash') |
| 65 | t.teardown(async () => { try { await ov.close() } catch {} }) |
| 66 | |
| 67 | const dir = tmpDir('seed-nullhash-files') |
| 68 | const body = crypto.randomBytes(4096) |
| 69 | const disk = path.join(dir, 'big.apk') |
| 70 | fs.writeFileSync(disk, body) |
| 71 | |
| 72 | await ov._index.putFile('release:big.apk', { contentHash: null, size: body.length, mtime: Date.now(), hashing: true }) |
| 73 | |
| 74 | const seen = [] |
| 75 | const res = await ov.seedFilePathsFromIndex((overlayPath, entry) => { |
| 76 | // The consumer must be able to tell a fully-seeded entry from a half-seeded |
| 77 | // one — it builds its "already known, skip repair" set from this callback. |
| 78 | seen.push({ overlayPath, contentHash: entry ? entry.contentHash : undefined }) |
| 79 | return disk |
| 80 | }) |
| 81 | |
| 82 | t.is(seen.length, 1, 'resolver called once') |
| 83 | t.is(seen[0].contentHash, null, |
| 84 | 'resolver RECEIVES the entry, so a null hash is visible to the consumer') |
| 85 | |
| 86 | t.is(res, 0, 'NOT counted as seeded — the count now means "servable by oid"') |
| 87 | |
| 88 | t.is(ov._filePaths.get('release:big.apk'), disk, 'path map still seeded') |
| 89 | t.is(ov._contentHashPaths.size, 0, 'hash map correctly empty — nothing to serve by oid') |
| 90 | }) |
| 91 | |
| 92 | test('seedFilePathsFromIndex: counts split so a healthy log cannot hide a broken entry', async (t) => { |
| 93 | const ov = await makeOverlay('mixed') |
| 94 | t.teardown(async () => { try { await ov.close() } catch {} }) |
| 95 | |
| 96 | const dir = tmpDir('seed-mixed-files') |
| 97 | const good = crypto.randomBytes(1024) |
| 98 | const goodPath = path.join(dir, 'good.bin') |
| 99 | fs.writeFileSync(goodPath, good) |
| 100 | const goodHash = crypto.data(good).toString('hex') |
| 101 | const badPath = path.join(dir, 'bad.bin') |
| 102 | fs.writeFileSync(badPath, crypto.randomBytes(1024)) |
| 103 | |
| 104 | await ov._index.putFile('release:good.bin', { contentHash: goodHash, size: good.length, mtime: Date.now() }) |
| 105 | await ov._index.putFile('release:bad.bin', { contentHash: null, size: 1024, mtime: Date.now(), hashing: true }) |
| 106 | |
| 107 | const res = await ov.seedFilePathsFromIndex((p) => path.join(dir, p.slice('release:'.length))) |
| 108 | |
| 109 | t.is(res, 1, 'only the servable one counts — a healthy total cannot hide a broken entry') |
| 110 | t.is(ov._contentHashPaths.get(goodHash), goodPath, 'good file servable by oid') |
| 111 | }) |
| 112 | |
| 113 | test('seedFilePathsFromIndex: an unresolvable path is skipped entirely', async (t) => { |
| 114 | const ov = await makeOverlay('unresolved') |
| 115 | t.teardown(async () => { try { await ov.close() } catch {} }) |
| 116 | |
| 117 | await ov._index.putFile('other:x.bin', { contentHash: 'ff'.repeat(32), size: 1, mtime: Date.now() }) |
| 118 | |
| 119 | // A consumer that does not own this namespace returns null for it. |
| 120 | const res = await ov.seedFilePathsFromIndex(() => null) |
| 121 | |
| 122 | t.is(res, 0, 'nothing seeded') |
| 123 | t.is(ov._filePaths.size, 0) |
| 124 | }) |