| … | … | @@ -185,6 +185,44 @@ export class HyperOverlay extends ReadyResource { |
| 185 | 185 | * meta.size file size (avoids an extra stat if known) |
| 186 | 186 | * @returns {Promise<{ contentHash, size } | null>} null if the source vanished |
| 187 | 187 | */ |
| 188 | // Seed the overlayPath -> disk and contentHash -> disk maps from the PERSISTED FileIndex, using a |
| 189 | // caller-supplied layout resolver. Returns the number of entries seeded. |
| 190 | // |
| 191 | // WHY THIS EXISTS. Both maps are in-memory and populated only by registerFile, and FileIndex |
| 192 | // deliberately persists no disk path (`putFile` stores contentHash/size/mtime) — layout belongs to |
| 193 | // the consumer, which is correct: a sidecar has one working tree per repo, an atproto peer has one |
| 194 | // records dir per DID. But it means that after a RESTART a node can serve only what it has written |
| 195 | // since boot, even though every file is still on disk and its chunk map is still in the index. |
| 196 | // `_onContentRequest`'s tier-3 fallback then finds the right index entry by hash and uses the |
| 197 | // OVERLAY path as if it were a filesystem path, which never exists — so the offer is silently |
| 198 | // dropped and the requester just times out. |
| 199 | // |
| 200 | // Measured cost of that: an atproto co-owner held a record's exact bytes on disk (hash verified |
| 201 | // equal to the requester's oid) and reported "no connected peer served this oid" for 75+ hours, |
| 202 | // which blocks the peer from signing at all. pgh-sidecar hit the same wall and solved it with a |
| 203 | // bespoke pre-seed (OverlayManager.populateFilePaths) plus a protocol subclass; this makes that a |
| 204 | // first-class operation so every consumer gets it instead of re-deriving it. |
| 205 | // |
| 206 | // Cheap on purpose: one index scan, NO re-hashing and NO file reads — the contentHash is already |
| 207 | // persisted per path, and the consumer supplies path -> disk. It also pays for itself, because |
| 208 | // tier 3 otherwise re-scans the whole index on EVERY content request; seeded, tier 1 hits instead. |
| 209 | // |
| 210 | // resolve: (overlayPath) => absolute disk path | null (null/throw skips that entry) |
| 211 | async seedFilePathsFromIndex (resolve) { |
| 212 | if (typeof resolve !== 'function') throw new Error('seedFilePathsFromIndex: a resolve(overlayPath) function is required') |
| 213 | await this._ensure() |
| 214 | let seeded = 0 |
| 215 | for (const f of await this._index.listFiles()) { |
| 216 | let diskPath = null |
| 217 | try { diskPath = resolve(f.path) } catch { continue } |
| 218 | if (!diskPath) continue |
| 219 | this._filePaths.set(f.path, diskPath) |
| 220 | if (f.contentHash) this._contentHashPaths.set(f.contentHash, diskPath) |
| 221 | seeded++ |
| 222 | } |
| 223 | return seeded |
| 224 | } |
| 225 | |
| 188 | 226 | async registerFile (overlayPath, diskPath, meta = {}) { |
| 189 | 227 | await this._ensure() |
| 190 | 228 | // Source can vanish between prepare and register (a file moved out of a |