| … | … | @@ -6,40 +6,153 @@ |
| 6 | 6 | * |
| 7 | 7 | * Hyperbee schema: |
| 8 | 8 | * file:<path> → { contentHash, size, mtime, hashing, executable, linkname } |
| 9 | | * chunkmap:<path> → [{ hash, offset, length }] |
| 9 | * chunkmap:<path> → [{ hash, offset, length }] (small) | { __paged: N } (large; see below) |
| 10 | * chunkmap-oid:<contentHash> → [{ hash, offset, length }] (content-addressed; reused across overlay keys + restarts) |
| 10 | 11 | * sync:<peerKey>:<path> → { lastSeq, lastHash } |
| 11 | 12 | * config:sync → { folders: [...] } |
| 12 | 13 | * |
| 13 | 14 | * tree:<hex-hash> → { entries: [{kind, exec, name, childHash, size}], size } |
| 14 | 15 | * treepath:<path> → <hex-hash> (secondary: path → current tree hash) |
| 16 | * |
| 17 | * Chunk-map paging. A chunk map for a very large file (a 1.1 TB |
| 18 | * file at tier 3 ≈ 1M entries ≈ ~120 MB of JSON) does not fit in one Hyperbee |
| 19 | * value: Hypercore caps a block at 15 MiB (MAX_SUGGESTED_BLOCK_SIZE) and throws |
| 20 | * BAD_ARGUMENT above it. Maps with > CHUNKS_PER_PAGE entries are stored as a |
| 21 | * header value { __paged: N } at the base key plus N page values keyed |
| 22 | * `<base>\x00<i>` (the NUL byte never occurs in a path or hex hash, so a page key |
| 23 | * can't collide with a real one). Small maps stay inline as a plain array — |
| 24 | * byte-identical to before, and back-compatible with already-stored maps. The |
| 25 | * public API (put/get/del) is unchanged; paging is transparent to callers. |
| 15 | 26 | */ |
| 16 | 27 | |
| 17 | 28 | import Hyperbee from 'hyperbee' |
| 18 | 29 | import ReadyResource from 'ready-resource' |
| 19 | 30 | |
| 31 | // Entries per chunk-map page. ~116 B/entry worst-case JSON ⇒ |
| 32 | // ≤ ~3.8 MB/page, well under Hypercore's 15 MiB block limit. |
| 33 | const CHUNKS_PER_PAGE = 32768 |
| 34 | |
| 35 | // Overlay-path prefix for host-registered, content-addressed serve entries. |
| 36 | // Hosts that make a file servable purely by content hash should register it at |
| 37 | // `${CONTENT_REGISTER_PREFIX}${contentHash}` so compact()/evictContent() can |
| 38 | // recognize (and reclaim) the entry once the hash is no longer served. |
| 39 | export const CONTENT_REGISTER_PREFIX = '/mir/' |
| 40 | |
| 41 | // The content hash an owner-side entry is addressed by (page suffix folded in), or |
| 42 | // null for entries not keyed by a single content hash (real-path file:/chunkmap:, |
| 43 | // tree:, sync:, config:). compact() drops the content-addressed entries whose hash is |
| 44 | // no longer served, and keeps everything else. |
| 45 | function contentHashOfKey (key) { |
| 46 | if (key.startsWith('chunkmap-oid:')) return key.slice('chunkmap-oid:'.length).split('\x00')[0] |
| 47 | if (key.startsWith('chunkmap:content:')) return key.slice('chunkmap:content:'.length).split('\x00')[0] |
| 48 | if (key.startsWith(`chunkmap:${CONTENT_REGISTER_PREFIX}`)) return key.slice(`chunkmap:${CONTENT_REGISTER_PREFIX}`.length).split('\x00')[0] |
| 49 | if (key.startsWith(`file:${CONTENT_REGISTER_PREFIX}`)) return key.slice(`file:${CONTENT_REGISTER_PREFIX}`.length) |
| 50 | return null |
| 51 | } |
| 52 | |
| 53 | // v1 keeps the original 'file-index' name so existing stores aren't orphaned on |
| 54 | // upgrade; compaction moves to file-index-v2, v3, … and purges the predecessor. |
| 55 | export function indexCoreName (version) { |
| 56 | return version === 1 ? 'file-index' : `file-index-v${version}` |
| 57 | } |
| 58 | |
| 20 | 59 | export class FileIndex extends ReadyResource { |
| 21 | 60 | constructor (store, opts = {}) { |
| 22 | 61 | super() |
| 23 | 62 | this._store = store |
| 24 | 63 | this._bee = null |
| 64 | this._meta = null |
| 65 | this._version = 1 |
| 25 | 66 | this._opts = opts |
| 26 | 67 | } |
| 27 | 68 | |
| 69 | // Local index cores are encrypted at rest when the caller passes an encryptionKey |
| 70 | // (opts.encryptionKey); absent, they stay plaintext. |
| 71 | _coreOpts (name) { |
| 72 | const o = { name, valueEncoding: 'binary' } |
| 73 | if (this._opts.encryptionKey) o.encryptionKey = this._opts.encryptionKey |
| 74 | return o |
| 75 | } |
| 76 | |
| 28 | 77 | async _open () { |
| 29 | | const core = this._store.get({ name: 'file-index', valueEncoding: 'binary' }) |
| 30 | | this._bee = new Hyperbee(core, { |
| 78 | this._meta = new Hyperbee(this._store.get(this._coreOpts('index-meta')), { |
| 31 | 79 | keyEncoding: 'utf-8', |
| 32 | 80 | valueEncoding: 'json' |
| 33 | 81 | }) |
| 34 | | await this._bee.ready() |
| 82 | await this._meta.ready() |
| 83 | this._version = (await this._meta.get('version'))?.value ?? 1 |
| 84 | // A prior purge can leave a version's core with a dangling name alias — its data and |
| 85 | // by-discovery-key alias were deleted but the by-name alias was not — which throws |
| 86 | // STORAGE_EMPTY on open. The index is rebuildable cache, so advance to a fresh core |
| 87 | // name and record it rather than failing open. |
| 88 | for (let attempts = 0; ; attempts++) { |
| 89 | const bee = new Hyperbee(this._store.get(this._coreOpts(indexCoreName(this._version))), { |
| 90 | keyEncoding: 'utf-8', |
| 91 | valueEncoding: 'json' |
| 92 | }) |
| 93 | try { |
| 94 | await bee.ready() |
| 95 | this._bee = bee |
| 96 | return |
| 97 | } catch (err) { |
| 98 | if (err.code !== 'STORAGE_EMPTY' || attempts >= 64) throw err |
| 99 | await bee.close().catch(() => {}) |
| 100 | this._version += 1 |
| 101 | await this._meta.put('version', this._version) |
| 102 | } |
| 103 | } |
| 35 | 104 | } |
| 36 | 105 | |
| 37 | 106 | async _close () { |
| 38 | 107 | if (this._bee) await this._bee.close() |
| 108 | if (this._meta) await this._meta.close() |
| 39 | 109 | } |
| 40 | 110 | |
| 41 | 111 | get bee () { return this._bee } |
| 42 | 112 | |
| 113 | get version () { return this._version } |
| 114 | |
| 115 | get cores () { return [this._bee?.core, this._meta?.core].filter(Boolean) } |
| 116 | |
| 117 | // Reclaim the append-only index: stream the still-served entries into a fresh |
| 118 | // versioned core, flip the version pointer, and return the old core so the caller |
| 119 | // can clear+purge it (the only way to return an append-only bee's disk to the OS). |
| 120 | // Content-addressed entries (maps + content-register entries) whose hash isServed() is false |
| 121 | // are dropped; real-path, tree, sync and config entries are always kept. |
| 122 | async compact ({ isServed }) { |
| 123 | // Skip the rewrite entirely when nothing is droppable — otherwise a compaction of |
| 124 | // an already-clean index just churns (a fresh version core + a version-marker |
| 125 | // append), which grows the index without reclaiming anything. |
| 126 | let droppable = false |
| 127 | for await (const { key } of this._bee.createReadStream()) { |
| 128 | const hash = contentHashOfKey(key) |
| 129 | if (hash !== null && !isServed(hash)) { droppable = true; break } |
| 130 | } |
| 131 | if (!droppable) return null |
| 132 | |
| 133 | const next = this._version + 1 |
| 134 | const dst = new Hyperbee(this._store.get(this._coreOpts(indexCoreName(next))), { |
| 135 | keyEncoding: 'utf-8', |
| 136 | valueEncoding: 'json' |
| 137 | }) |
| 138 | await dst.ready() |
| 139 | let batch = dst.batch() |
| 140 | let pending = 0 |
| 141 | for await (const { key, value } of this._bee.createReadStream()) { |
| 142 | const hash = contentHashOfKey(key) |
| 143 | if (hash !== null && !isServed(hash)) continue |
| 144 | await batch.put(key, value) |
| 145 | if (++pending >= 500) { await batch.flush(); batch = dst.batch(); pending = 0 } |
| 146 | } |
| 147 | await batch.flush() |
| 148 | |
| 149 | const oldCore = this._bee.core |
| 150 | await this._meta.put('version', next) |
| 151 | this._bee = dst |
| 152 | this._version = next |
| 153 | return oldCore // left open; caller clears + purges it |
| 154 | } |
| 155 | |
| 43 | 156 | // ── File entries ──────────────────────────────────────────── |
| 44 | 157 | |
| 45 | 158 | /** |
| … | … | @@ -84,7 +197,7 @@ export class FileIndex extends ReadyResource { |
| 84 | 197 | */ |
| 85 | 198 | async delFile (path) { |
| 86 | 199 | await this._bee.del(`file:${path}`) |
| 87 | | await this._bee.del(`chunkmap:${path}`) |
| 200 | await this._delPagedValue(`chunkmap:${path}`) |
| 88 | 201 | } |
| 89 | 202 | |
| 90 | 203 | /** |
| … | … | @@ -106,22 +219,15 @@ export class FileIndex extends ReadyResource { |
| 106 | 219 | return files |
| 107 | 220 | } |
| 108 | 221 | |
| 109 | | // ── Server index metadata ─────────────────────────────────── |
| 222 | // ── Index meta (0.2.14; preserved on port) ────────────────── |
| 223 | // Small key/value bag on the main bee under `meta:` — used by the |
| 224 | // fast manifest-diff boot path (e.g. hyper-deploy). compact() keeps these |
| 225 | // (contentHashOfKey returns null for `meta:` → not droppable). |
| 110 | 226 | |
| 111 | | /** |
| 112 | | * Store a small index-metadata value (e.g. the scan-schema version that |
| 113 | | * gates the fast manifest-diff boot path). Keyed under `meta:`. |
| 114 | | * @param {string} key |
| 115 | | * @param {any} value - JSON-encodable |
| 116 | | */ |
| 117 | 227 | async putIndexMeta (key, value) { |
| 118 | 228 | await this._bee.put(`meta:${key}`, value) |
| 119 | 229 | } |
| 120 | 230 | |
| 121 | | /** |
| 122 | | * @param {string} key |
| 123 | | * @returns {any|null} |
| 124 | | */ |
| 125 | 231 | async getIndexMeta (key) { |
| 126 | 232 | const entry = await this._bee.get(`meta:${key}`) |
| 127 | 233 | return entry ? entry.value : null |
| … | … | @@ -130,12 +236,14 @@ export class FileIndex extends ReadyResource { |
| 130 | 236 | // ── Chunk maps ────────────────────────────────────────────── |
| 131 | 237 | |
| 132 | 238 | /** |
| 133 | | * Store a chunk map for a file (hash + offset + length per chunk, no data) |
| 239 | * Store a chunk map for a file (hash + offset + length per chunk, no data). |
| 240 | * Large maps are paged transparently (see the header note) so they never exceed the |
| 241 | * Hypercore block limit. |
| 134 | 242 | * @param {string} path - file path |
| 135 | 243 | * @param {Array<{ hash: string, offset: number, length: number }>} chunks |
| 136 | 244 | */ |
| 137 | 245 | async putChunkMap (path, chunks) { |
| 138 | | await this._bee.put(`chunkmap:${path}`, chunks) |
| 246 | await this._putPagedValue(`chunkmap:${path}`, chunks) |
| 139 | 247 | } |
| 140 | 248 | |
| 141 | 249 | /** |
| … | … | @@ -144,8 +252,7 @@ export class FileIndex extends ReadyResource { |
| 144 | 252 | * @returns {Array<{ hash: string, offset: number, length: number }> | null} |
| 145 | 253 | */ |
| 146 | 254 | async getChunkMap (path) { |
| 147 | | const entry = await this._bee.get(`chunkmap:${path}`) |
| 148 | | return entry ? entry.value : null |
| 255 | return this._getPagedValue(`chunkmap:${path}`) |
| 149 | 256 | } |
| 150 | 257 | |
| 151 | 258 | /** |
| … | … | @@ -163,7 +270,119 @@ export class FileIndex extends ReadyResource { |
| 163 | 270 | * @param {string} path |
| 164 | 271 | */ |
| 165 | 272 | async delChunkMap (path) { |
| 166 | | await this._bee.del(`chunkmap:${path}`) |
| 273 | await this._delPagedValue(`chunkmap:${path}`) |
| 274 | } |
| 275 | |
| 276 | // Paged-value storage. Small maps stay inline as a plain array |
| 277 | // (unchanged on disk, back-compatible); maps over CHUNKS_PER_PAGE entries are |
| 278 | // split into a { __paged: N } header + N `<base>\x00<i>` page values, written |
| 279 | // in one Hyperbee batch so the whole map commits atomically. Stale pages from |
| 280 | // a previously-larger map at the same key are deleted in the same batch. |
| 281 | async _putPagedValue (baseKey, chunks) { |
| 282 | const prevPages = await this._pagedCount(baseKey) |
| 283 | |
| 284 | if (chunks.length <= CHUNKS_PER_PAGE) { |
| 285 | if (prevPages === 0) { |
| 286 | await this._bee.put(baseKey, chunks) |
| 287 | return |
| 288 | } |
| 289 | await this._batched(async (b) => { |
| 290 | await b.put(baseKey, chunks) |
| 291 | for (let i = 0; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`) |
| 292 | }) |
| 293 | return |
| 294 | } |
| 295 | |
| 296 | const pageCount = Math.ceil(chunks.length / CHUNKS_PER_PAGE) |
| 297 | await this._batched(async (b) => { |
| 298 | await b.put(baseKey, { __paged: pageCount }) |
| 299 | for (let i = 0; i < pageCount; i++) { |
| 300 | await b.put(`${baseKey}\x00${i}`, chunks.slice(i * CHUNKS_PER_PAGE, (i + 1) * CHUNKS_PER_PAGE)) |
| 301 | } |
| 302 | for (let i = pageCount; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`) |
| 303 | }) |
| 304 | } |
| 305 | |
| 306 | async _getPagedValue (baseKey) { |
| 307 | const entry = await this._bee.get(baseKey) |
| 308 | if (!entry) return null |
| 309 | if (Array.isArray(entry.value)) return entry.value // inline / legacy |
| 310 | const pageCount = entry.value.__paged |
| 311 | if (!pageCount) return null |
| 312 | const chunks = [] |
| 313 | for (let i = 0; i < pageCount; i++) { |
| 314 | const page = await this._bee.get(`${baseKey}\x00${i}`) |
| 315 | // The header + all pages commit in one atomic batch, so a missing or non-array |
| 316 | // page means the value is corrupt/incomplete. Return null — a clean "miss" so the |
| 317 | // caller re-chunks from the source file — never a silently truncated chunk map. |
| 318 | if (!page || !Array.isArray(page.value)) return null |
| 319 | for (const c of page.value) chunks.push(c) |
| 320 | } |
| 321 | return chunks |
| 322 | } |
| 323 | |
| 324 | async _delPagedValue (baseKey) { |
| 325 | const prevPages = await this._pagedCount(baseKey) |
| 326 | if (prevPages === 0) { |
| 327 | await this._bee.del(baseKey) |
| 328 | return |
| 329 | } |
| 330 | await this._batched(async (b) => { |
| 331 | await b.del(baseKey) |
| 332 | for (let i = 0; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`) |
| 333 | }) |
| 334 | } |
| 335 | |
| 336 | // Page count of an existing paged value, or 0 if the key is absent or inline. |
| 337 | async _pagedCount (baseKey) { |
| 338 | const entry = await this._bee.get(baseKey) |
| 339 | if (!entry || Array.isArray(entry.value)) return 0 |
| 340 | return entry.value.__paged || 0 |
| 341 | } |
| 342 | |
| 343 | // Run ops in a Hyperbee batch, closing it on error (flush releases the lock |
| 344 | // itself, even on append failure, via _appendBatch's finally). |
| 345 | async _batched (fn) { |
| 346 | const b = this._bee.batch() |
| 347 | try { |
| 348 | await fn(b) |
| 349 | await b.flush() |
| 350 | } catch (err) { |
| 351 | try { await b.close() } catch {} |
| 352 | throw err |
| 353 | } |
| 354 | } |
| 355 | |
| 356 | // Content-addressed chunk map. FastCDC is deterministic on bytes, so a |
| 357 | // given content hash always chunks identically regardless of which overlay path |
| 358 | // it was registered under. Keying by content hash lets the publish-time chunking |
| 359 | // be reused at serve time (a synthetic 'content:<hash>' key) and across restarts, |
| 360 | // instead of re-reading the whole file to re-chunk it before the first byte ships. |
| 361 | async putChunkMapByHash (contentHash, chunks) { |
| 362 | await this._putPagedValue(`chunkmap-oid:${contentHash}`, chunks) |
| 363 | } |
| 364 | |
| 365 | async getChunkMapByHash (contentHash) { |
| 366 | return this._getPagedValue(`chunkmap-oid:${contentHash}`) |
| 367 | } |
| 368 | |
| 369 | async hasChunkMapByHash (contentHash) { |
| 370 | return (await this._bee.get(`chunkmap-oid:${contentHash}`)) !== null |
| 371 | } |
| 372 | |
| 373 | async delChunkMapByHash (contentHash) { |
| 374 | await this._delPagedValue(`chunkmap-oid:${contentHash}`) |
| 375 | } |
| 376 | |
| 377 | // Evict every durable entry a served content hash leaves behind: the content- |
| 378 | // addressed map plus the deterministic owner-side synthetic paths (the serve |
| 379 | // copy at content:<hash> and the register entry under CONTENT_REGISTER_PREFIX). |
| 380 | // Peer-path-keyed receiver entries are unknowable from the hash — the host's |
| 381 | // own index sweep handles those. |
| 382 | async evictContent (contentHash) { |
| 383 | await this.delChunkMapByHash(contentHash) |
| 384 | await this.delChunkMap(`content:${contentHash}`) |
| 385 | await this.delFile(`${CONTENT_REGISTER_PREFIX}${contentHash}`) |
| 167 | 386 | } |
| 168 | 387 | |
| 169 | 388 | // ── Sync state ────────────────────────────────────────────── |
| … | … | @@ -345,7 +564,8 @@ export class FileIndex extends ReadyResource { |
| 345 | 564 | |
| 346 | 565 | for await (const entry of this._bee.createReadStream()) { |
| 347 | 566 | if (entry.key.startsWith('file:')) fileCount++ |
| 348 | | else if (entry.key.startsWith('chunkmap:')) chunkMapCount++ |
| 567 | // Count one per file — skip the `\x00`-suffixed page values. |
| 568 | else if (entry.key.startsWith('chunkmap:') && !entry.key.includes('\x00')) chunkMapCount++ |
| 349 | 569 | else if (entry.key.startsWith('sync:')) syncStateCount++ |
| 350 | 570 | else if (entry.key.startsWith('tree:')) treeCount++ |
| 351 | 571 | else if (entry.key.startsWith('treepath:')) treePathCount++ |