OverviewFilesCommitsIssuesDocsReleasesPeers

hyper-overlay commits / d84065b

overlay: spool peer-fetched chunks to content-addressed files (opt-in), not base64
The peer-fetch caching path (get/read/createReadStream/hydrate/getByHash) stored fetched chunks inline as base64 `cache` sources — a +33% size tax, and the deprecated-putStream bloat that putFile's `local` pointers avoid. Add a single _cacheFetchedChunk(hash, data) helper used by all five sites: - with `opts.chunkSpoolDir`: write raw bytes to <spoolDir>/<hash> (content-addressed, idempotent) + register a `local` pointer. Combined with the new ChunkRegistry cross-mode dedup, this avoids storing the same bytes twice and drops the base64 tax. - without it (default): falls back to the legacy inline base64 cache, so existing callers are unchanged. Any spool I/O error also falls back, so a fetch never fails on an unwritable spool. Authoring paths (put/putStream) are intentionally left as-is. Tests: spool writes a local pointer + reads back, idempotent repeat, default inline fallback. registry/get-by-hash/putfile-readback/adapter all still green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
d84065b9d72a49ab904e47f27353b820f0ca97fd
by plankatron <fleeky@gmail.com> · 2026-06-20T20:11:19.000Z

Changed files (3) +109 -14

index.js +45 -13
……@@ -39,6 +39,12 @@ export class HyperOverlay extends ReadyResource {
3939 this._registry = null
4040 this._protocol = null
4141 this._namespaceStore = corestore.namespace(this._namespace)
42 // Optional content-addressed spool for peer-fetched chunks. When set,
43 // fetched bytes are written here and registered as `local` pointers
44 // instead of base64-inflated inline `cache` sources. See
45 // _cacheFetchedChunk(). Null = legacy inline-cache behavior.
46 this._chunkSpoolDir = opts.chunkSpoolDir || null
47 this._spoolSeq = 0
4248 }
4349
4450 async _open () {
……@@ -329,11 +335,7 @@ export class HyperOverlay extends ReadyResource {
329335 data = await this._protocol.requestChunk(fileInfo.manifestHash, i)
330336 if (data) {
331337 // Cache the fetched chunk locally
332 await this._registry.putChunk(c.hash, {
333 size: data.length,
334 type: 'cache',
335 data: data.toString('base64')
336 })
338 await this._cacheFetchedChunk(c.hash, data)
337339 }
338340 } catch {
339341 // Peer fetch failed, continue
……@@ -483,7 +485,7 @@ export class HyperOverlay extends ReadyResource {
483485 try {
484486 data = await this._protocol.requestChunk(fileInfo.manifestHash, i)
485487 if (data) {
486 await this._registry.putChunk(c.hash, { size: data.length, type: 'cache', data: data.toString('base64') })
488 await this._cacheFetchedChunk(c.hash, data)
487489 }
488490 } catch {}
489491 }
……@@ -539,7 +541,7 @@ export class HyperOverlay extends ReadyResource {
539541 if (!data && overlay._protocol && fileInfo.manifestHash) {
540542 data = await overlay._protocol.requestChunk(fileInfo.manifestHash, idx)
541543 if (data) {
542 await overlay._registry.putChunk(c.hash, { size: data.length, type: 'cache', data: data.toString('base64') })
544 await overlay._cacheFetchedChunk(c.hash, data)
543545 }
544546 }
545547 if (!data) return cb(new Error('Chunk unavailable: ' + c.hash))
……@@ -583,7 +585,7 @@ export class HyperOverlay extends ReadyResource {
583585 const data = await this._protocol.requestChunk(fileInfo.manifestHash, i)
584586 if (!data) throw new Error('Chunk unavailable from peers: ' + c.hash)
585587
586 await this._registry.putChunk(c.hash, { size: data.length, type: 'cache', data: data.toString('base64') })
588 await this._cacheFetchedChunk(c.hash, data)
587589 fetched++
588590
589591 if (opts.onProgress) opts.onProgress({ fetched, total: chunkList.length })
……@@ -692,14 +694,44 @@ export class HyperOverlay extends ReadyResource {
692694 if (!remote) return null
693695
694696 // Cache the fetched bytes locally so subsequent calls are free
695 await this._registry.putChunk(hash, {
696 size: remote.length,
697 type: 'cache',
698 data: remote.toString('base64')
699 })
697 await this._cacheFetchedChunk(hash, remote)
700698
701699 return remote
702700 }
701
702 /**
703 * Persist a chunk just fetched from a peer.
704 *
705 * With a configured spool dir (`opts.chunkSpoolDir`) the raw bytes are
706 * written to a content-addressed file `<spoolDir>/<hash>` and registered
707 * as a `local` pointer — no base64 +33% tax, and ChunkRegistry's
708 * cross-mode dedup then prefers it over any inline copy. Without a spool
709 * dir (default) it falls back to the legacy inline base64 `cache` source,
710 * so behavior is unchanged for callers that don't opt in. Any spool I/O
711 * error also falls back to inline cache, so a fetch never fails just
712 * because the spool is unwritable.
713 *
714 * @param {string} hash - blake2b chunk hash (hex)
715 * @param {Buffer} data - the fetched chunk bytes
716 */
717 async _cacheFetchedChunk (hash, data) {
718 if (this._chunkSpoolDir) {
719 try {
720 const dest = path.join(this._chunkSpoolDir, hash)
721 if (!fs.existsSync(dest)) {
722 await fs.promises.mkdir(this._chunkSpoolDir, { recursive: true })
723 const tmp = dest + '.tmp-' + (++this._spoolSeq)
724 await fs.promises.writeFile(tmp, data)
725 await fs.promises.rename(tmp, dest)
726 }
727 await this._registry.putChunk(hash, { size: data.length, type: 'local', path: dest, offset: 0 })
728 return
729 } catch {
730 // fall through to inline cache on any spool error
731 }
732 }
733 await this._registry.putChunk(hash, { size: data.length, type: 'cache', data: data.toString('base64') })
734 }
703735}
704736
705737// Re-exports (v1 — kept for backward compatibility)
package.json +1 -1
……@@ -1,6 +1,6 @@
11{
22 "name": "hyper-overlay",
3 "version": "0.2.11",
3 "version": "0.2.12",
44 "description": "Content-addressed union filesystem — P2P file distribution with FastCDC chunking and zero duplication",
55 "type": "module",
66 "main": "index.js",
test/spool-cache.test.js +63 -0
……@@ -0,0 +1,63 @@
1import test from 'brittle'
2import { tmpStore, tmpDir, fs, path } from './helpers.js'
3import { HyperOverlay } from '../index.js'
4
5test('peer-fetch spool: stores fetched chunk as a local pointer, not base64', async (t) => {
6 const spoolDir = tmpDir('ov-spool')
7 const overlay = new HyperOverlay(tmpStore(), { chunkSpoolDir: spoolDir })
8 await overlay.ready()
9
10 const hash = 'deadbeefchunkhash'
11 const bytes = Buffer.from('peer-fetched chunk bytes')
12 await overlay._cacheFetchedChunk(hash, bytes)
13
14 // Bytes written to a content-addressed spool file.
15 const dest = path.join(spoolDir, hash)
16 t.ok(fs.existsSync(dest), 'chunk spooled to <spoolDir>/<hash>')
17 t.alike(fs.readFileSync(dest), bytes, 'spool file holds the raw bytes')
18
19 // Registered as a local pointer — no base64 inline cache.
20 const rec = await overlay.registry.getChunk(hash)
21 t.is(rec.sources.length, 1, 'single source')
22 t.is(rec.sources[0].type, 'local', 'registered as a local pointer')
23 t.is(rec.sources[0].path, dest, 'pointer targets the spool file')
24
25 // Reads back through the pointer.
26 t.alike(await overlay.registry.getChunkData(hash), bytes, 'getChunkData reads bytes from spool')
27
28 await overlay.close()
29 fs.rmSync(spoolDir, { recursive: true, force: true })
30})
31
32test('peer-fetch spool: idempotent on repeat fetch of the same chunk', async (t) => {
33 const spoolDir = tmpDir('ov-spool')
34 const overlay = new HyperOverlay(tmpStore(), { chunkSpoolDir: spoolDir })
35 await overlay.ready()
36
37 const hash = 'repeatchunkhash'
38 const bytes = Buffer.from('same bytes twice')
39 await overlay._cacheFetchedChunk(hash, bytes)
40 await overlay._cacheFetchedChunk(hash, bytes)
41
42 const rec = await overlay.registry.getChunk(hash)
43 t.is(rec.sources.length, 1, 'no duplicate source on repeat')
44 t.alike(await overlay.registry.getChunkData(hash), bytes)
45
46 await overlay.close()
47 fs.rmSync(spoolDir, { recursive: true, force: true })
48})
49
50test('no spool dir: falls back to inline base64 cache (legacy default)', async (t) => {
51 const overlay = new HyperOverlay(tmpStore())
52 await overlay.ready()
53
54 const hash = 'inlinechunkhash'
55 const bytes = Buffer.from('inline bytes')
56 await overlay._cacheFetchedChunk(hash, bytes)
57
58 const rec = await overlay.registry.getChunk(hash)
59 t.is(rec.sources[0].type, 'cache', 'default still inline-caches (unchanged behavior)')
60 t.alike(await overlay.registry.getChunkData(hash), bytes, 'inline bytes read back')
61
62 await overlay.close()
63})