| … | … | @@ -465,11 +465,32 @@ export class OverlayProtocolV2 { |
| 465 | 465 | } |
| 466 | 466 | |
| 467 | 467 | async _onContentRequest (peer, msg) { |
| 468 | // Find a local file whose content matches msg.contentHash. | |
| 469 | // Two lookup paths: | |
| 470 | // 1. Fast path: _contentHashPaths map (pre-populated by consumer) | |
| 471 | // 2. Slow path: scan fileIndex for any file:* entry with that hash | |
| 468 | // Resolve msg.contentHash → an on-disk path. Three lookup paths, in order: | |
| 469 | // 1. Fast path: _contentHashPaths map (warm — just-registered / consumer-set) | |
| 470 | // 2. Content-addressed spool (RESTART-DURABLE): <syncBaseDir>/<contentHash> | |
| 471 | // 3. Slow path: scan fileIndex for any file:* entry with that hash | |
| 472 | 472 | let diskPath = this._contentHashPaths.get(msg.contentHash) |
| 473 | if (diskPath && !fileExists(diskPath)) diskPath = null | |
| 474 | ||
| 475 | if (!diskPath) { | |
| 476 | // Content-addressed spool fallback. The canonical bytes are stored at | |
| 477 | // <syncBaseDir>/<contentHash> (pear-git-v3 spools an upload to spool/<oid> | |
| 478 | // where oid IS the content hash — router.js). The in-memory map is empty | |
| 479 | // after a daemon restart and FileIndex persists no disk path, so resolve | |
| 480 | // the hash straight from the content-addressed spool — a pure function of | |
| 481 | // the hash + dir, like the tree-store's blobs/<hash>. This is what lets a | |
| 482 | // forge keep serving its large files (hyper-tps LFS) across restarts. | |
| 483 | // (NOT gip bundles: those are stored by git oid, not by contentHash — they | |
| 484 | // need autobase-driven rehydration, tracked separately.) | |
| 485 | for (const baseDir of this._syncBaseDirs) { | |
| 486 | const candidate = path.join(baseDir, msg.contentHash) | |
| 487 | if (fileExists(candidate)) { | |
| 488 | diskPath = candidate | |
| 489 | this._contentHashPaths.set(msg.contentHash, candidate) // warm the map | |
| 490 | break | |
| 491 | } | |
| 492 | } | |
| 493 | } | |
| 473 | 494 | |
| 474 | 495 | if (!diskPath) { |
| 475 | 496 | // Scan fileIndex for a file whose contentHash matches. |
| … | … | @@ -480,7 +501,7 @@ export class OverlayProtocolV2 { |
| 480 | 501 | for (const f of files) { |
| 481 | 502 | if (f.contentHash === msg.contentHash) { |
| 482 | 503 | diskPath = this._filePaths.get(f.path) |
| 483 | if (!diskPath && f.path) diskPath = f.path | |
| 504 | if ((!diskPath || !fileExists(diskPath)) && f.path) diskPath = f.path | |
| 484 | 505 | break |
| 485 | 506 | } |
| 486 | 507 | } |
| … | … | @@ -573,3 +594,10 @@ export class OverlayProtocolV2 { |
| 573 | 594 | return null |
| 574 | 595 | } |
| 575 | 596 | } |
| 597 | ||
| 598 | // True iff `p` is an existing regular file. Used to validate a resolved disk path | |
| 599 | // before serving (a cold-restart map entry or an overlay-path masquerading as a | |
| 600 | // disk path won't exist) and to probe the content-addressed spool. | |
| 601 | function fileExists (p) { | |
| 602 | try { return fs.statSync(p).isFile() } catch { return false } | |
| 603 | } |
| … | … | @@ -1,6 +1,6 @@ |
| 1 | 1 | { |
| 2 | 2 | "name": "hyper-overlay", |
| 3 | "version": "0.2.7", | |
| 3 | "version": "0.2.8", | |
| 4 | 4 | "description": "Content-addressed union filesystem — P2P file distribution with FastCDC chunking and zero duplication", |
| 5 | 5 | "type": "module", |
| 6 | 6 | "main": "index.js", |
| … | … | @@ -0,0 +1,51 @@ |
| 1 | import test from 'brittle' | |
| 2 | import Protomux from 'protomux' | |
| 3 | import crypto from 'hypercore-crypto' | |
| 4 | import { Duplex } from 'streamx' | |
| 5 | import { tmpStore, tmpDir, fs, path } from './helpers.js' | |
| 6 | import { HyperOverlayV2 } from '../lib/overlay-v2.js' | |
| 7 | ||
| 8 | // Phase 2: a forge serves hyper-tps' large files (NOT stored in git) by content | |
| 9 | // hash over hyper-overlay — the deliberate "no hyperblobs" design. The canonical | |
| 10 | // bytes live at <spool>/<oid> (content-addressed; pear-git-v3 router.js spools an | |
| 11 | // upload to spool/<oid>, where oid IS the blake2b content hash). After a daemon | |
| 12 | // restart the in-memory _contentHashPaths map is empty and FileIndex persists no | |
| 13 | // disk path, so the publisher must resolve a content request straight from the | |
| 14 | // content-addressed spool. This reproduces a COLD publisher (bytes in the spool, | |
| 15 | // empty maps, never registerFile'd) and asserts it still serves — exactly the | |
| 16 | // post-restart condition that currently breaks cross-machine LFS serving. | |
| 17 | ||
| 18 | function makeDuplex () { | |
| 19 | let aWrite, bWrite | |
| 20 | const a = new Duplex({ write (d, cb) { bWrite(d); cb() }, read () {} }) | |
| 21 | const b = new Duplex({ write (d, cb) { aWrite(d); cb() }, read () {} }) | |
| 22 | aWrite = (d) => a.push(d) | |
| 23 | bWrite = (d) => b.push(d) | |
| 24 | return [a, b] | |
| 25 | } | |
| 26 | ||
| 27 | const settle = (ms = 400) => new Promise(r => setTimeout(r, ms)) | |
| 28 | ||
| 29 | test('RESTART DURABILITY: a cold publisher serves by content hash from the content-addressed spool', async (t) => { | |
| 30 | const spool = tmpDir('ov2-rd-spool') | |
| 31 | const content = crypto.randomBytes(300 * 1024) | |
| 32 | const oid = crypto.data(content).toString('hex') | |
| 33 | fs.writeFileSync(path.join(spool, oid), content) // canonical bytes at <spool>/<oid> | |
| 34 | ||
| 35 | // Cold publisher: fresh store, EMPTY in-memory maps, NEVER registerFile'd — | |
| 36 | // exactly the post-restart state (the spool persists on disk; the maps do not). | |
| 37 | const pub = new HyperOverlayV2(tmpStore('ov2-rd-pub'), { namespace: 'overlay-v2', syncBaseDirs: [spool], destDir: tmpDir('ov2-rd-pub-dest') }) | |
| 38 | const con = new HyperOverlayV2(tmpStore('ov2-rd-con'), { namespace: 'overlay-v2', destDir: tmpDir('ov2-rd-con-dest') }) | |
| 39 | await pub.ready(); await con.ready() | |
| 40 | t.teardown(async () => { try { await pub.close() } catch {}; try { await con.close() } catch {} }) | |
| 41 | ||
| 42 | const [sa, sb] = makeDuplex() | |
| 43 | pub.attachProtocol(Protomux.from(sa)) | |
| 44 | con.attachProtocol(Protomux.from(sb)) | |
| 45 | await settle() | |
| 46 | t.is(con.peerCount, 1, 'consumer paired with the cold publisher') | |
| 47 | ||
| 48 | const got = await con.fetchFile(oid, { timeout: 6000 }) | |
| 49 | t.ok(got, 'consumer fetched from the cold publisher (served via content-addressed spool)') | |
| 50 | if (got) t.is(Buffer.compare(got.data, content), 0, 'bytes match') | |
| 51 | }) |