| … | … | @@ -0,0 +1,121 @@ |
| 1 | /** |
| 2 | * Clean → smudge, cross-machine. Proves an overlay-LFS file committed on one |
| 3 | * machine is recoverable on another that never had the bytes — the property |
| 4 | * the whole overlay-LFS concept (and #66 bootstrap-via-LFS) depends on. |
| 5 | * |
| 6 | * clean (publisher): write a large file + register it with OverlayManager |
| 7 | * (chunks → overlay, returns the `oid` that the git pointer carries). |
| 8 | * smudge (consumer): given only the pointer's `oid` + a connection to the |
| 9 | * publisher, recover byte-identical content. This is exactly what |
| 10 | * `overlay-locate { oid, peer, ensure:true }` does under the smudge |
| 11 | * filter — join the peer's overlay topic, let the v2 protocol sync the |
| 12 | * file in, then resolve it locally by oid (findByOid). |
| 13 | * |
| 14 | * Same in-process Duplex pairing as overlay-e2e.test.js (no real swarm needed — |
| 15 | * identical code paths: MultiRepoOverlayProtocolV2 + SyncEngine + TransferManager). |
| 16 | * |
| 17 | * PASS = the consumer, which had NO local copy, ends up with the exact bytes, |
| 18 | * resolved from the pointer oid. DENY (the bug I feared) = it can't. |
| 19 | */ |
| 20 | |
| 21 | import test from 'brittle' |
| 22 | import Corestore from 'corestore' |
| 23 | import Protomux from 'protomux' |
| 24 | import { Duplex } from 'streamx' |
| 25 | import crypto from 'hypercore-crypto' |
| 26 | import fs from '#node:fs' |
| 27 | import path from '#node:path' |
| 28 | import os from '#node:os' |
| 29 | |
| 30 | import { OverlayManager } from '../lib/overlay-manager.js' |
| 31 | import { SyncEngine } from 'hyper-overlay/sync-engine' |
| 32 | import { TransferManager } from 'hyper-overlay/transfer' |
| 33 | import { MultiRepoOverlayProtocolV2 } from '../lib/multi-repo-overlay-protocol.js' |
| 34 | |
| 35 | function tmpDir (prefix) { return fs.mkdtempSync(path.join(os.tmpdir(), prefix + '-')) } |
| 36 | |
| 37 | function makeDuplex () { |
| 38 | let aWrite, bWrite |
| 39 | const a = new Duplex({ write (d, cb) { bWrite(d); cb() }, read () {} }) |
| 40 | const b = new Duplex({ write (d, cb) { aWrite(d); cb() }, read () {} }) |
| 41 | aWrite = (d) => a.push(d) |
| 42 | bWrite = (d) => b.push(d) |
| 43 | return [a, b] |
| 44 | } |
| 45 | |
| 46 | async function makeNode (label, t) { |
| 47 | const storeRoot = tmpDir('lfs-' + label + '-store') |
| 48 | const store = new Corestore(storeRoot) |
| 49 | await store.ready() |
| 50 | const filePaths = new Map() |
| 51 | const om = new OverlayManager(store, { filePaths }) |
| 52 | const fileIndex = await om.getFileIndex() |
| 53 | const syncEngine = new SyncEngine(fileIndex, store.namespace('overlay-sync-v2')) |
| 54 | await syncEngine.ready() |
| 55 | const transferManager = new TransferManager(fileIndex) |
| 56 | const protocol = new MultiRepoOverlayProtocolV2(syncEngine, transferManager, om, { filePaths }) |
| 57 | om.attachSyncEngine(syncEngine) |
| 58 | await om.warmRepoCache() |
| 59 | await om.populateFilePaths() |
| 60 | t.teardown(async () => { |
| 61 | protocol.destroy() |
| 62 | await syncEngine.close() |
| 63 | await om.close() |
| 64 | await store.close() |
| 65 | fs.rmSync(storeRoot, { recursive: true, force: true }) |
| 66 | }) |
| 67 | return { store, om, syncEngine, protocol } |
| 68 | } |
| 69 | |
| 70 | test('overlay-LFS clean→smudge recovers a large file on a peer that never had it', async (t) => { |
| 71 | const pub = await makeNode('pub', t) |
| 72 | const cons = await makeNode('cons', t) |
| 73 | |
| 74 | const pubRepo = tmpDir('lfs-pub-repo') |
| 75 | const consRepo = tmpDir('lfs-cons-repo') |
| 76 | t.teardown(() => fs.rmSync(pubRepo, { recursive: true, force: true })) |
| 77 | t.teardown(() => fs.rmSync(consRepo, { recursive: true, force: true })) |
| 78 | |
| 79 | // Both track the same logical repo (the consumer is a mirror of it). This is |
| 80 | // what a bootstrap-bearing repo like `pgh` would be on stortron. |
| 81 | await pub.om.initRepo('lfs-demo', pubRepo) |
| 82 | await cons.om.initRepo('lfs-demo', consRepo) |
| 83 | await pub.om.warmRepoCache() |
| 84 | await cons.om.warmRepoCache() |
| 85 | |
| 86 | // ── clean (publisher) ── a multi-chunk "tarball-shaped" file → overlay. |
| 87 | const data = crypto.randomBytes(3 * 1024 * 1024) // 3 MB, exercises chunking |
| 88 | const oid = crypto.data(data).toString('hex') // == the git pointer's oid |
| 89 | fs.writeFileSync(path.join(pubRepo, 'bootstrap.tar.gz'), data) |
| 90 | await pub.om.putFile('lfs-demo', 'bootstrap.tar.gz', { oid, size: data.length, mtime: Date.now() }) |
| 91 | |
| 92 | // The consumer genuinely does NOT have it yet. |
| 93 | t.absent(fs.existsSync(path.join(consRepo, 'bootstrap.tar.gz')), 'consumer has no copy before sync') |
| 94 | t.is(await cons.om.findByOid(oid), null, 'consumer cannot resolve the oid before sync') |
| 95 | |
| 96 | // Pair (stand-in for the consumer joining the publisher's overlay topic that |
| 97 | // `overlay-locate { ensure:true }` does on the real swarm). |
| 98 | const [streamA, streamB] = makeDuplex() |
| 99 | pub.protocol.attach(Protomux.from(streamA)) |
| 100 | cons.protocol.attach(Protomux.from(streamB)) |
| 101 | |
| 102 | // ── smudge (consumer) ── resolve the pointer's oid to bytes, cross-machine. |
| 103 | let resolved = null |
| 104 | const start = Date.now() |
| 105 | while (Date.now() - start < 15000) { |
| 106 | const hit = await cons.om.findByOid(oid).catch(() => null) |
| 107 | if (hit && hit.repoRoot) { |
| 108 | const abs = path.join(hit.repoRoot, hit.path) |
| 109 | if (fs.existsSync(abs) && crypto.data(fs.readFileSync(abs)).toString('hex') === oid) { |
| 110 | resolved = abs |
| 111 | break |
| 112 | } |
| 113 | } |
| 114 | await new Promise(r => setTimeout(r, 100)) |
| 115 | } |
| 116 | |
| 117 | t.ok(resolved, 'consumer resolved the pointer oid to a local file (cross-machine smudge)') |
| 118 | const got = fs.readFileSync(resolved) |
| 119 | t.is(got.length, data.length, 'recovered size matches') |
| 120 | t.is(crypto.data(got).toString('hex'), oid, 'recovered bytes are byte-identical (oid matches)') |
| 121 | }) |