pgh-sidecar / test / overlay-lfs-cross-machine.test.js

/**
 * Clean → smudge, cross-machine. Proves an overlay-LFS file committed on one
 * machine is recoverable on another that never had the bytes — the property
 * the whole overlay-LFS concept (and #66 bootstrap-via-LFS) depends on.
 *
 *   clean  (publisher): write a large file + register it with OverlayManager
 *          (chunks → overlay, returns the `oid` that the git pointer carries).
 *   smudge (consumer):  given only the pointer's `oid` + a connection to the
 *          publisher, recover byte-identical content. This is exactly what
 *          `overlay-locate { oid, peer, ensure:true }` does under the smudge
 *          filter — join the peer's overlay topic, let the v2 protocol sync the
 *          file in, then resolve it locally by oid (findByOid).
 *
 * Same in-process Duplex pairing as overlay-e2e.test.js (no real swarm needed —
 * identical code paths: MultiRepoOverlayProtocolV2 + SyncEngine + TransferManager).
 *
 * PASS = the consumer, which had NO local copy, ends up with the exact bytes,
 * resolved from the pointer oid. DENY (the bug I feared) = it can't.
 */

import test from 'brittle'
import Corestore from 'corestore'
import Protomux from 'protomux'
import { Duplex } from 'streamx'
import crypto from 'hypercore-crypto'
import fs from '#node:fs'
import path from '#node:path'
import os from '#node:os'

import { OverlayManager } from '../lib/overlay-manager.js'
import { SyncEngine } from 'hyper-overlay/sync-engine'
import { TransferManager } from 'hyper-overlay/transfer'
import { MultiRepoOverlayProtocolV2 } from '../lib/multi-repo-overlay-protocol.js'

function tmpDir (prefix) { return fs.mkdtempSync(path.join(os.tmpdir(), prefix + '-')) }

function makeDuplex () {
  let aWrite, bWrite
  const a = new Duplex({ write (d, cb) { bWrite(d); cb() }, read () {} })
  const b = new Duplex({ write (d, cb) { aWrite(d); cb() }, read () {} })
  aWrite = (d) => a.push(d)
  bWrite = (d) => b.push(d)
  return [a, b]
}

async function makeNode (label, t) {
  const storeRoot = tmpDir('lfs-' + label + '-store')
  const store = new Corestore(storeRoot)
  await store.ready()
  const filePaths = new Map()
  const om = new OverlayManager(store, { filePaths })
  const fileIndex = await om.getFileIndex()
  const syncEngine = new SyncEngine(fileIndex, store.namespace('overlay-sync-v2'))
  await syncEngine.ready()
  const transferManager = new TransferManager(fileIndex)
  const protocol = new MultiRepoOverlayProtocolV2(syncEngine, transferManager, om, { filePaths })
  om.attachSyncEngine(syncEngine)
  await om.warmRepoCache()
  await om.populateFilePaths()
  t.teardown(async () => {
    protocol.destroy()
    await syncEngine.close()
    await om.close()
    await store.close()
    fs.rmSync(storeRoot, { recursive: true, force: true })
  })
  return { store, om, syncEngine, protocol }
}

test('overlay-LFS clean→smudge recovers a large file on a peer that never had it', async (t) => {
  const pub = await makeNode('pub', t)
  const cons = await makeNode('cons', t)

  const pubRepo = tmpDir('lfs-pub-repo')
  const consRepo = tmpDir('lfs-cons-repo')
  t.teardown(() => fs.rmSync(pubRepo, { recursive: true, force: true }))
  t.teardown(() => fs.rmSync(consRepo, { recursive: true, force: true }))

  // Both track the same logical repo (the consumer is a mirror of it). This is
  // what a bootstrap-bearing repo like `pgh` would be on stortron.
  await pub.om.initRepo('lfs-demo', pubRepo)
  await cons.om.initRepo('lfs-demo', consRepo)
  await pub.om.warmRepoCache()
  await cons.om.warmRepoCache()

  // ── clean (publisher) ── a multi-chunk "tarball-shaped" file → overlay.
  const data = crypto.randomBytes(3 * 1024 * 1024) // 3 MB, exercises chunking
  const oid = crypto.data(data).toString('hex')    // == the git pointer's oid
  fs.writeFileSync(path.join(pubRepo, 'bootstrap.tar.gz'), data)
  await pub.om.putFile('lfs-demo', 'bootstrap.tar.gz', { oid, size: data.length, mtime: Date.now() })

  // The consumer genuinely does NOT have it yet.
  t.absent(fs.existsSync(path.join(consRepo, 'bootstrap.tar.gz')), 'consumer has no copy before sync')
  t.is(await cons.om.findByOid(oid), null, 'consumer cannot resolve the oid before sync')

  // Pair (stand-in for the consumer joining the publisher's overlay topic that
  // `overlay-locate { ensure:true }` does on the real swarm).
  const [streamA, streamB] = makeDuplex()
  pub.protocol.attach(Protomux.from(streamA))
  cons.protocol.attach(Protomux.from(streamB))

  // ── smudge (consumer) ── resolve the pointer's oid to bytes, cross-machine.
  let resolved = null
  const start = Date.now()
  while (Date.now() - start < 15000) {
    const hit = await cons.om.findByOid(oid).catch(() => null)
    if (hit && hit.repoRoot) {
      const abs = path.join(hit.repoRoot, hit.path)
      if (fs.existsSync(abs) && crypto.data(fs.readFileSync(abs)).toString('hex') === oid) {
        resolved = abs
        break
      }
    }
    await new Promise(r => setTimeout(r, 100))
  }

  t.ok(resolved, 'consumer resolved the pointer oid to a local file (cross-machine smudge)')
  const got = fs.readFileSync(resolved)
  t.is(got.length, data.length, 'recovered size matches')
  t.is(crypto.data(got).toString('hex'), oid, 'recovered bytes are byte-identical (oid matches)')
})