pgh-sidecar / test / overlay-mirror-reserve.test.js

/**
 * overlay-mirror, A2 path: a forge backend with NO working tree replicates a
 * foreign-published repo's large file by fetching it into a CONTENT-ADDRESSED
 * mirror-spool (<spool>/<oid>) and then RE-SERVES it to a third party — becoming
 * a second source. This is the substantive path of the sidecar's
 * `_overlayFetchOid` (raw MultiRepoOverlayProtocolV2.fetchContent into a
 * content-addressed destPath) + the content-addressed serve fallback
 * (hyper-overlay 0.2.8 `<syncBaseDir>/<contentHash>`). The only piece not
 * exercised here is the Hyperswarm topic join (plumbing; copied verbatim from
 * the proven overlay-locate path).
 *
 * Same in-process Duplex pairing as overlay-lfs-cross-machine.test.js.
 */

import test from 'brittle'
import Corestore from 'corestore'
import Protomux from 'protomux'
import { Duplex } from 'streamx'
import crypto from 'hypercore-crypto'
import fs from '#node:fs'
import path from '#node:path'
import os from '#node:os'

import { OverlayManager } from '../lib/overlay-manager.js'
import { SyncEngine } from 'hyper-overlay/sync-engine'
import { TransferManager } from 'hyper-overlay/transfer'
import { MultiRepoOverlayProtocolV2 } from '../lib/multi-repo-overlay-protocol.js'

function tmpDir (prefix) { return fs.mkdtempSync(path.join(os.tmpdir(), prefix + '-')) }
const settle = (ms = 400) => new Promise(r => setTimeout(r, ms))

function makeDuplex () {
  let aWrite, bWrite
  const a = new Duplex({ write (d, cb) { bWrite(d); cb() }, read () {} })
  const b = new Duplex({ write (d, cb) { aWrite(d); cb() }, read () {} })
  aWrite = (d) => a.push(d)
  bWrite = (d) => b.push(d)
  return [a, b]
}

async function makeNode (label, t, opts = {}) {
  const storeRoot = tmpDir('mir-' + label + '-store')
  const store = new Corestore(storeRoot)
  await store.ready()
  const filePaths = new Map()
  const om = new OverlayManager(store, { filePaths })
  const fileIndex = await om.getFileIndex()
  const syncEngine = new SyncEngine(fileIndex, store.namespace('overlay-sync-v2'))
  await syncEngine.ready()
  const transferManager = new TransferManager(fileIndex)
  const protocol = new MultiRepoOverlayProtocolV2(syncEngine, transferManager, om, { filePaths, syncBaseDirs: opts.syncBaseDirs || [] })
  om.attachSyncEngine(syncEngine)
  await om.warmRepoCache()
  await om.populateFilePaths()
  t.teardown(async () => {
    try { protocol.destroy() } catch {}
    try { await syncEngine.close() } catch {}
    try { await om.close() } catch {}
    try { await store.close() } catch {}
    try { fs.rmSync(storeRoot, { recursive: true, force: true }) } catch {}
  })
  return { store, om, syncEngine, protocol }
}

async function fetchInto (node, oid, destPath, timeout = 10000) {
  const peers = [...node.protocol._peers.values()]
  if (peers.length === 0) throw new Error('no peers')
  await node.protocol.fetchContent(oid, peers, { destPath, timeout })
}

test('overlay-mirror A2: no-working-tree backend fetches into the spool and re-serves', async (t) => {
  // ── Publisher: a registered repo with a real working-tree file (the origin). ──
  const pub = await makeNode('pub', t)
  const pubRepo = tmpDir('mir-pub-repo')
  t.teardown(() => { try { fs.rmSync(pubRepo, { recursive: true, force: true }) } catch {} })
  await pub.om.initRepo('hyper-tps', pubRepo)
  await pub.om.warmRepoCache()
  await pub.om.populateFilePaths()

  const data = crypto.randomBytes(2 * 1024 * 1024) // 2 MB, multi-chunk
  const oid = crypto.data(data).toString('hex')    // == the git pointer oid
  fs.mkdirSync(path.join(pubRepo, 'assets'), { recursive: true })
  fs.writeFileSync(path.join(pubRepo, 'assets', 'foo.png'), data)
  await pub.om.putFile('hyper-tps', 'assets/foo.png', { oid, size: data.length, mtime: Date.now() })

  // ── Mirror: NO working tree for the repo; only a content-addressed spool. ──
  const mirrorSpool = tmpDir('mir-spool')
  const mir = await makeNode('mir', t, { syncBaseDirs: [mirrorSpool] })
  t.is(await mir.om.findByOid(oid), null, 'mirror cannot resolve the oid before fetch (no working tree)')

  // Pair publisher ↔ mirror (stand-in for the mirror joining the publisher's
  // overlay topic, which _overlayFetchOid does on the real swarm).
  const [sa, sb] = makeDuplex()
  pub.protocol.attach(Protomux.from(sa))
  mir.protocol.attach(Protomux.from(sb))
  await settle()

  // The core of _overlayFetchOid: pull the oid into <mirrorSpool>/<oid>.
  const mirDest = path.join(mirrorSpool, oid)
  await fetchInto(mir, oid, mirDest)
  t.ok(fs.existsSync(mirDest), 'mirror stored the blob content-addressed at <mirrorSpool>/<oid>')
  t.is(crypto.data(fs.readFileSync(mirDest)).toString('hex'), oid, 'mirror bytes verify against the oid')

  // ── Re-serve: a consumer connects to the MIRROR (not the publisher) and pulls
  // the oid. The mirror serves from its spool via the content-addressed fallback
  // — no working tree, no in-memory registration needed. ──
  const cons = await makeNode('cons', t)
  const [sc, sd] = makeDuplex()
  mir.protocol.attach(Protomux.from(sc))
  cons.protocol.attach(Protomux.from(sd))
  await settle()

  const consDest = path.join(tmpDir('mir-cons-dest'), oid)
  await fetchInto(cons, oid, consDest)
  t.ok(fs.existsSync(consDest), 'consumer fetched the oid FROM the mirror')
  t.is(crypto.data(fs.readFileSync(consDest)).toString('hex'), oid, 'consumer bytes byte-identical — the mirror is now a real second source')
})