OverviewFilesCommitsIssuesDocsReleasesPeers

hyper-overlay / test / integration / fetch.test.js

141 lines · javascript ref:
import test from 'brittle'
import Protomux from 'protomux'
import { Duplex } from 'streamx'
import crypto from 'hypercore-crypto'
import { tmpStore, tmpDir, fs, path } from '../helpers.js'
import { HyperOverlay } from '../../lib/overlay.js'

// In-memory duplex pair (same pattern as e2e-two-peers.js).
function makeDuplex () {
  let aWrite, bWrite
  const a = new Duplex({ write (d, cb) { bWrite(d); cb() }, read () {} })
  const b = new Duplex({ write (d, cb) { aWrite(d); cb() }, read () {} })
  aWrite = (d) => a.push(d)
  bWrite = (d) => b.push(d)
  return [a, b]
}

async function makePeer (label) {
  const ov = new HyperOverlay(tmpStore('ov2-' + label), {
    namespace: 'overlay-v2',
    destDir: tmpDir('ov2-' + label + '-dest')
  })
  await ov.ready()
  return ov
}

test('V2 facade: cross-peer fetch by content hash (multi-chunk, from scratch)', async (t) => {
  const alice = await makePeer('alice')
  const bob = await makePeer('bob')
  t.teardown(async () => { try { await alice.close() } catch {}; try { await bob.close() } catch {} })

  const [sa, sb] = makeDuplex()
  alice.attachProtocol(Protomux.from(sa))
  bob.attachProtocol(Protomux.from(sb))
  await new Promise(r => setTimeout(r, 150)) // channel handshake

  // Alice writes a REAL multi-chunk file to disk and registers it. 200KB at
  // tier 0 (avg 16KB) is ~12 chunks — exactly the bug-82 multi-chunk case the
  // old getByHash(oid) could never serve.
  const filesDir = tmpDir('ov2-alice-files')
  const content = crypto.randomBytes(200 * 1024)
  const diskPath = path.join(filesDir, 'pfp.bin')
  fs.writeFileSync(diskPath, content)
  const oid = crypto.data(content).toString('hex')

  const reg = await alice.registerFile('/attachments/' + oid, diskPath, {
    contentHash: oid, size: content.length
  })
  t.is(reg.contentHash, oid, 'registered under its content hash')

  // Bob has NOTHING locally — fetch purely by content hash over the wire.
  const got = await bob.fetchFile(oid)
  t.ok(got, 'bob resolved the file from a peer')
  t.is(got.local, false, 'came from a peer, not local disk')
  t.is(Buffer.compare(fs.readFileSync(got.destPath), content), 0, 'bytes match alice’s file')

  // Alice serves its own copy from the local fast path (no wire).
  const localGot = await alice.fetchFile(oid)
  t.ok(localGot && localGot.local, 'alice serves its own copy locally')
  t.is(Buffer.compare(fs.readFileSync(localGot.destPath), content), 0, 'local bytes match')
})

test('V2 multi-source: chunks fetched from MULTIPLE peers in parallel', async (t) => {
  const R = await makePeer('req')
  const S1 = await makePeer('seed1')
  const S2 = await makePeer('seed2')
  t.teardown(async () => {
    for (const ov of [R, S1, S2]) { try { await ov.close() } catch {} }
  })

  // Requester connects to BOTH seeders (two independent protomux links).
  const [r1, s1] = makeDuplex(); R.attachProtocol(Protomux.from(r1)); S1.attachProtocol(Protomux.from(s1))
  const [r2, s2] = makeDuplex(); R.attachProtocol(Protomux.from(r2)); S2.attachProtocol(Protomux.from(s2))
  await new Promise(r => setTimeout(r, 250))

  // Both seeders hold a FULL copy of the same content → identical chunk set
  // (content-addressed), so either can serve any chunk.
  const content = crypto.randomBytes(900 * 1024) // ~56 chunks at tier 0
  const oid = crypto.data(content).toString('hex')
  for (const [ov, label] of [[S1, 's1'], [S2, 's2']]) {
    const dir = tmpDir('ov2-' + label + '-files')
    const f = path.join(dir, 'big.bin')
    fs.writeFileSync(f, content)
    await ov.registerFile('/attachments/' + oid, f, { contentHash: oid, size: content.length })
  }

  const peers = [...R._protocol._peers.values()]
  t.is(peers.length, 2, 'requester is connected to both seeders')

  const destPath = path.join(tmpDir('ov2-req-dest'), 'out.bin')
  const stats = await R._protocol.fetchContent(oid, peers, { destPath })

  const got = fs.readFileSync(destPath)
  t.is(Buffer.compare(got, content), 0, 'assembled file matches the seeded content')
  t.ok(stats.peerCount >= 2,
    'chunks were pulled from BOTH seeders (multi-source) — peerCount=' + stats.peerCount)
})

test('V2 re-seed: a fetcher becomes a seeder (multiplication)', async (t) => {
  const S1 = await makePeer('seed')
  const F = await makePeer('fetcher')
  const G = await makePeer('late')
  t.teardown(async () => {
    for (const ov of [S1, F, G]) { try { await ov.close() } catch {} }
  })

  // S1 holds the file.
  const content = crypto.randomBytes(150 * 1024)
  const oid = crypto.data(content).toString('hex')
  const dir = tmpDir('ov2-seed-files')
  const f = path.join(dir, 'x.bin')
  fs.writeFileSync(f, content)
  await S1.registerFile('/attachments/' + oid, f, { contentHash: oid, size: content.length })

  // F connects ONLY to S1 and fetches → should re-seed.
  const [fa, sa] = makeDuplex()
  F.attachProtocol(Protomux.from(fa)); S1.attachProtocol(Protomux.from(sa))
  await new Promise(r => setTimeout(r, 200))
  const got1 = await F.fetchFile(oid)
  t.ok(got1 && Buffer.compare(fs.readFileSync(got1.destPath), content) === 0, 'F fetched the blob from S1')

  // After fetching, F serves it from its own local copy.
  const localOnF = await F.fetchFile(oid)
  t.ok(localOnF && localOnF.local, 'F re-seeded the blob (now serves it locally)')

  // G connects ONLY to F (never to S1) — it can only succeed if F is now a seeder.
  const [gf, fg] = makeDuplex()
  G.attachProtocol(Protomux.from(gf)); F.attachProtocol(Protomux.from(fg))
  await new Promise(r => setTimeout(r, 200))
  const got2 = await G.fetchFile(oid)
  t.ok(got2 && !got2.local, 'G fetched from a peer (F), not local')
  t.is(got2 && Buffer.compare(fs.readFileSync(got2.destPath), content), 0, 'G got the blob — F successfully re-seeded it')
})

test('V2 facade: fetch by unknown hash with no peers returns null', async (t) => {
  const solo = await makePeer('solo')
  t.teardown(async () => { try { await solo.close() } catch {} })
  const got = await solo.fetchFile('a'.repeat(64))
  t.is(got, null, 'no peers, not local → null (not a throw)')
})