hyper-vdrive / test / gc-bitfield.mjs

// Step 2 — GC / bitfield pairing.
// Q1: after the overlay drops bytes, does the core still advertise the block?
// Q2: what does a PEER experience when it requests an advertised-but-missing block?
// Q3: does core.clear() on that range make the lie go away?
import { createRequire } from 'module'
const require = createRequire(import.meta.url)

const fs = require('fs')
const crypto = require('hypercore-crypto')
const Corestore = require('corestore')
const Hyperdrive = require('hyperdrive')
const { ContentStore, build, stats } = require('../lib/storage.cjs')
const { chunk } = await import('../../hyper-overlay/lib/chunker.js')

const ROOT = process.argv[2]

// A block we advertise but cannot produce surfaces as an UNCAUGHT REQUEST_TIMEOUT
// from the replicator's timer, not as a rejection of get(). Capture it so the
// probe can finish and report rather than dying.
const uncaught = []
process.on('uncaughtException', (err) => {
  uncaught.push(err.code || err.message)
})

const srcDir = ROOT + '/src'
fs.mkdirSync(srcDir, { recursive: true })

const FILES = [
  { name: '/keep.bin', bytes: crypto.randomBytes(3 * 1024 * 1024) },
  { name: '/victim.bin', bytes: crypto.randomBytes(3 * 1024 * 1024) }
]
for (const f of FILES) {
  f.file = srcDir + f.name
  fs.writeFileSync(f.file, f.bytes)
  f.chunks = chunk(f.bytes)
}

const content = new ContentStore()
for (const f of FILES) for (const c of f.chunks) content.add(c.hash, f.file, c.offset, c.length)

const Storage = build(content)
const hostDir = ROOT + '/host'

function write (ws, buf) {
  return new Promise((resolve) => {
    if (ws.write(buf)) return resolve()
    ws.once('drain', resolve)
  })
}

async function openHost () {
  const store = new Corestore(new Storage(hostDir), { primaryKey: Buffer.alloc(32, 7), unsafe: true })
  const drive = new Hyperdrive(store)
  await drive.ready()
  await drive.getBlobs()
  return { store, drive }
}

// publish
{
  const { store, drive } = await openHost()
  for (const f of FILES) {
    const ws = drive.createWriteStream(f.name, { dedup: true })
    for (const c of f.chunks) await write(ws, c.data)
    await new Promise((resolve, reject) => {
      ws.once('error', reject)
      ws.once('close', resolve)
      ws.end()
    })
  }
  await drive.close()
  await store.close()
}

const host = await openHost()
const driveKey = host.drive.key

// which blocks belong to /victim.bin?
const victimEntry = await host.drive.entry('/victim.bin')
const victimBlob = victimEntry.value.blob
const victimMap = await host.drive.blobs.getBlockMap(victimBlob)
const victimBlocks = victimMap.blocks.map((b) => b.index)

// peer setup
const client = new Corestore(ROOT + '/peer', { primaryKey: Buffer.alloc(32, 9), unsafe: true })
const clientDrive = new Hyperdrive(client, driveKey)
await clientDrive.ready()
const s1 = host.store.replicate(true)
const s2 = client.replicate(false)
s1.pipe(s2).pipe(s1)
await clientDrive.core.update({ wait: true })

const result = {}

// baseline: peer can fetch a file normally
const keep = await clientDrive.get('/keep.bin')
result.peerFetchedKeepOk = keep !== null && keep.equals(FILES[0].bytes)

// Q1 — the overlay GCs victim.bin's bytes
for (const c of FILES[1].chunks) content.locations.delete(c.hash)
result.hostStillAdvertises_afterBytesGone = await host.drive.blobs.core.has(victimBlocks[0])

// Q2 — peer requests the advertised-but-missing block
const t0 = Date.now()
try {
  const got = await clientDrive.get('/victim.bin', { timeout: 5000 })
  result.peerResult = got === null ? 'null' : 'bytes:' + got.length
} catch (err) {
  result.peerResult = 'throw:' + (err.code || err.message)
}
result.peerWaitedMs = Date.now() - t0

// Q3 — pair the GC with a clear of those blocks
await host.drive.blobs.core.clear(victimBlocks[0], victimBlocks[victimBlocks.length - 1] + 1)
result.hostAdvertises_afterClear = await host.drive.blobs.core.has(victimBlocks[0])

// peer asks again, from a clean session
const client2 = new Corestore(ROOT + '/peer2', { primaryKey: Buffer.alloc(32, 11), unsafe: true })
const clientDrive2 = new Hyperdrive(client2, driveKey)
await clientDrive2.ready()
const s3 = host.store.replicate(true)
const s4 = client2.replicate(false)
s3.pipe(s4).pipe(s3)
await clientDrive2.core.update({ wait: true })

const t1 = Date.now()
try {
  const got = await clientDrive2.get('/victim.bin', { timeout: 5000 })
  result.peerResultAfterClear = got === null ? 'null' : 'bytes:' + got.length
} catch (err) {
  result.peerResultAfterClear = 'throw:' + (err.code || err.message)
}
result.peerWaitedMsAfterClear = Date.now() - t1

// and the file that was never GC'd still works
const keep2 = await clientDrive2.get('/keep.bin', { timeout: 5000 })
result.keepStillFetchableAfterClear = keep2 !== null && keep2.equals(FILES[0].bytes)

result.victimBlockCount = victimBlocks.length
result.stats = { ...stats }
result.uncaught = uncaught

for (const s of [s1, s2, s3, s4]) s.destroy()
await clientDrive2.close()
await client2.close()
await clientDrive.close()
await client.close()
await host.drive.close()
await host.store.close()
content.close()

console.log(JSON.stringify(result, null, 2))