hyper-vdrive / test / truncate-clear.mjs

// truncate / clear — does deleting blocks free pointer records, and does the
// virtual drive behave the same as a stock one?
//   node test/truncate-clear.mjs <workDir>
import { createRequire } from 'module'
const require = createRequire(import.meta.url)

const fs = require('fs')
const { execFileSync } = require('child_process')
const crypto = require('hypercore-crypto')
const Corestore = require('corestore')
const Hyperdrive = require('hyperdrive')
const CorestoreStorage = require('hypercore-storage')
const { ContentStore, build, stats } = require('../lib/storage.cjs')
const { chunk } = await import('../../hyper-overlay/lib/chunker.js')

const ROOT = process.argv[2]
const PRIMARY = Buffer.alloc(32, 7)
const du = (p) => Number(execFileSync('du', ['-sb', p]).toString().split('\t')[0])

async function compactedSize (dir) {
  const s = new CorestoreStorage(dir)
  await s.ready()
  await s.compact()
  await s.close()
  return du(dir)
}

const srcDir = ROOT + '/src'
fs.mkdirSync(srcDir, { recursive: true })

// v1 and v2 of the same path share most blocks (dedup seeds from the previous
// entry at that path) — that sharing is what makes clear() dangerous.
const v1 = crypto.randomBytes(4 * 1024 * 1024)
const v2 = Buffer.concat([v1.subarray(0, 1024 * 1024), crypto.randomBytes(256 * 1024), v1.subarray(1024 * 1024 + 256 * 1024)])
const other = crypto.randomBytes(2 * 1024 * 1024)

const FILES = [
  { name: '/app.bin', bytes: v1, file: srcDir + '/v1' },
  { name: '/app.bin', bytes: v2, file: srcDir + '/v2' },
  { name: '/other.bin', bytes: other, file: srcDir + '/other' }
]
for (const f of FILES) fs.writeFileSync(f.file, f.bytes)

const content = new ContentStore()
for (const f of FILES) {
  f.chunks = chunk(f.bytes)
  for (const c of f.chunks) content.add(c.hash, f.file, c.offset, c.length)
}

function write (ws, buf) {
  return new Promise((resolve) => {
    if (ws.write(buf)) return resolve()
    ws.once('drain', resolve)
  })
}

async function open (dir, virtual) {
  const Storage = virtual ? build(content) : null
  const store = new Corestore(virtual ? new Storage(dir) : dir, { primaryKey: PRIMARY, unsafe: true })
  const drive = new Hyperdrive(store)
  await drive.ready()
  await drive.getBlobs()
  return { store, drive }
}

async function run (dir, virtual) {
  const out = { virtual }

  {
    const { store, drive } = await open(dir, virtual)
    for (const f of FILES) {
      const ws = drive.createWriteStream(f.name, { dedup: true })
      for (const c of f.chunks) await write(ws, c.data)
      await new Promise((resolve, reject) => {
        ws.once('error', reject)
        ws.once('close', resolve)
        ws.end()
      })
    }
    out.blobsLength = drive.blobs.core.length
    await drive.close()
    await store.close()
  }
  out.sizeAfterPublish = await compactedSize(dir)

  // clear the current /app.bin entry
  {
    const { store, drive } = await open(dir, virtual)
    const entry = await drive.entry('/app.bin')
    const map = await drive.blobs.getBlockMap(entry.value.blob)
    const blocks = map.blocks.map((b) => b.index)
    out.appBlocks = blocks.length

    await drive.clear('/app.bin')

    out.hasAfterClear = await drive.blobs.core.has(blocks[0])
    const app = await drive.get('/app.bin', { timeout: 2000 }).catch((e) => e.code || 'throw')
    out.readAppAfterClear = typeof app === 'string' ? app : app === null ? 'null' : 'bytes:' + app.length
    const oth = await drive.get('/other.bin', { timeout: 2000 }).catch((e) => e.code || 'throw')
    out.readOtherAfterClear =
      typeof oth === 'string' ? oth : oth === null ? 'null' : oth.equals(other) ? 'ok' : 'CORRUPT'
    await drive.close()
    await store.close()
  }
  out.sizeAfterClear = await compactedSize(dir)

  // truncate the blobs core outright
  {
    const { store, drive } = await open(dir, virtual)
    const before = drive.blobs.core.length
    const target = Math.floor(before / 2)
    let truncateResult = 'ok'
    try {
      await drive.blobs.core.truncate(target)
    } catch (err) {
      truncateResult = (err.code || 'ERR') + ': ' + err.message
    }
    out.truncate = { before, target, result: truncateResult, after: drive.blobs.core.length }

    const oth = await drive.get('/other.bin', { timeout: 2000 }).catch((e) => e.code || 'throw')
    out.readOtherAfterTruncate =
      typeof oth === 'string' ? oth : oth === null ? 'null' : oth.equals(other) ? 'ok' : 'CORRUPT'
    await drive.close()
    await store.close()
  }
  out.sizeAfterTruncate = await compactedSize(dir)

  return out
}

const stock = await run(ROOT + '/stock', false)
const virt = await run(ROOT + '/virtual', true)
content.close()

console.log(JSON.stringify({ stock, virtual: virt, stats }, null, 2))