#!/usr/bin/env node
// vdrive — publish, inspect and seed a Hyperdrive that stores no blob bytes.
//
// vdrive publish <srcDir> --store <driveDir> [--only a,b]
// vdrive info --store <driveDir>
// vdrive seed --store <driveDir> --root <srcDir> [--timeout <s>]
//
// The source tree IS the byte store: the drive holds pointers, and reads are
// served from the files under <srcDir> (or --root when seeding).
import { createRequire } from 'module'
const require = createRequire(import.meta.url)
const fs = require('fs')
const path = require('path')
const { execFileSync } = require('child_process')
const idEnc = require('hypercore-id-encoding')
const Corestore = require('corestore')
const Hyperdrive = require('hyperdrive')
const CorestoreStorage = require('hypercore-storage')
const { ContentStore, build, stats } = require('../lib/storage.cjs')
const { publishTree } = require('../lib/publish.cjs')
const { chunk } = await import('../../hyper-overlay/lib/chunker.js')
const USAGE = `vdrive — a Hyperdrive that stores no blob bytes
vdrive publish <srcDir> --store <driveDir> [--only a,b]
vdrive info --store <driveDir>
vdrive seed --store <driveDir> --root <srcDir> [--timeout <seconds>]
Options
--store <dir> where the drive lives (pointers, tree, metadata)
--root <dir> source tree the bytes are read from when seeding
--only <list> comma-separated paths to publish, relative to <srcDir>
--json machine-readable output
`
function parseArgs (argv) {
const out = { _: [], flags: {} }
for (let i = 0; i < argv.length; i++) {
const a = argv[i]
if (a.startsWith('--')) {
const key = a.slice(2)
const next = argv[i + 1]
if (next === undefined || next.startsWith('--')) out.flags[key] = true
else out.flags[key] = argv[++i]
} else out._.push(a)
}
return out
}
function walk (dir, base = dir, out = []) {
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
const full = path.join(dir, e.name)
if (e.isDirectory()) walk(full, base, out)
else if (e.isFile()) out.push('/' + path.relative(base, full).split(path.sep).join('/'))
}
return out
}
const du = (p) => {
try {
return Number(execFileSync('du', ['-sb', p]).toString().split('\t')[0])
} catch {
return 0
}
}
function indexTree (root, names) {
const content = new ContentStore()
const files = []
for (const name of names) {
const file = path.join(root, name)
const bytes = fs.readFileSync(file)
const chunks = chunk(bytes)
for (const c of chunks) content.add(c.hash, file, c.offset, c.length)
files.push({ name, file, size: bytes.length, chunks })
}
return { content, files }
}
function openDrive (storeDir, content) {
const Storage = build(content)
const store = new Corestore(new Storage(storeDir))
return { store, drive: new Hyperdrive(store) }
}
function write (ws, buf) {
return new Promise((resolve) => {
if (ws.write(buf)) return resolve()
ws.once('drain', resolve)
})
}
async function compact (dir) {
const s = new CorestoreStorage(dir)
await s.ready()
await s.compact()
await s.close()
}
async function publish (args) {
const src = args._[1]
const storeDir = args.flags.store
if (!src || !storeDir) return fail('publish needs <srcDir> and --store <driveDir>')
let names = walk(src)
if (typeof args.flags.only === 'string') {
const want = new Set(args.flags.only.split(',').map((s) => '/' + s.trim().replace(/^\//, '')))
names = names.filter((n) => want.has(n))
}
if (names.length === 0) return fail('nothing to publish under ' + src)
// --root anchors the locations index. Successive releases usually live in
// sibling directories under one root; pass it so the index written now still
// resolves the bytes of releases published earlier.
const root = args.flags.root ? path.resolve(args.flags.root) : path.resolve(src)
const result = await publishTree({
storeDir,
root,
chunk,
files: names.map((name) => ({ name, file: path.join(src, name) }))
})
await compact(storeDir)
return { ...result, driveStoreBytes: du(storeDir) }
}
async function info (args) {
const storeDir = args.flags.store
if (!storeDir) return fail('info needs --store <driveDir>')
const content = new ContentStore()
const { store, drive } = openDrive(storeDir, content)
await drive.ready()
await drive.getBlobs()
const files = []
for await (const entry of drive.list('/')) files.push({ path: entry.key, size: entry.value.blob ? entry.value.blob.byteLength : 0 })
const result = {
link: 'pear://' + idEnc.normalize(drive.key),
key: drive.key.toString('hex'),
files: files.length,
bytes: files.reduce((a, f) => a + f.size, 0),
blocks: drive.blobs.core.length,
driveStoreBytes: du(storeDir)
}
await drive.close()
await store.close()
content.close()
return result
}
async function seed (args) {
const storeDir = args.flags.store
const root = args.flags.root
if (!storeDir || !root) return fail('seed needs --store <driveDir> and --root <srcDir>')
const Hyperswarm = require('hyperswarm')
const { content } = indexTree(root, walk(root))
const { store, drive } = openDrive(storeDir, content)
await drive.ready()
await drive.getBlobs()
const swarm = new Hyperswarm()
swarm.on('connection', (conn) => store.replicate(conn))
const discovery = swarm.join(drive.discoveryKey, { server: true, client: false })
await discovery.flushed()
const link = 'pear://' + idEnc.normalize(drive.key)
process.stdout.write(JSON.stringify({ seeding: link, driveStoreBytes: du(storeDir) }) + '\n')
const timeout = args.flags.timeout ? Number(args.flags.timeout) * 1000 : 0
if (timeout) setTimeout(shutdown, timeout)
process.on('SIGINT', shutdown)
process.on('SIGTERM', shutdown)
async function shutdown () {
process.stdout.write(
JSON.stringify({ served: stats.bytesServed, blocks: stats.getRedirected }) + '\n'
)
await swarm.destroy()
await drive.close()
await store.close()
content.close()
process.exit(0)
}
return null
}
function fail (msg) {
process.stderr.write('vdrive: ' + msg + '\n')
process.exit(1)
}
const args = parseArgs(process.argv.slice(2))
const cmd = args._[0]
const handlers = { publish, info, seed }
if (!cmd || !handlers[cmd]) {
process.stdout.write(USAGE)
process.exit(cmd ? 1 : 0)
}
const result = await handlers[cmd](args)
if (result !== null) {
if (args.flags.json) process.stdout.write(JSON.stringify(result) + '\n')
else {
for (const [k, v] of Object.entries(result)) process.stdout.write(k + ': ' + v + '\n')
}
}