| … | … | @@ -0,0 +1,225 @@ |
| 1 | #!/usr/bin/env node |
| 2 | // vdrive — publish, inspect and seed a Hyperdrive that stores no blob bytes. |
| 3 | // |
| 4 | // vdrive publish <srcDir> --store <driveDir> [--only a,b] |
| 5 | // vdrive info --store <driveDir> |
| 6 | // vdrive seed --store <driveDir> --root <srcDir> [--timeout <s>] |
| 7 | // |
| 8 | // The source tree IS the byte store: the drive holds pointers, and reads are |
| 9 | // served from the files under <srcDir> (or --root when seeding). |
| 10 | import { createRequire } from 'module' |
| 11 | const require = createRequire(import.meta.url) |
| 12 | |
| 13 | const fs = require('fs') |
| 14 | const path = require('path') |
| 15 | const { execFileSync } = require('child_process') |
| 16 | const idEnc = require('hypercore-id-encoding') |
| 17 | const Corestore = require('corestore') |
| 18 | const Hyperdrive = require('hyperdrive') |
| 19 | const CorestoreStorage = require('hypercore-storage') |
| 20 | const { ContentStore, build, stats } = require('../lib/storage.cjs') |
| 21 | const { chunk } = await import('../../hyper-overlay/lib/chunker.js') |
| 22 | |
| 23 | const USAGE = `vdrive — a Hyperdrive that stores no blob bytes |
| 24 | |
| 25 | vdrive publish <srcDir> --store <driveDir> [--only a,b] |
| 26 | vdrive info --store <driveDir> |
| 27 | vdrive seed --store <driveDir> --root <srcDir> [--timeout <seconds>] |
| 28 | |
| 29 | Options |
| 30 | --store <dir> where the drive lives (pointers, tree, metadata) |
| 31 | --root <dir> source tree the bytes are read from when seeding |
| 32 | --only <list> comma-separated paths to publish, relative to <srcDir> |
| 33 | --json machine-readable output |
| 34 | ` |
| 35 | |
| 36 | function parseArgs (argv) { |
| 37 | const out = { _: [], flags: {} } |
| 38 | for (let i = 0; i < argv.length; i++) { |
| 39 | const a = argv[i] |
| 40 | if (a.startsWith('--')) { |
| 41 | const key = a.slice(2) |
| 42 | const next = argv[i + 1] |
| 43 | if (next === undefined || next.startsWith('--')) out.flags[key] = true |
| 44 | else out.flags[key] = argv[++i] |
| 45 | } else out._.push(a) |
| 46 | } |
| 47 | return out |
| 48 | } |
| 49 | |
| 50 | function walk (dir, base = dir, out = []) { |
| 51 | for (const e of fs.readdirSync(dir, { withFileTypes: true })) { |
| 52 | const full = path.join(dir, e.name) |
| 53 | if (e.isDirectory()) walk(full, base, out) |
| 54 | else if (e.isFile()) out.push('/' + path.relative(base, full).split(path.sep).join('/')) |
| 55 | } |
| 56 | return out |
| 57 | } |
| 58 | |
| 59 | const du = (p) => { |
| 60 | try { |
| 61 | return Number(execFileSync('du', ['-sb', p]).toString().split('\t')[0]) |
| 62 | } catch { |
| 63 | return 0 |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | function indexTree (root, names) { |
| 68 | const content = new ContentStore() |
| 69 | const files = [] |
| 70 | for (const name of names) { |
| 71 | const file = path.join(root, name) |
| 72 | const bytes = fs.readFileSync(file) |
| 73 | const chunks = chunk(bytes) |
| 74 | for (const c of chunks) content.add(c.hash, file, c.offset, c.length) |
| 75 | files.push({ name, file, size: bytes.length, chunks }) |
| 76 | } |
| 77 | return { content, files } |
| 78 | } |
| 79 | |
| 80 | function openDrive (storeDir, content) { |
| 81 | const Storage = build(content) |
| 82 | const store = new Corestore(new Storage(storeDir)) |
| 83 | return { store, drive: new Hyperdrive(store) } |
| 84 | } |
| 85 | |
| 86 | function write (ws, buf) { |
| 87 | return new Promise((resolve) => { |
| 88 | if (ws.write(buf)) return resolve() |
| 89 | ws.once('drain', resolve) |
| 90 | }) |
| 91 | } |
| 92 | |
| 93 | async function compact (dir) { |
| 94 | const s = new CorestoreStorage(dir) |
| 95 | await s.ready() |
| 96 | await s.compact() |
| 97 | await s.close() |
| 98 | } |
| 99 | |
| 100 | async function publish (args) { |
| 101 | const src = args._[1] |
| 102 | const storeDir = args.flags.store |
| 103 | if (!src || !storeDir) return fail('publish needs <srcDir> and --store <driveDir>') |
| 104 | |
| 105 | let names = walk(src) |
| 106 | if (typeof args.flags.only === 'string') { |
| 107 | const want = new Set(args.flags.only.split(',').map((s) => '/' + s.trim().replace(/^\//, ''))) |
| 108 | names = names.filter((n) => want.has(n)) |
| 109 | } |
| 110 | if (names.length === 0) return fail('nothing to publish under ' + src) |
| 111 | |
| 112 | const { content, files } = indexTree(src, names) |
| 113 | const { store, drive } = openDrive(storeDir, content) |
| 114 | await drive.ready() |
| 115 | |
| 116 | for (const f of files) { |
| 117 | // 0-byte + dedup is an upstream crash on read (hyperblobs seekBlockMap) |
| 118 | const ws = drive.createWriteStream(f.name, f.size === 0 ? {} : { dedup: true }) |
| 119 | for (const c of f.chunks) await write(ws, c.data) |
| 120 | await new Promise((resolve, reject) => { |
| 121 | ws.once('error', reject) |
| 122 | ws.once('close', resolve) |
| 123 | ws.end() |
| 124 | }) |
| 125 | } |
| 126 | |
| 127 | await drive.getBlobs() |
| 128 | const result = { |
| 129 | link: 'pear://' + idEnc.normalize(drive.key), |
| 130 | key: drive.key.toString('hex'), |
| 131 | files: files.length, |
| 132 | bytes: files.reduce((a, f) => a + f.size, 0), |
| 133 | blocks: drive.blobs.core.length, |
| 134 | pointerBlocks: stats.putRedirected, |
| 135 | literalBlocks: stats.putStored |
| 136 | } |
| 137 | await drive.close() |
| 138 | await store.close() |
| 139 | await compact(storeDir) |
| 140 | result.driveStoreBytes = du(storeDir) |
| 141 | content.close() |
| 142 | return result |
| 143 | } |
| 144 | |
| 145 | async function info (args) { |
| 146 | const storeDir = args.flags.store |
| 147 | if (!storeDir) return fail('info needs --store <driveDir>') |
| 148 | const content = new ContentStore() |
| 149 | const { store, drive } = openDrive(storeDir, content) |
| 150 | await drive.ready() |
| 151 | await drive.getBlobs() |
| 152 | const files = [] |
| 153 | for await (const entry of drive.list('/')) files.push({ path: entry.key, size: entry.value.blob ? entry.value.blob.byteLength : 0 }) |
| 154 | const result = { |
| 155 | link: 'pear://' + idEnc.normalize(drive.key), |
| 156 | key: drive.key.toString('hex'), |
| 157 | files: files.length, |
| 158 | bytes: files.reduce((a, f) => a + f.size, 0), |
| 159 | blocks: drive.blobs.core.length, |
| 160 | driveStoreBytes: du(storeDir) |
| 161 | } |
| 162 | await drive.close() |
| 163 | await store.close() |
| 164 | content.close() |
| 165 | return result |
| 166 | } |
| 167 | |
| 168 | async function seed (args) { |
| 169 | const storeDir = args.flags.store |
| 170 | const root = args.flags.root |
| 171 | if (!storeDir || !root) return fail('seed needs --store <driveDir> and --root <srcDir>') |
| 172 | const Hyperswarm = require('hyperswarm') |
| 173 | |
| 174 | const { content } = indexTree(root, walk(root)) |
| 175 | const { store, drive } = openDrive(storeDir, content) |
| 176 | await drive.ready() |
| 177 | await drive.getBlobs() |
| 178 | |
| 179 | const swarm = new Hyperswarm() |
| 180 | swarm.on('connection', (conn) => store.replicate(conn)) |
| 181 | const discovery = swarm.join(drive.discoveryKey, { server: true, client: false }) |
| 182 | await discovery.flushed() |
| 183 | |
| 184 | const link = 'pear://' + idEnc.normalize(drive.key) |
| 185 | process.stdout.write(JSON.stringify({ seeding: link, driveStoreBytes: du(storeDir) }) + '\n') |
| 186 | |
| 187 | const timeout = args.flags.timeout ? Number(args.flags.timeout) * 1000 : 0 |
| 188 | if (timeout) setTimeout(shutdown, timeout) |
| 189 | process.on('SIGINT', shutdown) |
| 190 | process.on('SIGTERM', shutdown) |
| 191 | |
| 192 | async function shutdown () { |
| 193 | process.stdout.write( |
| 194 | JSON.stringify({ served: stats.bytesServed, blocks: stats.getRedirected }) + '\n' |
| 195 | ) |
| 196 | await swarm.destroy() |
| 197 | await drive.close() |
| 198 | await store.close() |
| 199 | content.close() |
| 200 | process.exit(0) |
| 201 | } |
| 202 | return null |
| 203 | } |
| 204 | |
| 205 | function fail (msg) { |
| 206 | process.stderr.write('vdrive: ' + msg + '\n') |
| 207 | process.exit(1) |
| 208 | } |
| 209 | |
| 210 | const args = parseArgs(process.argv.slice(2)) |
| 211 | const cmd = args._[0] |
| 212 | |
| 213 | const handlers = { publish, info, seed } |
| 214 | if (!cmd || !handlers[cmd]) { |
| 215 | process.stdout.write(USAGE) |
| 216 | process.exit(cmd ? 1 : 0) |
| 217 | } |
| 218 | |
| 219 | const result = await handlers[cmd](args) |
| 220 | if (result !== null) { |
| 221 | if (args.flags.json) process.stdout.write(JSON.stringify(result) + '\n') |
| 222 | else { |
| 223 | for (const [k, v] of Object.entries(result)) process.stdout.write(k + ': ' + v + '\n') |
| 224 | } |
| 225 | } |