| … | … | @@ -0,0 +1,135 @@ |
| 1 | // truncate / clear — does deleting blocks free pointer records, and does the |
| 2 | // virtual drive behave the same as a stock one? |
| 3 | // node test/truncate-clear.mjs <workDir> |
| 4 | import { createRequire } from 'module' |
| 5 | const require = createRequire(import.meta.url) |
| 6 | |
| 7 | const fs = require('fs') |
| 8 | const { execFileSync } = require('child_process') |
| 9 | const crypto = require('hypercore-crypto') |
| 10 | const Corestore = require('corestore') |
| 11 | const Hyperdrive = require('hyperdrive') |
| 12 | const CorestoreStorage = require('hypercore-storage') |
| 13 | const { ContentStore, build, stats } = require('../lib/storage.cjs') |
| 14 | const { chunk } = await import('../../hyper-overlay/lib/chunker.js') |
| 15 | |
| 16 | const ROOT = process.argv[2] |
| 17 | const PRIMARY = Buffer.alloc(32, 7) |
| 18 | const du = (p) => Number(execFileSync('du', ['-sb', p]).toString().split('\t')[0]) |
| 19 | |
| 20 | async function compactedSize (dir) { |
| 21 | const s = new CorestoreStorage(dir) |
| 22 | await s.ready() |
| 23 | await s.compact() |
| 24 | await s.close() |
| 25 | return du(dir) |
| 26 | } |
| 27 | |
| 28 | const srcDir = ROOT + '/src' |
| 29 | fs.mkdirSync(srcDir, { recursive: true }) |
| 30 | |
| 31 | // v1 and v2 of the same path share most blocks (dedup seeds from the previous |
| 32 | // entry at that path) — that sharing is what makes clear() dangerous. |
| 33 | const v1 = crypto.randomBytes(4 * 1024 * 1024) |
| 34 | const v2 = Buffer.concat([v1.subarray(0, 1024 * 1024), crypto.randomBytes(256 * 1024), v1.subarray(1024 * 1024 + 256 * 1024)]) |
| 35 | const other = crypto.randomBytes(2 * 1024 * 1024) |
| 36 | |
| 37 | const FILES = [ |
| 38 | { name: '/app.bin', bytes: v1, file: srcDir + '/v1' }, |
| 39 | { name: '/app.bin', bytes: v2, file: srcDir + '/v2' }, |
| 40 | { name: '/other.bin', bytes: other, file: srcDir + '/other' } |
| 41 | ] |
| 42 | for (const f of FILES) fs.writeFileSync(f.file, f.bytes) |
| 43 | |
| 44 | const content = new ContentStore() |
| 45 | for (const f of FILES) { |
| 46 | f.chunks = chunk(f.bytes) |
| 47 | for (const c of f.chunks) content.add(c.hash, f.file, c.offset, c.length) |
| 48 | } |
| 49 | |
| 50 | function write (ws, buf) { |
| 51 | return new Promise((resolve) => { |
| 52 | if (ws.write(buf)) return resolve() |
| 53 | ws.once('drain', resolve) |
| 54 | }) |
| 55 | } |
| 56 | |
| 57 | async function open (dir, virtual) { |
| 58 | const Storage = virtual ? build(content) : null |
| 59 | const store = new Corestore(virtual ? new Storage(dir) : dir, { primaryKey: PRIMARY, unsafe: true }) |
| 60 | const drive = new Hyperdrive(store) |
| 61 | await drive.ready() |
| 62 | await drive.getBlobs() |
| 63 | return { store, drive } |
| 64 | } |
| 65 | |
| 66 | async function run (dir, virtual) { |
| 67 | const out = { virtual } |
| 68 | |
| 69 | { |
| 70 | const { store, drive } = await open(dir, virtual) |
| 71 | for (const f of FILES) { |
| 72 | const ws = drive.createWriteStream(f.name, { dedup: true }) |
| 73 | for (const c of f.chunks) await write(ws, c.data) |
| 74 | await new Promise((resolve, reject) => { |
| 75 | ws.once('error', reject) |
| 76 | ws.once('close', resolve) |
| 77 | ws.end() |
| 78 | }) |
| 79 | } |
| 80 | out.blobsLength = drive.blobs.core.length |
| 81 | await drive.close() |
| 82 | await store.close() |
| 83 | } |
| 84 | out.sizeAfterPublish = await compactedSize(dir) |
| 85 | |
| 86 | // clear the current /app.bin entry |
| 87 | { |
| 88 | const { store, drive } = await open(dir, virtual) |
| 89 | const entry = await drive.entry('/app.bin') |
| 90 | const map = await drive.blobs.getBlockMap(entry.value.blob) |
| 91 | const blocks = map.blocks.map((b) => b.index) |
| 92 | out.appBlocks = blocks.length |
| 93 | |
| 94 | await drive.clear('/app.bin') |
| 95 | |
| 96 | out.hasAfterClear = await drive.blobs.core.has(blocks[0]) |
| 97 | const app = await drive.get('/app.bin', { timeout: 2000 }).catch((e) => e.code || 'throw') |
| 98 | out.readAppAfterClear = typeof app === 'string' ? app : app === null ? 'null' : 'bytes:' + app.length |
| 99 | const oth = await drive.get('/other.bin', { timeout: 2000 }).catch((e) => e.code || 'throw') |
| 100 | out.readOtherAfterClear = |
| 101 | typeof oth === 'string' ? oth : oth === null ? 'null' : oth.equals(other) ? 'ok' : 'CORRUPT' |
| 102 | await drive.close() |
| 103 | await store.close() |
| 104 | } |
| 105 | out.sizeAfterClear = await compactedSize(dir) |
| 106 | |
| 107 | // truncate the blobs core outright |
| 108 | { |
| 109 | const { store, drive } = await open(dir, virtual) |
| 110 | const before = drive.blobs.core.length |
| 111 | const target = Math.floor(before / 2) |
| 112 | let truncateResult = 'ok' |
| 113 | try { |
| 114 | await drive.blobs.core.truncate(target) |
| 115 | } catch (err) { |
| 116 | truncateResult = (err.code || 'ERR') + ': ' + err.message |
| 117 | } |
| 118 | out.truncate = { before, target, result: truncateResult, after: drive.blobs.core.length } |
| 119 | |
| 120 | const oth = await drive.get('/other.bin', { timeout: 2000 }).catch((e) => e.code || 'throw') |
| 121 | out.readOtherAfterTruncate = |
| 122 | typeof oth === 'string' ? oth : oth === null ? 'null' : oth.equals(other) ? 'ok' : 'CORRUPT' |
| 123 | await drive.close() |
| 124 | await store.close() |
| 125 | } |
| 126 | out.sizeAfterTruncate = await compactedSize(dir) |
| 127 | |
| 128 | return out |
| 129 | } |
| 130 | |
| 131 | const stock = await run(ROOT + '/stock', false) |
| 132 | const virt = await run(ROOT + '/virtual', true) |
| 133 | content.close() |
| 134 | |
| 135 | console.log(JSON.stringify({ stock, virtual: virt, stats }, null, 2)) |