2e9fe5b · hyper-vdrive

Changed files (4) +292 -10

README.md +7 -4

…	…	@@ -99,10 +99,13 @@ clear/truncate, and crash-during-append leaving no tree node without its pointer
99	99
100	100	Known gaps, in the order they'd bite:
101	101
102		1. GC must be paired. Dropping bytes does not update the bitfield — the core goes
103		on advertising blocks it can't produce. GC has to clear the block range too. And with
104		`dedup: true`, old releases share blocks with new ones, so ranges must be computed
105		from the content store's refcounts, never from a version's block range.
	102	1. GC should be paired, but need not be atomic. Dropping bytes does not update the
	103	bitfield, so the core goes on advertising blocks it cannot produce. Measured with two
	104	seeders, that is survivable — the replicator routes around it and a second source
	105	serves the data in the same ~120 ms. So clearing the block range is hygiene rather
	106	than a correctness requirement, and can be done opportunistically. What does matter:
	107	with `dedup: true`, old releases share blocks with new ones, so ranges must be
	108	computed from the content store's refcounts, never from a version's block range.
106	109	2. Block streams (`createBlockStream`, used by audit/copy-prologue) yield raw tagged
107	110	records; they need decoding like `getBlock` does.
108	111	3. Encryption defeats virtualisation — measured: 0 of 105 blocks redirect, the store

RESULTS.md +27 -6

…	…	@@ -240,12 +240,33 @@ after core.clear(range): host advertises FALSE
240	240	unrelated file still fetchable TRUE
241	241	```
242	242
243		`core.clear()` on the affected range is the correct pairing. Not demonstrated: that
244		clearing turns a stall into "ask another peer" — with one seeder both look like a
245		timeout, so proving it needs a two-seeder test. One run of the probe died with an
246		uncaught `REQUEST_TIMEOUT` thrown from `hypercore/lib/replicator.js:3371` rather than
247		rejecting `get()`; the rerun did not reproduce it. Racy, but a long-lived seeder in that
248		state could take an uncaught exception.
	243	`core.clear()` on the affected range makes the advertisement truthful again. One run of
	244	the probe died with an uncaught `REQUEST_TIMEOUT` thrown from
	245	`hypercore/lib/replicator.js:3371` rather than rejecting `get()`; the rerun did not
	246	reproduce it. Racy, but a long-lived seeder in that state could take an uncaught
	247	exception.
	248
	249	### Two seeders — the lie is survivable (corrects the framing above)
	250
	251	A (virtual) publishes, B (stock) mirrors it fully, then A's bytes are GC'd. A fresh
	252	client connected to both asks for the file.
	253
	254	\| \| outcome \| elapsed \| blocks served by B \|
	255	\|---\|---\|---\|---\|
	256	\| A lies (GC, no clear) \| ok \| 117–135 ms \| 93–109 of ~110 \|
	257	\| A cleared the range \| ok \| 104–133 ms \| 92–109 of ~110 \|
	258
	259	Four runs each; the distributions overlap, so the elapsed difference is noise, not
	260	signal. Both recover, and the second seeder serves essentially all of it.
	261
	262	So an un-cleared GC is not the correctness hazard the single-seeder probe implied.
	263	Hypercore's replicator routes around a peer that fails to deliver, and a second source
	264	covers it transparently. With no second source, cleared or not, the client times out
	265	identically — the bytes are gone either way.
	266
	267	Clearing after GC is therefore hygiene, not a requirement: it keeps advertisement
	268	honest and avoids wasted requests, and can be done opportunistically rather than
	269	atomically with the byte drop.
249	270
250	271	## Upstream bug found
251	272

test/canary.mjs +117 -0

…	…	@@ -0,0 +1,117 @@
	1	// Canary — the seam is not a supported contract. This asserts every assumption
	2	// the adapter makes about hypercore-storage and hypercore, so a version bump
	3	// fails here loudly instead of somewhere subtle later.
	4	// node test/canary.mjs <workDir>
	5	import { createRequire } from 'module'
	6	const require = createRequire(import.meta.url)
	7
	8	const crypto = require('hypercore-crypto')
	9	const Corestore = require('corestore')
	10	const Hyperdrive = require('hyperdrive')
	11	const CorestoreStorage = require('hypercore-storage')
	12	const { ContentStore, build } = require('../lib/storage.cjs')
	13
	14	const ROOT = process.argv[2] \|\| './work/canary'
	15	const failures = []
	16	const advisories = []
	17	const checks = []
	18
	19	// required: the adapter cannot work if this is false.
	20	// advisory: only matters for work not yet done (encryption re-derivation).
	21	function check (name, ok, detail, severity = 'required') {
	22	checks.push({ name, ok: !!ok, severity, detail })
	23	if (ok) return
	24	const line = name + (detail ? ' — ' + detail : '')
	25	if (severity === 'required') failures.push(line)
	26	else advisories.push(line)
	27	}
	28
	29	const versions = {}
	30	for (const m of ['hypercore', 'hypercore-storage', 'corestore', 'hyperdrive', 'hyperblobs']) {
	31	try {
	32	versions[m] = require(m + '/package.json').version
	33	} catch {
	34	versions[m] = 'missing'
	35	}
	36	}
	37
	38	// 1. the two factories Core calls to obtain per-core storage
	39	for (const m of ['resumeCore', 'createCore']) {
	40	check('CorestoreStorage.' + m, typeof CorestoreStorage.prototype[m] === 'function')
	41	}
	42
	43	// 2. duck-type used by Hypercore.defaultStorage to accept a storage instance
	44	check(
	45	'isCoreStorage duck-type (setDefaultDiscoveryKey)',
	46	typeof CorestoreStorage.prototype.setDefaultDiscoveryKey === 'function'
	47	)
	48
	49	// 3. drive a real publish through the adapter and observe the seam
	50	const payload = crypto.randomBytes(512 * 1024)
	51	const content = new ContentStore()
	52	// deliberately register nothing: every block should fall through as literal,
	53	// which still exercises read()/write()/getBlock/putBlock
	54	const Base = build(content)
	55
	56	const seen = { resumeCore: 0, createCore: 0, read: 0, write: 0 }
	57	class Observed extends Base {
	58	async resumeCore (dk) {
	59	seen.resumeCore++
	60	return super.resumeCore(dk)
	61	}
	62
	63	async createCore (data) {
	64	seen.createCore++
	65	return super.createCore(data)
	66	}
	67	}
	68
	69	const store = new Corestore(new Observed(ROOT), { primaryKey: Buffer.alloc(32, 5), unsafe: true })
	70	const drive = new Hyperdrive(store)
	71	await drive.ready()
	72	await drive.getBlobs()
	73
	74	const coreStorage = drive.blobs.core.state.storage
	75	for (const m of ['read', 'write', 'snapshot', 'atomize', 'createBlockStream']) {
	76	check('HypercoreStorage.' + m, typeof coreStorage[m] === 'function')
	77	}
	78
	79	const rx = coreStorage.read()
	80	check('CoreRX.getBlock', typeof rx.getBlock === 'function')
	81	check(
	82	'CoreRX carries fork (only needed if encryption re-derivation is built)',
	83	'fork' in rx.__unwrapped,
	84	null,
	85	'advisory'
	86	)
	87	rx.destroy()
	88
	89	const tx = coreStorage.write()
	90	check('CoreTX.putBlock', typeof tx.putBlock === 'function')
	91	check('CoreTX.deleteBlockRange', typeof tx.deleteBlockRange === 'function')
	92
	93	await drive.put('/probe.bin', payload)
	94	const got = await drive.get('/probe.bin')
	95	check('readback through the adapter', got !== null && got.equals(payload))
	96
	97	check('Core called resumeCore or createCore', seen.resumeCore + seen.createCore > 0,
	98	JSON.stringify(seen))
	99
	100	await drive.close()
	101	await store.close()
	102
	103	console.log(
	104	JSON.stringify(
	105	{
	106	versions,
	107	checks,
	108	failed: failures,
	109	advisories,
	110	verdict: failures.length === 0 ? 'PASS' : 'FAIL'
	111	},
	112	null,
	113	2
	114	)
	115	)
	116
	117	if (failures.length) process.exit(1)

test/two-seeder-gc.mjs +141 -0

…	…	@@ -0,0 +1,141 @@
	1	// Two-seeder GC routing — the claim left undemonstrated by test/gc-bitfield.mjs.
	2	//
	3	// A (virtual) publishes. B (stock) mirrors it fully. Then A's bytes are GC'd from
	4	// the content store. A client connected to BOTH asks for the file.
	5	// case "lie": A advertises blocks it cannot produce -> does the client recover from B?
	6	// case "paired": A also clears the range, so it stops advertising -> B serves cleanly.
	7	//
	8	// node test/two-seeder-gc.mjs <workDir>
	9	import { createRequire } from 'module'
	10	const require = createRequire(import.meta.url)
	11
	12	const fs = require('fs')
	13	const crypto = require('hypercore-crypto')
	14	const Corestore = require('corestore')
	15	const Hyperdrive = require('hyperdrive')
	16	const { ContentStore, build } = require('../lib/storage.cjs')
	17	const { chunk } = await import('../../hyper-overlay/lib/chunker.js')
	18
	19	const ROOT = process.argv[2]
	20	const NAME = '/app.bin'
	21	const srcDir = ROOT + '/src'
	22	fs.mkdirSync(srcDir, { recursive: true })
	23
	24	const payload = crypto.randomBytes(8 * 1024 * 1024)
	25	const file = srcDir + '/payload.bin'
	26	fs.writeFileSync(file, payload)
	27	const chunks = chunk(payload)
	28
	29	function write (ws, buf) {
	30	return new Promise((resolve) => {
	31	if (ws.write(buf)) return resolve()
	32	ws.once('drain', resolve)
	33	})
	34	}
	35
	36	function pipe (s1, s2) {
	37	s1.pipe(s2).pipe(s1)
	38	return () => {
	39	s1.destroy()
	40	s2.destroy()
	41	}
	42	}
	43
	44	async function scenario (tag, { clearAfterGc }) {
	45	const base = `${ROOT}/${tag}`
	46	const content = new ContentStore()
	47	for (const c of chunks) content.add(c.hash, file, c.offset, c.length)
	48	const Storage = build(content)
	49
	50	// A — virtual seeder
	51	const aStore = new Corestore(new Storage(base + '/a'), { primaryKey: Buffer.alloc(32, 7), unsafe: true })
	52	const aDrive = new Hyperdrive(aStore)
	53	await aDrive.ready()
	54	const ws = aDrive.createWriteStream(NAME, { dedup: true })
	55	for (const c of chunks) await write(ws, c.data)
	56	await new Promise((resolve, reject) => {
	57	ws.once('error', reject)
	58	ws.once('close', resolve)
	59	ws.end()
	60	})
	61	await aDrive.getBlobs()
	62	const driveKey = aDrive.key
	63
	64	// B — stock mirror, pulls everything from A
	65	const bStore = new Corestore(base + '/b', { primaryKey: Buffer.alloc(32, 8), unsafe: true })
	66	const bDrive = new Hyperdrive(bStore, driveKey)
	67	await bDrive.ready()
	68	const unpipeAB = pipe(aStore.replicate(true), bStore.replicate(false))
	69	await bDrive.core.update({ wait: true })
	70	const mirrored = await bDrive.get(NAME)
	71	const bHasAll = mirrored !== null && mirrored.equals(payload)
	72	await bDrive.getBlobs()
	73	unpipeAB()
	74
	75	// count who actually serves
	76	let servedByA = 0
	77	let servedByB = 0
	78	aDrive.blobs.core.on('upload', () => servedByA++)
	79	bDrive.blobs.core.on('upload', () => servedByB++)
	80
	81	// the overlay GCs A's bytes
	82	const entry = await aDrive.entry(NAME)
	83	const map = await aDrive.blobs.getBlockMap(entry.value.blob)
	84	const blocks = map.blocks.map((b) => b.index)
	85	for (const c of chunks) content.locations.delete(c.hash)
	86	if (clearAfterGc) await aDrive.blobs.core.clear(blocks[0], blocks[blocks.length - 1] + 1)
	87	const aAdvertises = await aDrive.blobs.core.has(blocks[0])
	88
	89	// C — fresh client, connected to BOTH
	90	const cStore = new Corestore(base + '/c', { primaryKey: Buffer.alloc(32, 9), unsafe: true })
	91	const cDrive = new Hyperdrive(cStore, driveKey)
	92	await cDrive.ready()
	93	const unpipeCA = pipe(aStore.replicate(true), cStore.replicate(false))
	94	const unpipeCB = pipe(bStore.replicate(true), cStore.replicate(false))
	95	await cDrive.core.update({ wait: true })
	96
	97	const t0 = Date.now()
	98	let outcome
	99	try {
	100	const got = await cDrive.get(NAME, { timeout: 15000 })
	101	outcome = got === null ? 'null' : got.equals(payload) ? 'ok' : 'CORRUPT'
	102	} catch (err) {
	103	outcome = 'throw:' + (err.code \|\| err.message)
	104	}
	105	const elapsedMs = Date.now() - t0
	106
	107	unpipeCA()
	108	unpipeCB()
	109	for (const d of [cDrive, bDrive, aDrive]) await d.close().catch(() => {})
	110	for (const s of [cStore, bStore, aStore]) await s.close().catch(() => {})
	111	content.close()
	112
	113	return { tag, bHasAll, aAdvertisesAfterGc: aAdvertises, outcome, elapsedMs, servedByA, servedByB }
	114	}
	115
	116	const lie = await scenario('lie', { clearAfterGc: false })
	117	console.log(JSON.stringify(lie))
	118	const paired = await scenario('paired', { clearAfterGc: true })
	119	console.log(JSON.stringify(paired))
	120
	121	// NB: do NOT read the elapsed difference as signal — across repeated runs the two
	122	// cases overlap (lie 117-135ms, paired 104-133ms). What matters is that BOTH
	123	// recover, and that the second seeder serves essentially all of it.
	124	console.log(
	125	JSON.stringify(
	126	{
	127	bothRecovered: lie.outcome === 'ok' && paired.outcome === 'ok',
	128	servedBySecondSeeder: {
	129	lie: lie.servedByB + '/' + (lie.servedByA + lie.servedByB),
	130	paired: paired.servedByB + '/' + (paired.servedByA + paired.servedByB)
	131	},
	132	ms: { lie: lie.elapsedMs, paired: paired.elapsedMs },
	133	verdict:
	134	lie.outcome === 'ok' && paired.outcome === 'ok'
	135	? 'the bitfield lie is survivable when another source exists'
	136	: 'FAIL'
	137	},
	138	null,
	139	2
	140	)
	141	)

hyper-vdrive commits / 2e9fe5b

Changed files (4) +292 -10