fda1abe · hyper-vdrive

Changed files (3) +152 -4

README.md +7 -4

…	…	@@ -105,13 +105,16 @@ Known gaps, in the order they'd bite:
105	105	from the content store's refcounts, never from a version's block range.
106	106	2. Block streams (`createBlockStream`, used by audit/copy-prologue) yield raw tagged
107	107	records; they need decoding like `getBlock` does.
108		3. Encryption sits above storage and prepends 8 bytes of padding, so block bytes stop
109		equalling chunk bytes. Run unencrypted, or teach the adapter to re-derive.
	108	3. Encryption defeats virtualisation — measured: 0 of 105 blocks redirect, the store
	109	goes from 0.14 MB to 8.53 MB. Correctness is unaffected (fall-through-by-hash
	110	degrades to a normal drive). Recoverable in principle: the cipher is deterministic
	111	per (index, fork, key), so an adapter holding the key could decrypt on write and
	112	re-encrypt on read, with dedup surviving because hyperblobs hashes plaintext. Until
	113	then, publish public apps unencrypted.
110	114	4. Unsupported seam. `hypercore-storage` has no public pluggable-storage contract
111	115	since `random-access-storage` was dropped. Pin versions and run the tests as a canary.
112	116	5. clear-then-truncate is unsupported upstream — `clear()` prunes merkle tree nodes,
113	117	so a later `truncate()` fails with `Expected tree node N from storage, got (nil)`.
114	118	Same on a stock drive. And `clear()` reclaims no disk promptly: on a virtual drive
115	119	that does not matter, because the bytes are the overlay's to free.
116		6. Untested: power loss (SIGKILL is covered and passes; fsync durability is not),
117		and encryption.
	120	6. Untested: power loss — SIGKILL is covered and passes, fsync durability is not.

RESULTS.md +33 -0

…	…	@@ -198,6 +198,39 @@ Two things worth noting:
198	198	Caveat: SIGKILL ends the process but the OS still flushes its page cache. This is not a
199	199	power-loss test, and nothing here should be read as an fsync-durability claim.
200	200
	201	## Encryption
	202
	203	Same 8 MB payload, same adapter, with and without `encryptionKey`. Sizes compacted.
	204
	205	\| \| plaintext \| encrypted \|
	206	\|---\|---\|---\|
	207	\| blocks redirected \| 102 \| 0 \|
	208	\| blocks stored literally \| 3 \| 105 \|
	209	\| store \| 0.14 MB \| 8.53 MB \|
	210	\| readback \| ok \| ok \|
	211	\| drive key \| identical \| identical \|
	212
	213	Virtualisation is lost completely under encryption, and correctness is unaffected.
	214	Hypercore encrypts in `_preappend`, above the storage layer, so the bytes reaching
	215	`putBlock` are ciphertext with 8 bytes of prepended padding. Their hashes are not chunk
	216	hashes, so every block falls through to real storage — the fall-through-by-hash design
	217	degrades safely to a normal drive rather than corrupting anything.
	218
	219	It is recoverable. The cipher is a keyed stream XOR nonced by block index and
	220	blinded fork (`hypercore/lib/default-encryption.js:49-69`), and the probe confirms it:
	221	same plaintext at the same index gives identical ciphertext, a different index gives
	222	different ciphertext. So an adapter holding the encryption key could decrypt in
	223	`putBlock` to recover the plaintext hash, and re-encrypt in `getBlock` from the stored
	224	plaintext chunk. Dedup survives that, because hyperblobs hashes what the caller
	225	writes — plaintext — and only storage ever sees ciphertext.
	226
	227	Two things to get right if that is built: `CoreRX` already carries `fork`, but `CoreTX`
	228	does not obviously, and the fork participates in the nonce. And it hands the storage
	229	layer a key it currently never sees, which is a real trust-boundary change.
	230
	231	Until then: publish public apps unencrypted — which is what `pear://` distribution
	232	is anyway — and treat encrypted private repos as a separate track.
	233
201	234	## GC / bitfield
202	235
203	236	```

test/encryption.mjs +112 -0

…	…	@@ -0,0 +1,112 @@
	1	// Encryption — hypercore encrypts ABOVE storage, so the bytes reaching putBlock
	2	// are ciphertext with 8 bytes of padding, not our chunk bytes. Expected outcome:
	3	// still correct, but nothing virtualises. This measures how completely.
	4	// node test/encryption.mjs <workDir>
	5	import { createRequire } from 'module'
	6	const require = createRequire(import.meta.url)
	7
	8	const fs = require('fs')
	9	const { execFileSync } = require('child_process')
	10	const crypto = require('hypercore-crypto')
	11	const Corestore = require('corestore')
	12	const Hyperdrive = require('hyperdrive')
	13	const CorestoreStorage = require('hypercore-storage')
	14	const DefaultEncryption = require('hypercore/lib/default-encryption')
	15	const { ContentStore, build, stats } = require('../lib/storage.cjs')
	16	const { chunk } = await import('../../hyper-overlay/lib/chunker.js')
	17
	18	const ROOT = process.argv[2]
	19	const PRIMARY = Buffer.alloc(32, 7)
	20	const ENCRYPTION_KEY = Buffer.alloc(32, 42)
	21	const du = (p) => Number(execFileSync('du', ['-sb', p]).toString().split('\t')[0])
	22
	23	async function compactedSize (dir) {
	24	const s = new CorestoreStorage(dir)
	25	await s.ready()
	26	await s.compact()
	27	await s.close()
	28	return du(dir)
	29	}
	30
	31	const srcDir = ROOT + '/src'
	32	fs.mkdirSync(srcDir, { recursive: true })
	33	const payload = crypto.randomBytes(8 * 1024 * 1024)
	34	const file = srcDir + '/payload.bin'
	35	fs.writeFileSync(file, payload)
	36
	37	const chunks = chunk(payload)
	38	const content = new ContentStore()
	39	for (const c of chunks) content.add(c.hash, file, c.offset, c.length)
	40
	41	function write (ws, buf) {
	42	return new Promise((resolve) => {
	43	if (ws.write(buf)) return resolve()
	44	ws.once('drain', resolve)
	45	})
	46	}
	47
	48	async function run (dir, { encrypted }) {
	49	const before = { ...stats }
	50	const Storage = build(content)
	51	const store = new Corestore(new Storage(dir), { primaryKey: PRIMARY, unsafe: true })
	52	const drive = new Hyperdrive(store, encrypted ? { encryptionKey: ENCRYPTION_KEY } : {})
	53	await drive.ready()
	54
	55	const ws = drive.createWriteStream('/payload.bin', { dedup: true })
	56	for (const c of chunks) await write(ws, c.data)
	57	await new Promise((resolve, reject) => {
	58	ws.once('error', reject)
	59	ws.once('close', resolve)
	60	ws.end()
	61	})
	62
	63	await drive.getBlobs()
	64	const got = await drive.get('/payload.bin')
	65	const out = {
	66	driveKey: drive.key.toString('hex'),
	67	blobsLength: drive.blobs.core.length,
	68	readbackOk: got !== null && got.equals(payload),
	69	redirected: stats.putRedirected - before.putRedirected,
	70	literal: stats.putStored - before.putStored
	71	}
	72	await drive.close()
	73	await store.close()
	74	out.storeBytes = await compactedSize(dir)
	75	return out
	76	}
	77
	78	const plain = await run(ROOT + '/plain', { encrypted: false })
	79	const enc = await run(ROOT + '/encrypted', { encrypted: true })
	80
	81	// Is the cipher deterministic? If so, an adapter given the key could re-derive
	82	// ciphertext from plaintext on read, and virtualisation would work encrypted too.
	83	const keys = DefaultEncryption.deriveKeys(ENCRYPTION_KEY, Buffer.alloc(32, 1), {})
	84	function encryptOnce (plaintext, index, fork) {
	85	const block = Buffer.concat([Buffer.alloc(DefaultEncryption.PADDING), plaintext])
	86	DefaultEncryption.encrypt(index, block, fork, keys.block, keys.blinding)
	87	return block
	88	}
	89	const sample = payload.subarray(0, 4096)
	90	const a = encryptOnce(sample, 7, 0)
	91	const b = encryptOnce(sample, 7, 0)
	92	const c = encryptOnce(sample, 8, 0)
	93
	94	content.close()
	95
	96	console.log(
	97	JSON.stringify(
	98	{
	99	plain,
	100	encrypted: enc,
	101	sameDriveKey: plain.driveKey === enc.driveKey,
	102	virtualisationLostWhenEncrypted: enc.redirected === 0,
	103	cipher: {
	104	deterministicSameIndex: a.equals(b),
	105	differsAcrossIndex: !a.equals(c),
	106	paddingBytes: DefaultEncryption.PADDING
	107	}
	108	},
	109	null,
	110	2
	111	)
	112	)

hyper-vdrive commits / fda1abe

Changed files (3) +152 -4