577b061 · hyper-vdrive

Changed files (5) +257 -6

README.md +24 -6

…	…	@@ -56,18 +56,36 @@ already the leaf hashes, and a chunk maps directly onto a block.
56	56
57	57	## Usage
58	58
	59	Against a real hyper-overlay index:
	60
59	61	```js
60		const { ContentStore, build } = require('hyper-vdrive')
	62	const { build } = require('hyper-vdrive')
	63	const { OverlayContentStore } = require('hyper-vdrive/overlay')
61	64
62		const content = new ContentStore()
63		// tell it where bytes live: contentHash -> (file, offset, length)
64		for (const c of chunk(buf)) content.add(c.hash, file, c.offset, c.length)
	65	const content = new OverlayContentStore(fileIndex, {
	66	roots: [repoRoot], // working tree — bytes at <root>/<path>
	67	syncBaseDirs: [spoolDir] // content spool — bytes at <dir>/<oid>, for mirrors
	68	})
	69	await content.indexAll() // or indexPath(p) / indexContent(oid) per artifact
65	70
66		const Storage = build(content)
67		const store = new Corestore(new Storage(dir))
	71	const store = new Corestore(new build(content)(dir))
68	72	const drive = new Hyperdrive(store)
69	73	```
70	74
	75	Or standalone, if you are managing locations yourself:
	76
	77	```js
	78	const { ContentStore, build } = require('hyper-vdrive')
	79	const content = new ContentStore()
	80	for (const c of chunk(buf)) content.add(c.hash, file, c.offset, c.length)
	81	```
	82
	83	Note on the reverse index. `FileIndex` maps path → chunks and oid → chunks; it
	84	has no chunk hash → location index. `OverlayContentStore` builds that in memory by
	85	reading the chunk maps of the paths you register, which is O(total chunks) — fine for a
	86	publish or a seeder boot, not something to redo per request. A persistent reverse index
	87	in the overlay would remove it.
	88
71	89	Write files chunk-by-chunk so block boundaries are your chunk boundaries:
72	90
73	91	```js

RESULTS.md +22 -0

…	…	@@ -268,6 +268,28 @@ Clearing after GC is therefore hygiene, not a requirement: it keeps advertis
268	268	honest and avoids wasted requests, and can be done opportunistically rather than
269	269	atomically with the byte drop.
270	270
	271	## Bound to hyper-overlay's FileIndex
	272
	273	A vdrive published with bytes resolved through a real `FileIndex`, exercising both
	274	byte sources — a working-tree file and a content-addressed spool blob (what a mirror
	275	holds, having no working tree).
	276
	277	```
	278	indexed 2 files -> 1 path + 1 spool blob, 114 chunks
	279	blocks 114 redirected, 5 literal
	280	readback working tree ok, spool ok
	281	hash failures 0
	282	input 8.39 MB
	283	drive store 0.14 MB
	284	```
	285
	286	`indexAll()` walks `listFiles()`, tries `getChunkMap(path)` against the working-tree
	287	roots first, then falls back to `getChunkMapByHash(contentHash)` against the spool dirs.
	288
	289	One thing to know when populating an index by hand: `chunk()` returns `data` subarrays
	290	alongside the locations, and putting those into the bee blows past Hypercore's 15 MiB
	291	block cap. Store `{hash, offset, length}` only.
	292
271	293	## Upstream bug found
272	294
273	295	A 0-byte file written with `dedup: true` cannot be read back: hyperblobs writes an

lib/overlay-content-store.cjs +89 -0

…	…	@@ -0,0 +1,89 @@
	1	// Bind the adapter to hyper-overlay's FileIndex.
	2	//
	3	// The adapter needs contentHash -> (file, offset, length). FileIndex stores the
	4	// other direction: path -> chunks[{hash, offset, length}] (`chunkmap:<path>`) and
	5	// contentHash -> the same list (`chunkmap-oid:<hash>`). There is no reverse index,
	6	// so we build one by reading the chunk maps of the paths we intend to publish.
	7	//
	8	// Two byte sources, matching how the overlay actually stores things:
	9	// * working tree — bytes at <root>/<path>, chunk map from getChunkMap(path)
	10	// * content spool — bytes at <syncBaseDir>/<oid> (a mirror has no working tree),
	11	// chunk map from getChunkMapByHash(oid)
	12	//
	13	// `index` is duck-typed: anything with getChunkMap / getChunkMapByHash works, so
	14	// this file does not import hyper-overlay (ESM) into a CJS module.
	15
	16	const fs = require('fs')
	17	const path = require('path')
	18	const { ContentStore } = require('./storage.cjs')
	19
	20	class OverlayContentStore extends ContentStore {
	21	constructor (index, { roots = [], syncBaseDirs = [] } = {}) {
	22	super()
	23	this.index = index
	24	this.roots = Array.isArray(roots) ? roots : [roots]
	25	this.syncBaseDirs = Array.isArray(syncBaseDirs) ? syncBaseDirs : [syncBaseDirs]
	26	this.indexed = new Set()
	27	}
	28
	29	_locate (dirs, rel) {
	30	for (const dir of dirs) {
	31	const full = path.join(dir, rel)
	32	try {
	33	if (fs.statSync(full).isFile()) return full
	34	} catch {}
	35	}
	36	return null
	37	}
	38
	39	// Register one working-tree path. Returns how many chunks it contributed.
	40	async indexPath (rel) {
	41	if (this.indexed.has('p:' + rel)) return 0
	42	const chunks = await this.index.getChunkMap(rel)
	43	if (!chunks \|\| chunks.length === 0) return 0
	44	const file = this._locate(this.roots, rel)
	45	if (file === null) return 0
	46	for (const c of chunks) this.add(c.hash, file, c.offset, c.length)
	47	this.indexed.add('p:' + rel)
	48	return chunks.length
	49	}
	50
	51	// Register one content-addressed blob from the spool (mirror nodes).
	52	async indexContent (oid) {
	53	if (this.indexed.has('o:' + oid)) return 0
	54	const chunks = await this.index.getChunkMapByHash(oid)
	55	if (!chunks \|\| chunks.length === 0) return 0
	56	const file = this._locate(this.syncBaseDirs, oid)
	57	if (file === null) return 0
	58	for (const c of chunks) this.add(c.hash, file, c.offset, c.length)
	59	this.indexed.add('o:' + oid)
	60	return chunks.length
	61	}
	62
	63	// Register everything the index knows about, working tree first then spool.
	64	// O(total chunks) — fine for a publish, not something to do on every boot.
	65	async indexAll (dir = '') {
	66	const files = await this.index.listFiles(dir)
	67	let chunks = 0
	68	let paths = 0
	69	let blobs = 0
	70	for (const f of files) {
	71	const n = await this.indexPath(f.path)
	72	if (n > 0) {
	73	chunks += n
	74	paths++
	75	continue
	76	}
	77	if (f.contentHash) {
	78	const m = await this.indexContent(f.contentHash)
	79	if (m > 0) {
	80	chunks += m
	81	blobs++
	82	}
	83	}
	84	}
	85	return { files: files.length, paths, blobs, chunks }
	86	}
	87	}
	88
	89	module.exports = { OverlayContentStore }

package.json +1 -0

…	…	@@ -5,6 +5,7 @@
5	5	"main": "lib/storage.cjs",
6	6	"exports": {
7	7	".": "./lib/storage.cjs",
	8	"./overlay": "./lib/overlay-content-store.cjs",
8	9	"./passthrough": "./lib/passthrough.cjs"
9	10	},
10	11	"scripts": {

test/overlay-binding.mjs +121 -0

…	…	@@ -0,0 +1,121 @@
	1	// Publish a vdrive whose bytes are resolved through a REAL hyper-overlay FileIndex,
	2	// from both byte sources: a working tree and a content-addressed spool.
	3	// node test/overlay-binding.mjs <workDir>
	4	import { createRequire } from 'module'
	5	const require = createRequire(import.meta.url)
	6
	7	const fs = require('fs')
	8	const path = require('path')
	9	const { execFileSync } = require('child_process')
	10	const crypto = require('hypercore-crypto')
	11	const Corestore = require('corestore')
	12	const Hyperdrive = require('hyperdrive')
	13	const CorestoreStorage = require('hypercore-storage')
	14	const { build, stats } = require('../lib/storage.cjs')
	15	const { OverlayContentStore } = require('../lib/overlay-content-store.cjs')
	16
	17	const { FileIndex } = await import('../../hyper-overlay/lib/file-index.js')
	18	const { chunk } = await import('../../hyper-overlay/lib/chunker.js')
	19
	20	const ROOT = process.argv[2]
	21	const du = (p) => Number(execFileSync('du', ['-sb', p]).toString().split('\t')[0])
	22
	23	const treeDir = ROOT + '/worktree'
	24	const spoolDir = ROOT + '/spool'
	25	fs.mkdirSync(treeDir, { recursive: true })
	26	fs.mkdirSync(spoolDir, { recursive: true })
	27
	28	// a working-tree file and a spool blob (as a mirror node would hold it)
	29	const treeBytes = crypto.randomBytes(5 * 1024 * 1024)
	30	const spoolBytes = crypto.randomBytes(3 * 1024 * 1024)
	31	fs.writeFileSync(path.join(treeDir, 'app.bin'), treeBytes)
	32	const spoolOid = crypto.data(spoolBytes).toString('hex')
	33	fs.writeFileSync(path.join(spoolDir, spoolOid), spoolBytes)
	34
	35	// ── populate a real FileIndex the way the overlay does ──────────────────────
	36	const idxStore = new Corestore(ROOT + '/overlay-store')
	37	const index = new FileIndex(idxStore)
	38	await index.ready()
	39
	40	// the index stores locations only — chunk() also hands back `data` subarrays,
	41	// and putting those in the bee blows past the 15 MiB block cap
	42	const meta = (chunks) => chunks.map((c) => ({ hash: c.hash, offset: c.offset, length: c.length }))
	43
	44	const treeChunks = chunk(treeBytes)
	45	await index.putFile('app.bin', {
	46	contentHash: crypto.data(treeBytes).toString('hex'),
	47	size: treeBytes.length,
	48	mtime: Date.now()
	49	})
	50	await index.putChunkMap('app.bin', meta(treeChunks))
	51
	52	const spoolChunks = chunk(spoolBytes)
	53	await index.putFile('blob.bin', {
	54	contentHash: spoolOid,
	55	size: spoolBytes.length,
	56	mtime: Date.now()
	57	})
	58	await index.putChunkMapByHash(spoolOid, meta(spoolChunks))
	59
	60	// ── resolve through the index, publish a vdrive ─────────────────────────────
	61	const content = new OverlayContentStore(index, { roots: [treeDir], syncBaseDirs: [spoolDir] })
	62	const indexed = await content.indexAll()
	63
	64	const Storage = build(content)
	65	const driveDir = ROOT + '/virtual'
	66	const store = new Corestore(new Storage(driveDir), { primaryKey: Buffer.alloc(32, 7), unsafe: true })
	67	const drive = new Hyperdrive(store)
	68	await drive.ready()
	69
	70	function write (ws, buf) {
	71	return new Promise((resolve) => {
	72	if (ws.write(buf)) return resolve()
	73	ws.once('drain', resolve)
	74	})
	75	}
	76
	77	for (const [name, chunks] of [['/app.bin', treeChunks], ['/blob.bin', spoolChunks]]) {
	78	const ws = drive.createWriteStream(name, { dedup: true })
	79	for (const c of chunks) await write(ws, c.data)
	80	await new Promise((resolve, reject) => {
	81	ws.once('error', reject)
	82	ws.once('close', resolve)
	83	ws.end()
	84	})
	85	}
	86
	87	await drive.getBlobs()
	88	const readTree = await drive.get('/app.bin')
	89	const readSpool = await drive.get('/blob.bin')
	90	const driveKey = drive.key.toString('hex')
	91	const blobsLength = drive.blobs.core.length
	92	await drive.close()
	93	await store.close()
	94	await index.close()
	95	await idxStore.close()
	96
	97	const s = new CorestoreStorage(driveDir)
	98	await s.ready()
	99	await s.compact()
	100	await s.close()
	101
	102	content.close()
	103
	104	console.log(
	105	JSON.stringify(
	106	{
	107	indexed,
	108	driveKey,
	109	blobsLength,
	110	workingTreeReadbackOk: readTree !== null && readTree.equals(treeBytes),
	111	spoolReadbackOk: readSpool !== null && readSpool.equals(spoolBytes),
	112	redirected: stats.putRedirected,
	113	literal: stats.putStored,
	114	hashFailures: stats.hashFailures,
	115	inputBytes: treeBytes.length + spoolBytes.length,
	116	driveStoreBytes: du(driveDir)
	117	},
	118	null,
	119	2
	120	)
	121	)

hyper-vdrive commits / 577b061

Changed files (5) +257 -6