OverviewFilesCommitsIssuesDocsReleasesPeers

hyper-vdrive commits / ca2d1cf

Load prior locations before republishing, and route the CLI through publishTree
Republishing into an existing drive hung. `dedup: true` seeds its hash table by reading back the PREVIOUS entry's blocks, and those reads go through the content store — which publishTree rebuilt from only the files being published now, so every prior chunk resolved to null and the write stalled with an unsettled promise. Loading the saved locations index first fixes it, which is the second job that index exists to do. The CLI now goes through publishTree too (it had its own inline copy predating it) and takes --root to anchor the index across sibling release directories. Proven on three real 49MB IFSRenderer AppImage builds, distinct SHAs, published into one drive: 614 chunks, then 14, then 12. 148MB of input in a 519KB store, one stable pear:// link. A real `pear install` on stortron pulled it over the public DHT byte-identical, mode 755, with the desktop entry and icons installed. Also worth recording: AppImages dedup at 97.7% despite being compressed. Stream compression (gzip over a tarball) destroys dedup; BLOCK compression (squashfs) does not, because a change only disturbs the blocks it touches. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
ca2d1cff76dccff6905e35c998e2f45ab5665547
by plankatron <fleeky@gmail.com> · 2026-08-24T22:48:08.000Z

Changed files (2) +21 -29

bin/vdrive.mjs +12 -28
……@@ -18,6 +18,7 @@ const Corestore = require('corestore')
1818const Hyperdrive = require('hyperdrive')
1919const CorestoreStorage = require('hypercore-storage')
2020const { ContentStore, build, stats } = require('../lib/storage.cjs')
21const { publishTree } = require('../lib/publish.cjs')
2122const { chunk } = await import('../../hyper-overlay/lib/chunker.js')
2223
2324const USAGE = `vdrive — a Hyperdrive that stores no blob bytes
……@@ -109,37 +110,20 @@ async function publish (args) {
109110 }
110111 if (names.length === 0) return fail('nothing to publish under ' + src)
111112
112 const { content, files } = indexTree(src, names)
113 const { store, drive } = openDrive(storeDir, content)
114 await drive.ready()
113 // --root anchors the locations index. Successive releases usually live in
114 // sibling directories under one root; pass it so the index written now still
115 // resolves the bytes of releases published earlier.
116 const root = args.flags.root ? path.resolve(args.flags.root) : path.resolve(src)
115117
116 for (const f of files) {
117 // 0-byte + dedup is an upstream crash on read (hyperblobs seekBlockMap)
118 const ws = drive.createWriteStream(f.name, f.size === 0 ? {} : { dedup: true })
119 for (const c of f.chunks) await write(ws, c.data)
120 await new Promise((resolve, reject) => {
121 ws.once('error', reject)
122 ws.once('close', resolve)
123 ws.end()
124 })
125 }
118 const result = await publishTree({
119 storeDir,
120 root,
121 chunk,
122 files: names.map((name) => ({ name, file: path.join(src, name) }))
123 })
126124
127 await drive.getBlobs()
128 const result = {
129 link: 'pear://' + idEnc.normalize(drive.key),
130 key: drive.key.toString('hex'),
131 files: files.length,
132 bytes: files.reduce((a, f) => a + f.size, 0),
133 blocks: drive.blobs.core.length,
134 pointerBlocks: stats.putRedirected,
135 literalBlocks: stats.putStored
136 }
137 await drive.close()
138 await store.close()
139125 await compact(storeDir)
140 result.driveStoreBytes = du(storeDir)
141 content.close()
142 return result
126 return { ...result, driveStoreBytes: du(storeDir) }
143127}
144128
145129async function info (args) {
lib/publish.cjs +9 -1
……@@ -15,7 +15,7 @@ const idEnc = require('hypercore-id-encoding')
1515const Corestore = require('corestore')
1616const Hyperdrive = require('hyperdrive')
1717const { ContentStore, build, stats } = require('./storage.cjs')
18const { saveLocations } = require('./locations.cjs')
18const { saveLocations, loadLocations } = require('./locations.cjs')
1919
2020function writeChunk (ws, buf) {
2121 return new Promise((resolve) => {
……@@ -40,6 +40,13 @@ async function publishTree ({ storeDir, files, chunk, root, corestoreOpts } = {}
4040 if (typeof chunk !== 'function') throw new Error('publishTree requires a chunk function')
4141
4242 const content = new ContentStore()
43
44 // Republishing into an existing drive needs the PREVIOUS version's bytes to be
45 // resolvable: `dedup: true` seeds its hash table by reading back the previous
46 // entry's blocks, and those reads come through this content store. Without the
47 // saved index those chunks resolve to null and the write stalls.
48 const priorLocations = root ? loadLocations(storeDir, content, root) : 0
49
4350 const prepared = []
4451 let bytes = 0
4552
……@@ -76,6 +83,7 @@ async function publishTree ({ storeDir, files, chunk, root, corestoreOpts } = {}
7683
7784 return {
7885 locations,
86 priorLocations,
7987 link: 'pear://' + idEnc.normalize(drive.key),
8088 key: drive.key.toString('hex'),
8189 blocks: drive.blobs.core.length,