OverviewFilesCommitsIssuesDocsReleasesPeers

hyper-overlay commits / 39aff78

port(0.3.0): zero-shim restructure — adopt Ok engine, keep our superset
lib/ IS the engine now (Ok ok/hyper-overlay-v3 names), no lib/v3, no shim files. DELETED v1 (library becomes pure content-addressed distribution): the v1 HyperOverlay path-facade (index.js), lib/protocol.js (alpha wire), v1 lib/messages.js, http.js, chunk-registry.js, fuse.js, pairing.js, watch-manager.js + their 12 tests. Deleted the 8 transitional shim files. PROMOTED lib/v3/*.js -> lib/ (chunker, chunk-scheduler, file-index, messages, overlay, protocol, sync-engine, transfer). KEPT OUR ARCHITECTURE ON TOP (critical adoption, not a swap): - our single-flight backpressure discipline in protocol.js _onChunkNeed - our getIndexMeta/putIndexMeta fast-boot in file-index.js - our tree.js/pointer.js/ignore.js/manifest.js (Ok dropped them; pgh, hyper-deploy, pear-git-v3 depend on them via subpaths) package.json: 0.3.0, engines.bare, exports = Ok subpaths + our superset (/tree /pointer /ignore /manifest) + transitional aliases (/protocol-v2 -> ./lib/protocol.js, /overlay-v2, /messages-v2) so 0.3.0 lands WITHOUT forcing simultaneous edits to the three live consumers. Kept #fs/#path/#os/#zlib imports map (tree/ignore/manifest need it); added sodium-universal dep; pruned v1-only deps to devDeps. index.js: Ok engine exports + our kept superset re-exports + V2 aliases. Suite after restructure: KEEP tests (chunker/file-index/tree/tree-peer/ pointer/manifest/sync-engine/ignore-partial) green; remaining reds are the 6 engine tests to adopt to Ok API (next commits) + git-filter/ignore watch-dep. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
39aff7851c0643dad1dcd633d2a0514fa9173ba5
by plankatron <fleeky@gmail.com> · 2026-07-11T17:23:18.000Z

Changed files (45) +4124 -8293

index.js +56 -746
……@@ -1,753 +1,63 @@
11/**
2 * HyperOverlay — Content-addressed union filesystem
2 * hyper-overlay — content-addressed P2P file distribution over Protomux.
33 *
4 * P2P file distribution with FastCDC chunking and zero duplication.
5 * Local files stay on disk. Chunks are content-addressed and deduped.
6 * Transfer via Protomux (P2P) or HTTP (fallback).
4 * The canonical bytes are REAL FILES ON DISK; the overlay stores only
5 * metadata + chunk maps (Hyperbee), so there is no blob-storage balloon.
6 * Files are chunked with FastCDC, fetched multi-source (torrent-style),
7 * hash-verified incrementally, and written straight to the requester's disk.
78 *
8 * Usage:
9 * const overlay = new HyperOverlay(corestore, { namespace: 'my-overlay' })
10 * await overlay.ready()
11 * await overlay.put('/file.bin', buffer)
12 * const data = await overlay.get('/file.bin')
9 * Main entry point: `HyperOverlay` (also the default export). The submodules
10 * are available as subpath exports (`hyper-overlay/chunker`, …) for consumers
11 * that only need a slice of the stack.
12 *
13 * Runs on Bare. The pure modules — chunker, messages, chunk-scheduler,
14 * file-index, sync-engine — also run on Node.
15 *
16 * 0.3.0 adopts the hardened engine from ok/hyper-overlay-v3 (journaled resume,
17 * wire paging, index GC, prepareForServe) while KEEPING our v2 architecture on
18 * top: our single-flight backpressure discipline in the serve loop, our
19 * getIndexMeta/putIndexMeta fast-boot, and the tree/pointer/ignore/manifest
20 * modules Ok's fork dropped (pgh / hyper-deploy / pear-git-v3 depend on them).
1321 */
1422
15import ReadyResource from 'ready-resource'
16import { Readable } from 'streamx'
17import { pipelinePromise } from 'streamx'
18import fs from '#fs'
19import path from '#path'
20import os from '#os'
21import crypto from 'hypercore-crypto'
22import { chunk, chunkStream, createStreamingHasher, selectTier, hashChunk } from './lib/chunker.js'
23import { serialize, parse, isPointer, INLINE_MAX } from './lib/pointer.js'
24import { encode as encodeManifest, decode as decodeManifest } from './lib/manifest.js'
25import { ChunkRegistry } from './lib/chunk-registry.js'
26import { OverlayProtocol } from './lib/protocol.js'
27import { serveOverlay } from './lib/http.js'
28
29// Files below this size bypass FastCDC and store as a single whole-file blob.
30// Sub-file dedup doesn't earn its keep until the file is large enough that
31// chunking can save real bytes on partial updates / resumes.
32const CHUNK_THRESHOLD = 1048576 // 1 MB
33
34export class HyperOverlay extends ReadyResource {
35 constructor (corestore, opts = {}) {
36 super()
37 this._corestore = corestore
38 this._namespace = opts.namespace || 'overlay'
39 this._registry = null
40 this._protocol = null
41 this._namespaceStore = corestore.namespace(this._namespace)
42 // Optional content-addressed spool for peer-fetched chunks. When set,
43 // fetched bytes are written here and registered as `local` pointers
44 // instead of base64-inflated inline `cache` sources. See
45 // _cacheFetchedChunk(). Null = legacy inline-cache behavior.
46 this._chunkSpoolDir = opts.chunkSpoolDir || null
47 this._spoolSeq = 0
48 }
49
50 async _open () {
51 this._registry = new ChunkRegistry(this._namespaceStore)
52 await this._registry.ready()
53 this._protocol = new OverlayProtocol(this._registry)
54 }
55
56 async _close () {
57 if (this._protocol) this._protocol.destroy()
58 if (this._registry) await this._registry.close()
59 }
60
61 // ── Properties ──────────────────────────────────────────────
62
63 get registry () { return this._registry }
64
65 // ── File Operations ─────────────────────────────────────────
66
67 /**
68 * Store a file — chunks it and registers in the overlay
69 * @param {string} path - file path
70 * @param {Buffer} data - file content
71 * @returns {{ oid: string, chunkCount: number, pointer: string }}
72 */
73 async put (path, data) {
74 const fileHash = hashChunk(data)
75
76 // Files below CHUNK_THRESHOLD bypass FastCDC and store as a single
77 // whole-file blob. Sub-file dedup doesn't help small files, and the
78 // chunker overhead is wasted when most files would be 1 chunk anyway.
79 const chunks = data.length < CHUNK_THRESHOLD
80 ? [{ hash: fileHash, offset: 0, length: data.length, data }]
81 : chunk(data)
82
83 // Register each chunk
84 for (const c of chunks) {
85 await this._registry.putChunk(c.hash, {
86 size: c.length,
87 type: 'cache',
88 data: c.data.toString('base64')
89 })
90 }
91
92 // Build manifest if needed
93 let manifestHash = null
94 if (chunks.length > INLINE_MAX) {
95 const manifest = encodeManifest(
96 chunks.map(c => ({ hash: c.hash, size: c.length })),
97 fileHash
98 )
99 manifestHash = hashChunk(manifest)
100 await this._registry.putManifest(manifestHash, manifest)
101 }
102
103 // Store namespace entry
104 const entry = {
105 contentHash: fileHash,
106 size: data.length,
107 chunkCount: chunks.length,
108 manifestHash,
109 chunks: chunks.length <= INLINE_MAX
110 ? chunks.map(c => ({ hash: c.hash, size: c.length }))
111 : null,
112 createdAt: Date.now()
113 }
114 await this._registry._bee.put(`file:${path}`, entry)
115
116 // Build pointer
117 const pointer = serialize({
118 oid: fileHash,
119 size: data.length,
120 chunkCount: chunks.length,
121 chunks: entry.chunks,
122 manifestHash
123 })
124
125 return { oid: fileHash, chunkCount: chunks.length, pointer }
126 }
127
128 /**
129 * @deprecated Use a caller-managed disk spool + `putFile` instead.
130 *
131 * putStream stores chunks as `type:'cache'` (base64 inside the bee),
132 * which inflates corestore growth ~135% and pushes RSS to ~1.5× input
133 * for large uploads. The architecturally correct path:
134 *
135 * 1. Spool the Readable to a stable, content-addressed path
136 * (e.g. `<storage>/upload-spool/<oid>`)
137 * 2. Call `overlay.putFile(logicalPath, spoolPath)` — chunks store
138 * as `type:'local'` pointers, RAM bounded, corestore stays small
139 *
140 * See `lib/http.js` upload route for a worked example.
141 *
142 * Streaming variant of `put` — ingests a Readable without ever
143 * materializing the whole file in memory. Memory peak is one
144 * FastCDC chunk (≤ tier maxSize, ≤ 4 MB at tier 3) regardless of
145 * input size. Disk peak is the spooled tmpfile (linear, freed in
146 * the `finally` block).
147 *
148 * Usage (server-side upload handler):
149 * await overlay.putStream(filePath, req) // node http req
150 * await overlay.putStream(path, fs.createReadStream(localFile))
151 *
152 * Returns the same shape as `put`: { oid, size, chunkCount, pointer }.
153 *
154 * Why two passes (req → tmpfile → chunkStream)?
155 * The hyper-overlay file digest matches `hashChunk(file)` —
156 * blake2b(LEAF_TYPE || uint64-LE(size) || data). The size prefix
157 * forces us to know `size` before hashing the bytes, which a
158 * chunked HTTP upload doesn't tell us up front. The tmpfile
159 * bridges that gap: stream in, stat for size, stream out through
160 * chunker + hasher in one read.
161 */
162 async putStream (path_, readable) {
163 const tmpDir = path.join(os.tmpdir(), 'hyper-overlay-upload')
164 fs.mkdirSync(tmpDir, { recursive: true })
165 // crypto.randomBytes for uniqueness — no `process.pid` here because
166 // Bare doesn't expose `process` as a global.
167 const tmpFile = path.join(tmpDir,
168 'up-' + Date.now() + '-' + crypto.randomBytes(6).toString('hex'))
169
170 try {
171 // Pass 0: input → tmpfile (size unknown until EOF)
172 const writer = fs.createWriteStream(tmpFile)
173 await pipelinePromise(readable, writer)
174 const size = fs.statSync(tmpFile).size
175 if (size === 0) throw new Error('Empty body')
176
177 // Pass 1: chunk + hash file in one read
178 const tier = selectTier(size)
179 const hasher = createStreamingHasher({ size })
180 const chunkMeta = []
181 const reader = fs.createReadStream(tmpFile)
182 for await (const c of chunkStream(reader, { tier })) {
183 hasher.update(c.data)
184 // Register each chunk as it streams. Chunk data is at most
185 // tier.maxSize bytes — capped, GC'd as the loop iterates.
186 await this._registry.putChunk(c.hash, {
187 size: c.length,
188 type: 'cache',
189 data: c.data.toString('base64')
190 })
191 chunkMeta.push({ hash: c.hash, offset: c.offset, length: c.length })
192 }
193 const fileHash = hasher.digest()
194
195 // Build manifest if needed
196 let manifestHash = null
197 if (chunkMeta.length > INLINE_MAX) {
198 const manifest = encodeManifest(
199 chunkMeta.map(c => ({ hash: c.hash, size: c.length })),
200 fileHash
201 )
202 manifestHash = hashChunk(manifest)
203 await this._registry.putManifest(manifestHash, manifest)
204 }
205
206 // Store namespace entry
207 const entry = {
208 contentHash: fileHash,
209 size,
210 chunkCount: chunkMeta.length,
211 manifestHash,
212 chunks: chunkMeta.length <= INLINE_MAX
213 ? chunkMeta.map(c => ({ hash: c.hash, size: c.length }))
214 : null,
215 createdAt: Date.now()
216 }
217 await this._registry._bee.put(`file:${path_}`, entry)
218
219 const pointer = serialize({
220 oid: fileHash,
221 size,
222 chunkCount: chunkMeta.length,
223 chunks: entry.chunks,
224 manifestHash
225 })
226
227 return { oid: fileHash, size, chunkCount: chunkMeta.length, pointer }
228 } finally {
229 try { fs.unlinkSync(tmpFile) } catch {}
230 }
231 }
232
233 /**
234 * Like `putStream`, but for files already on disk. Skips the tmpfile
235 * spool that `putStream` uses because the size is known up front.
236 *
237 * Use this when you have a path; use `putStream` when the source size
238 * isn't known until EOF (e.g., HTTP chunked upload).
239 *
240 * Returns { oid, size, chunkCount, pointer } — same shape as `put` and `putStream`.
241 */
242 async putFile (path_, srcPath) {
243 const size = fs.statSync(srcPath).size
244 if (size === 0) throw new Error('Empty file at ' + srcPath)
245
246 const tier = selectTier(size)
247 const hasher = createStreamingHasher({ size })
248 const chunkMeta = []
249 // Collect chunk records and commit in a single Hyperbee batch at the
250 // end of the stream. Individual per-chunk puts on a multi-GB file
251 // (~500 chunks at tier 3) accumulate ~250 MB of transient RSS from
252 // b-tree node rewrites + Hypercore's append cache; batching folds
253 // those into one flush.
254 const pendingRegistrations = []
255 const reader = fs.createReadStream(srcPath)
256 for await (const c of chunkStream(reader, { tier })) {
257 hasher.update(c.data)
258 pendingRegistrations.push({
259 hash: c.hash,
260 source: { size: c.length, type: 'local', path: srcPath, offset: c.offset }
261 })
262 chunkMeta.push({ hash: c.hash, offset: c.offset, length: c.length })
263 }
264 await this._registry.putChunks(pendingRegistrations)
265 const fileHash = hasher.digest()
266
267 let manifestHash = null
268 if (chunkMeta.length > INLINE_MAX) {
269 const manifest = encodeManifest(
270 chunkMeta.map(c => ({ hash: c.hash, size: c.length })),
271 fileHash
272 )
273 manifestHash = hashChunk(manifest)
274 await this._registry.putManifest(manifestHash, manifest)
275 }
276
277 const entry = {
278 contentHash: fileHash,
279 size,
280 chunkCount: chunkMeta.length,
281 manifestHash,
282 chunks: chunkMeta.length <= INLINE_MAX
283 ? chunkMeta.map(c => ({ hash: c.hash, size: c.length }))
284 : null,
285 createdAt: Date.now()
286 }
287 await this._registry._bee.put(`file:${path_}`, entry)
288
289 const pointer = serialize({
290 oid: fileHash,
291 size,
292 chunkCount: chunkMeta.length,
293 chunks: entry.chunks,
294 manifestHash
295 })
296
297 return { oid: fileHash, size, chunkCount: chunkMeta.length, pointer }
298 }
299
300 /**
301 * Retrieve a file — assembles from chunks
302 * @param {string} path - file path
303 * @returns {Buffer|null}
304 */
305 async get (path) {
306 const entry = await this._registry._bee.get(`file:${path}`)
307 if (!entry) return null
308
309 const fileInfo = entry.value
310
311 // Get chunk list
312 let chunkList
313 if (fileInfo.chunks) {
314 chunkList = fileInfo.chunks
315 } else if (fileInfo.manifestHash) {
316 const manifestData = await this._registry.getManifest(fileInfo.manifestHash)
317 if (!manifestData) return null
318 const manifest = decodeManifest(manifestData)
319 chunkList = manifest.chunks
320 } else {
321 return null
322 }
323
324 // Assemble file from chunks
325 const parts = []
326 for (let i = 0; i < chunkList.length; i++) {
327 const c = chunkList[i]
328
329 // Try local first
330 let data = await this._registry.getChunkData(c.hash)
331
332 // Try fetching from peers if missing locally
333 if (!data && this._protocol && fileInfo.manifestHash) {
334 try {
335 data = await this._protocol.requestChunk(fileInfo.manifestHash, i)
336 if (data) {
337 // Cache the fetched chunk locally
338 await this._cacheFetchedChunk(c.hash, data)
339 }
340 } catch {
341 // Peer fetch failed, continue
342 }
343 }
344
345 if (!data) return null
346 parts.push(data)
347 }
348
349 return Buffer.concat(parts)
350 }
351
352 /**
353 * Delete a file from the overlay
354 */
355 async del (path) {
356 await this._registry._bee.del(`file:${path}`)
357 }
358
359 /**
360 * List files in the overlay
361 * @param {string} [dir] - directory prefix (default: list all)
362 * @returns {Array<{ path: string, size: number, chunkCount: number }>}
363 */
364 async list (dir = '') {
365 const prefix = dir ? `file:${dir}` : 'file:'
366 const files = []
367
368 for await (const entry of this._registry._bee.createReadStream({
369 gt: prefix, lt: prefix + '\xff'
370 })) {
371 files.push({
372 path: entry.key.slice('file:'.length),
373 size: entry.value.size,
374 chunkCount: entry.value.chunkCount
375 })
376 }
377
378 return files
379 }
380
381 /**
382 * Get file metadata without content
383 */
384 async entry (path) {
385 const e = await this._registry._bee.get(`file:${path}`)
386 return e ? e.value : null
387 }
388
389 /**
390 * Check if file exists
391 */
392 async exists (path) {
393 const e = await this._registry._bee.get(`file:${path}`)
394 return e !== null
395 }
396
397 // ── Adapter Interface ────────────────────────────────────────
398 // These methods are the surface that FUSE, Cloud Files API,
399 // FileProvider, and WebDAV adapters consume.
400
401 /**
402 * Get file or directory metadata
403 * @param {string} path
404 * @returns {{ size, mtime, type, contentHash, chunkCount, hydrated } | null}
405 */
406 async stat (path) {
407 if (path === '/' || path === '') {
408 return { size: 0, mtime: Date.now(), type: 'dir', contentHash: null, chunkCount: 0, hydrated: true }
409 }
410
411 const e = await this._registry._bee.get(`file:${path}`)
412 if (e) {
413 const v = e.value
414 const availability = await this._getChunkAvailability(v)
415 return {
416 size: v.size,
417 mtime: v.createdAt || Date.now(),
418 type: 'file',
419 contentHash: v.contentHash,
420 chunkCount: v.chunkCount,
421 hydrated: availability.local === availability.total
422 }
423 }
424
425 // Check if path is a directory (has children)
426 const prefix = `file:${path.endsWith('/') ? path : path + '/'}`
427 for await (const _ of this._registry._bee.createReadStream({ gt: prefix, lt: prefix + '\xff', limit: 1 })) { // eslint-disable-line
428 return { size: 0, mtime: Date.now(), type: 'dir', contentHash: null, chunkCount: 0, hydrated: true }
429 }
430
431 return null
432 }
433
434 /**
435 * List immediate children of a directory
436 * @param {string} dir
437 * @returns {string[]} child names (not full paths)
438 */
439 async readdir (dir = '/') {
440 const normalized = dir.endsWith('/') ? dir : dir + '/'
441 const prefix = `file:${normalized}`
442 const names = new Set()
443
444 for await (const entry of this._registry._bee.createReadStream({ gt: prefix, lt: prefix + '\xff' })) {
445 const relative = entry.key.slice(prefix.length)
446 const firstSegment = relative.split('/')[0]
447 if (firstSegment) names.add(firstSegment)
448 }
449
450 return [...names]
451 }
452
453 /**
454 * Read a byte range from a file (the critical adapter primitive)
455 * Maps byte range to chunk indices, reads only needed chunks.
456 * @param {string} path
457 * @param {number} offset - byte offset
458 * @param {number} length - bytes to read
459 * @returns {Buffer | null}
460 */
461 async read (path, offset, length) {
462 const chunkList = await this._getChunkList(path)
463 if (!chunkList) return null
464
465 const fileInfo = await this.entry(path)
466 if (!fileInfo) return null
467
468 // Clamp to file size
469 const end = Math.min(offset + length, fileInfo.size)
470 if (offset >= fileInfo.size) return Buffer.alloc(0)
471
472 // Find which chunks cover the requested byte range
473 let pos = 0
474 const parts = []
475
476 for (let i = 0; i < chunkList.length; i++) {
477 const c = chunkList[i]
478 const chunkEnd = pos + c.size
479
480 if (chunkEnd <= offset) { pos = chunkEnd; continue }
481 if (pos >= end) break
482
483 let data = await this._registry.getChunkData(c.hash)
484 if (!data && this._protocol && fileInfo.manifestHash) {
485 try {
486 data = await this._protocol.requestChunk(fileInfo.manifestHash, i)
487 if (data) {
488 await this._cacheFetchedChunk(c.hash, data)
489 }
490 } catch {}
491 }
492 if (!data) return null
493
494 // Slice to requested range within this chunk
495 const sliceStart = Math.max(offset - pos, 0)
496 const sliceEnd = Math.min(end - pos, c.size)
497 parts.push(data.subarray(sliceStart, sliceEnd))
498
499 pos = chunkEnd
500 }
501
502 return Buffer.concat(parts)
503 }
504
505 /**
506 * Create a readable stream for a file
507 * @param {string} path
508 * @param {{ start?: number, end?: number }} [opts]
509 * @returns {Readable | null}
510 */
511 createReadStream (path, opts = {}) {
512 const overlay = this
513 const start = opts.start || 0
514 const end = opts.end || Infinity
515
516 let chunkList = null
517 let fileInfo = null
518 let pos = 0
519 let idx = 0
520
521 return new Readable({
522 async open (cb) {
523 try {
524 chunkList = await overlay._getChunkList(path)
525 fileInfo = await overlay.entry(path)
526 if (!chunkList || !fileInfo) return cb(new Error('File not found'))
527 cb(null)
528 } catch (err) { cb(err) }
529 },
530
531 async read (cb) {
532 try {
533 while (idx < chunkList.length) {
534 const c = chunkList[idx]
535 const chunkEnd = pos + c.size
536
537 if (chunkEnd <= start) { pos = chunkEnd; idx++; continue }
538 if (pos >= end) { this.push(null); return cb(null) }
539
540 let data = await overlay._registry.getChunkData(c.hash)
541 if (!data && overlay._protocol && fileInfo.manifestHash) {
542 data = await overlay._protocol.requestChunk(fileInfo.manifestHash, idx)
543 if (data) {
544 await overlay._cacheFetchedChunk(c.hash, data)
545 }
546 }
547 if (!data) return cb(new Error('Chunk unavailable: ' + c.hash))
548
549 const sliceStart = Math.max(start - pos, 0)
550 const sliceEnd = Math.min(end - pos, c.size)
551 this.push(data.subarray(sliceStart, sliceEnd))
552
553 pos = chunkEnd
554 idx++
555 return cb(null)
556 }
557 this.push(null)
558 cb(null)
559 } catch (err) { cb(err) }
560 }
561 })
562 }
563
564 /**
565 * Fetch all missing chunks for a file from peers
566 * @param {string} path
567 * @param {{ onProgress?: function }} [opts]
568 */
569 async hydrate (path, opts = {}) {
570 const chunkList = await this._getChunkList(path)
571 if (!chunkList) throw new Error('File not found: ' + path)
572
573 const fileInfo = await this.entry(path)
574 let fetched = 0
575
576 for (let i = 0; i < chunkList.length; i++) {
577 const c = chunkList[i]
578 const has = await this._registry.getChunkData(c.hash)
579 if (has) { fetched++; continue }
580
581 if (!this._protocol || !fileInfo.manifestHash) {
582 throw new Error('Chunk unavailable locally and no peer connection: ' + c.hash)
583 }
584
585 const data = await this._protocol.requestChunk(fileInfo.manifestHash, i)
586 if (!data) throw new Error('Chunk unavailable from peers: ' + c.hash)
587
588 await this._cacheFetchedChunk(c.hash, data)
589 fetched++
590
591 if (opts.onProgress) opts.onProgress({ fetched, total: chunkList.length })
592 }
593
594 return { fetched, total: chunkList.length }
595 }
596
597 /**
598 * Check chunk availability for a file
599 * @param {string} path
600 * @returns {{ total: number, local: number, ratio: number }}
601 */
602 async getAvailability (path) {
603 const fileInfo = await this.entry(path)
604 if (!fileInfo) return { total: 0, local: 0, ratio: 0 }
605 return this._getChunkAvailability(fileInfo)
606 }
607
608 // ── Internal helpers ────────────────────────────────────────
609
610 async _getChunkList (path) {
611 const entry = await this._registry._bee.get(`file:${path}`)
612 if (!entry) return null
613
614 const fileInfo = entry.value
615 if (fileInfo.chunks) return fileInfo.chunks
616
617 if (fileInfo.manifestHash) {
618 const manifestData = await this._registry.getManifest(fileInfo.manifestHash)
619 if (!manifestData) return null
620 return decodeManifest(manifestData).chunks
621 }
622
623 return null
624 }
625
626 async _getChunkAvailability (fileInfo) {
627 let chunkList = fileInfo.chunks
628 if (!chunkList && fileInfo.manifestHash) {
629 try {
630 const manifestData = await this._registry.getManifest(fileInfo.manifestHash)
631 if (manifestData) chunkList = decodeManifest(manifestData).chunks
632 } catch {}
633 }
634 if (!chunkList) chunkList = []
635
636 let local = 0
637 for (const c of chunkList) {
638 if (await this._registry.hasChunk(c.hash)) local++
639 }
640
641 return { total: chunkList.length, local, ratio: chunkList.length ? local / chunkList.length : 1 }
642 }
643
644 // ── Replication ─────────────────────────────────────────────
645
646 /**
647 * Attach overlay protocol to a Protomux connection
648 */
649 attachProtocol (mux) {
650 return this._protocol.attach(mux)
651 }
652
653 /**
654 * Mount HTTP chunk + upload endpoints on a server
655 *
656 * @param {Object} httpServer
657 * @param {string} prefix
658 * @param {Object} [opts]
659 * @param {string} [opts.uploadSpoolDir] - persistent dir for upload
660 * spool files. Required if you want POST {prefix}/upload to work;
661 * the dir is the canonical storage for upload bytes (type:'local'
662 * chunk pointers reference it).
663 */
664 serveHTTP (httpServer, prefix, opts = {}) {
665 serveOverlay(this._registry, httpServer, prefix, this, opts)
666 }
667
668 // ── Low-level ───────────────────────────────────────────────
669
670 async getChunk (hash) { return this._registry.getChunk(hash) }
671 async hasChunk (hash) { return this._registry.hasChunk(hash) }
672 async stats () { return this._registry.stats() }
673
674 /**
675 * Content-addressed fetch by chunk hash — looks up the chunk locally,
676 * and if missing, asks connected peers over the mux protocol.
677 * Fetched chunks are cached locally so repeat fetches are free.
678 *
679 * Works for any chunk that lives in the registry, including small files
680 * (<1MB) where the file hash is also the single chunk's hash. For larger
681 * multi-chunk files the caller needs the file's manifest hash and should
682 * use the manifest+chunkRequest flow instead (hydrate / read / get).
683 *
684 * @param {string} hash - blake2b chunk hash (hex)
685 * @returns {Promise<Buffer|null>}
686 */
687 async getByHash (hash) {
688 const local = await this._registry.getChunkData(hash)
689 if (local) return local
690
691 if (!this._protocol) return null
692
693 const remote = await this._protocol.requestChunkByHash(hash)
694 if (!remote) return null
695
696 // Cache the fetched bytes locally so subsequent calls are free
697 await this._cacheFetchedChunk(hash, remote)
698
699 return remote
700 }
701
702 /**
703 * Persist a chunk just fetched from a peer.
704 *
705 * With a configured spool dir (`opts.chunkSpoolDir`) the raw bytes are
706 * written to a content-addressed file `<spoolDir>/<hash>` and registered
707 * as a `local` pointer — no base64 +33% tax, and ChunkRegistry's
708 * cross-mode dedup then prefers it over any inline copy. Without a spool
709 * dir (default) it falls back to the legacy inline base64 `cache` source,
710 * so behavior is unchanged for callers that don't opt in. Any spool I/O
711 * error also falls back to inline cache, so a fetch never fails just
712 * because the spool is unwritable.
713 *
714 * @param {string} hash - blake2b chunk hash (hex)
715 * @param {Buffer} data - the fetched chunk bytes
716 */
717 async _cacheFetchedChunk (hash, data) {
718 if (this._chunkSpoolDir) {
719 try {
720 const dest = path.join(this._chunkSpoolDir, hash)
721 if (!fs.existsSync(dest)) {
722 await fs.promises.mkdir(this._chunkSpoolDir, { recursive: true })
723 const tmp = dest + '.tmp-' + (++this._spoolSeq)
724 await fs.promises.writeFile(tmp, data)
725 await fs.promises.rename(tmp, dest)
726 }
727 await this._registry.putChunk(hash, { size: data.length, type: 'local', path: dest, offset: 0 })
728 return
729 } catch {
730 // fall through to inline cache on any spool error
731 }
732 }
733 await this._registry.putChunk(hash, { size: data.length, type: 'cache', data: data.toString('base64') })
734 }
735}
736
737// Re-exports (v1 — kept for backward compatibility)
738export { chunk, hashChunk, createStreamingHasher } from './lib/chunker.js'
739export { serialize, parse, isPointer } from './lib/pointer.js'
740export { encode as encodeManifest, decode as decodeManifest } from './lib/manifest.js'
741export { ChunkRegistry } from './lib/chunk-registry.js'
23export { HyperOverlay, HyperOverlay as default } from './lib/overlay.js'
74224export { OverlayProtocol } from './lib/protocol.js'
743export { serveOverlay } from './lib/http.js'
744
745// v2 exports
746export { FileIndex } from './lib/file-index.js'
25export {
26 TransferManager,
27 PARTIAL_SUFFIX,
28 partialPathFor,
29 journalNameFor,
30 cleanupOrphanedJournals
31} from './lib/transfer.js'
32export { ChunkScheduler } from './lib/chunk-scheduler.js'
33export { FileIndex, CONTENT_REGISTER_PREFIX } from './lib/file-index.js'
74734export { SyncEngine, STRATEGIES } from './lib/sync-engine.js'
748export { TransferManager } from './lib/transfer.js'
749export { OverlayProtocolV2 } from './lib/protocol-v2.js'
750export { HyperOverlayV2 } from './lib/overlay-v2.js'
751export { WatchManager } from './lib/watch-manager.js'
752export { createMatcher, loadIgnoreFile } from './lib/ignore.js'
753export { selectTier, getTierParams, TIERS } from './lib/chunker.js'
35export {
36 chunk,
37 chunkStream,
38 hashChunk,
39 createStreamingHasher,
40 selectTier,
41 getTierParams,
42 TIERS,
43 MIN_SIZE,
44 AVG_SIZE,
45 MAX_SIZE
46} from './lib/chunker.js'
47export * as messages from './lib/messages.js'
48
49// Transitional aliases for consumers still on the pre-3.0 v2 naming
50// (`HyperOverlayV2` / `OverlayProtocolV2`, and the `/protocol-v2`, `/overlay-v2`,
51// `/messages-v2` subpaths in package.json). Same classes — prefer the new names.
52// These let 0.3.0 land without forcing simultaneous edits to the live consumers.
53export { HyperOverlay as HyperOverlayV2 } from './lib/overlay.js'
54export { OverlayProtocol as OverlayProtocolV2 } from './lib/protocol.js'
55
56// Our kept superset — dropped by ok/hyper-overlay-v3, retained here because
57// pgh (tree/pointer/ignore), hyper-deploy (tree), and the forge (manifest)
58// import them via subpaths. Not application concerns; they are the
59// directory→Merkle-manifest + LFS-pointer primitives the distribution layer needs.
60export { serialize, parse, isPointer, INLINE_MAX } from './lib/pointer.js'
61export { encode as encodeManifest, decode as decodeManifest } from './lib/manifest.js'
62export { createMatcher } from './lib/ignore.js'
63export * as tree from './lib/tree.js'
lib/chunk-registry.js +0 -243
……@@ -1,243 +0,0 @@
1/**
2 * ChunkRegistry — Content-addressed chunk storage backed by Hyperbee
3 *
4 * Maps chunk hashes to source locations (local file path + offset, peer, cache).
5 * Also stores compressed manifests for large files.
6 *
7 * Hyperbee schema:
8 * chunk:<hash> → { size, sources: [{ type, path?, offset?, peer? }] }
9 * manifest:<hash> → raw compressed manifest bytes
10 */
11
12import fs from '#fs'
13import Hyperbee from 'hyperbee'
14import ReadyResource from 'ready-resource'
15
16export class ChunkRegistry extends ReadyResource {
17 constructor (store, opts = {}) {
18 super()
19 this._store = store
20 this._bee = null
21 this._opts = opts
22 }
23
24 async _open () {
25 const core = this._store.get({ name: 'chunk-registry', valueEncoding: 'binary' })
26 this._bee = new Hyperbee(core, {
27 keyEncoding: 'utf-8',
28 valueEncoding: 'json'
29 })
30 await this._bee.ready()
31 }
32
33 async _close () {
34 if (this._bee) await this._bee.close()
35 }
36
37 /**
38 * Register a chunk's location
39 * @param {string} hash - blake2b hex hash
40 * @param {{ size: number, type: string, path?: string, offset?: number }} source
41 */
42 async putChunk (hash, source) {
43 const key = `chunk:${hash}`
44 const existing = await this._bee.get(key)
45
46 if (existing) {
47 const val = existing.value
48 if (mergeSource(val, source)) await this._bee.put(key, val)
49 } else {
50 await this._bee.put(key, {
51 size: source.size,
52 sources: [source]
53 })
54 }
55 }
56
57 /**
58 * Register many chunks in a single Hyperbee batch. Collapses N
59 * individual puts (each rewriting b-tree nodes + holding hypercore
60 * append blocks in memory) into one flush. Used by putFile for
61 * multi-chunk files where the per-put overhead dominated RSS.
62 *
63 * `records` shape: [{ hash, source }, ...] where source matches putChunk.
64 */
65 async putChunks (records) {
66 if (!records || records.length === 0) return
67
68 // Single-pass: read existing entries via the live bee (batches don't
69 // see their own writes for `get`), merge per chunk, write through the
70 // batch. Records arrive in stream order and may contain duplicates
71 // (same chunk hash from different offsets) — coalesce in a Map.
72 const merged = new Map()
73 for (const { hash, source } of records) {
74 let cur = merged.get(hash)
75 if (!cur) { cur = { size: source.size, sources: [] }; merged.set(hash, cur) }
76 mergeSource(cur, source)
77 }
78
79 const batch = this._bee.batch()
80 try {
81 for (const [hash, val] of merged) {
82 const existing = await this._bee.get(`chunk:${hash}`)
83 if (existing) {
84 const cur = existing.value
85 let changed = false
86 for (const s of val.sources) {
87 if (mergeSource(cur, s)) changed = true
88 }
89 if (changed) await batch.put(`chunk:${hash}`, cur)
90 } else {
91 await batch.put(`chunk:${hash}`, val)
92 }
93 }
94 await batch.flush()
95 } catch (err) {
96 try { await batch.destroy() } catch {}
97 throw err
98 }
99 }
100
101 /**
102 * Get chunk metadata
103 * @param {string} hash
104 * @returns {{ size: number, sources: Array } | null}
105 */
106 async getChunk (hash) {
107 const entry = await this._bee.get(`chunk:${hash}`)
108 return entry ? entry.value : null
109 }
110
111 /**
112 * Check if chunk exists
113 * @param {string} hash
114 * @returns {boolean}
115 */
116 async hasChunk (hash) {
117 const entry = await this._bee.get(`chunk:${hash}`)
118 return entry !== null
119 }
120
121 /**
122 * Remove chunk from registry
123 * @param {string} hash
124 */
125 async deleteChunk (hash) {
126 await this._bee.del(`chunk:${hash}`)
127 }
128
129 /**
130 * Store a compressed manifest
131 * @param {string} hash - manifest hash
132 * @param {Buffer} data - compressed manifest bytes
133 */
134 async putManifest (hash, data) {
135 // Store as base64 since the bee uses JSON encoding
136 await this._bee.put(`manifest:${hash}`, {
137 data: data.toString('base64'),
138 size: data.length,
139 createdAt: Date.now()
140 })
141 }
142
143 /**
144 * Get a compressed manifest
145 * @param {string} hash
146 * @returns {Buffer | null}
147 */
148 async getManifest (hash) {
149 const entry = await this._bee.get(`manifest:${hash}`)
150 if (!entry) return null
151 return Buffer.from(entry.value.data, 'base64')
152 }
153
154 /**
155 * Get raw chunk data as a Buffer
156 * @param {string} hash - blake2b hex hash
157 * @returns {Buffer | null}
158 */
159 async getChunkData (hash) {
160 const entry = await this.getChunk(hash)
161 if (!entry) return null
162
163 const cached = entry.sources.find(s => s.data)
164 if (cached) {
165 return typeof cached.data === 'string'
166 ? Buffer.from(cached.data, 'base64')
167 : cached.data
168 }
169
170 const local = entry.sources.find(s => s.type === 'local' && s.path)
171 if (local) {
172 const start = typeof local.offset === 'number' ? local.offset : 0
173 const end = start + entry.size - 1
174 try {
175 const stream = fs.createReadStream(local.path, { start, end })
176 const parts = []
177 for await (const buf of stream) parts.push(buf)
178 return Buffer.concat(parts)
179 } catch {
180 return null
181 }
182 }
183
184 return null
185 }
186
187 /**
188 * Get registry stats
189 * @returns {{ chunkCount: number, manifestCount: number, totalSize: number }}
190 */
191 async stats () {
192 let chunkCount = 0
193 let manifestCount = 0
194 let totalSize = 0
195
196 for await (const entry of this._bee.createReadStream()) {
197 if (entry.key.startsWith('chunk:')) {
198 chunkCount++
199 totalSize += entry.value.size || 0
200 } else if (entry.key.startsWith('manifest:')) {
201 manifestCount++
202 }
203 }
204
205 return { chunkCount, manifestCount, totalSize }
206 }
207}
208
209// A source "yields bytes" locally if reading it returns the chunk content
210// without a network round-trip: an inline `cache` copy, or a working-tree
211// `local` pointer. A bare `peer` routing hint carries no bytes.
212function byteYielding (s) {
213 return (s.type === 'cache' && s.data != null) || (s.type === 'local' && s.path != null)
214}
215
216// Merge `source` into a chunk record `val` ({ size, sources }) applying
217// cross-mode dedup. Returns true iff the record changed.
218//
219// The rules close the "same bytes stored two ways" gap (working-tree pointer
220// + base64 inline cache for one chunk hash):
221// 1. Never append a redundant inline `cache` copy when a byte-yielding
222// source already exists — that duplicates the bytes (and base64 inflates
223// them +33%). Peer routing hints don't yield bytes, so they never block.
224// 2. A durable `local` pointer supersedes any inline `cache` copy of the
225// same bytes: keep the pointer (smaller, no base64 tax, working tree is
226// the source of truth) and drop the now-redundant cache copies.
227function mergeSource (val, source) {
228 if (source.type === 'cache' && val.sources.some(byteYielding)) return false
229
230 const has = val.sources.some(s =>
231 s.type === source.type && s.path === source.path && s.offset === source.offset
232 )
233 let changed = false
234 if (!has) { val.sources.push(source); changed = true }
235
236 if (source.type === 'local' && source.path != null) {
237 const before = val.sources.length
238 val.sources = val.sources.filter(s => s.type !== 'cache')
239 if (val.sources.length !== before) changed = true
240 }
241
242 return changed
243}
lib/chunk-scheduler.js +340 -2
……@@ -1,2 +1,340 @@
1// SHIM: v0.3.0 engine lives in lib/v3/ (Ok mirror). Re-export under our name/subpath.
2export * from './v3/chunk-scheduler.js'
1/**
2 * ChunkScheduler — multi-source (torrent-style) fetch of one content-addressed
3 * file across several peers, modelled on Hypercore's block replicator.
4 *
5 * Chunks are content-addressed, so any peer holding the file can serve any
6 * chunk. The scheduler:
7 * - requests the chunk list from each peer (identical, content-addressed),
8 * - tracks a global `inflight` map so each chunk is requested from exactly
9 * ONE peer at a time (no wasted bandwidth),
10 * - keeps each peer busy up to a per-peer inflight cap (parallel download),
11 * - reassigns a peer's inflight chunks to others on stall/disconnect,
12 * - hash-verifies every chunk (via TransferManager.writeChunk) and the whole
13 * file before finalizing.
14 *
15 * It drives the EXISTING v2 wire messages (chunk-need / chunk-data) — no new
16 * message type is needed for the full-copy-seeder case. The protocol delegates
17 * `_onChunkHashes` / `_onChunkData` to a scheduler ONLY when one is registered
18 * for the synthetic `content:<hash>` path, so the legacy single-peer flow
19 * is untouched.
20 */
21
22import { hashChunk } from './chunker.js'
23
24const DEFAULT_CAP = 8
25// Head-of-line fairness — cede the event loop every YIELD_EVERY accepted chunks
26// so a burst of buffered chunkData frames (protomux dispatches every frame from
27// one socket read synchronously) can't starve other listeners multiplexed on the
28// same connection (e.g. Hypercore replication). Kept local (not imported from
29// transfer.js) so this module stays Node-loadable for the unit suite.
30const yieldToLoop = () => new Promise((resolve) => setTimeout(resolve, 0))
31const YIELD_EVERY = 16
32// Idle (no-progress) timeout, reset on every accepted chunk and on the
33// initial chunk-list response. A steadily-progressing transfer never trips it —
34// only a genuine stall (no bytes for this long) fails. A fixed OVERALL timeout
35// here capped large transfers (e.g. a multi-GB file at any bandwidth always
36// exceeded 30s) and aborted them mid-stream.
37const DEFAULT_IDLE_TIMEOUT = 30000
38// Local write-error codes that may recover on retry (vs ENOSPC/EACCES/…
39// which are fatal): a transient one keeps the chunk retryable instead of failing
40// the whole multi-source fetch.
41const TRANSIENT_WRITE_CODES = new Set(['EBUSY', 'EAGAIN', 'EINTR', 'EMFILE', 'ENFILE'])
42// Min interval between have-progress reports to holders (their sender-side bar).
43// Throttles the transferProgress frames so a fast transfer can't spam holders.
44const DEFAULT_REPORT_INTERVAL = 1000
45
46export class ChunkScheduler {
47 /**
48 * @param {object} opts
49 * opts.path synthetic path key ('content:<hash>')
50 * opts.destPath where the assembled file is written
51 * opts.transfer TransferManager
52 * opts.sendNeed (peer, indices[]) => void — sends a chunk-need message
53 * opts.cap per-peer inflight cap (default 8)
54 * opts.timeout idle (no-progress) timeout ms (default 30000)
55 * opts.onProgress optional (receivedBytes, totalBytes) => void on each accepted chunk
56 */
57 constructor (opts) {
58 this.path = opts.path
59 this.destPath = opts.destPath
60 this._transfer = opts.transfer
61 this._sendNeed = opts.sendNeed
62 this._cap = opts.cap || DEFAULT_CAP
63 this._onProgress = opts.onProgress || null
64 this._onVerify = opts.onVerify || null
65 this._onBaseline = opts.onBaseline || null
66 this._reportInterval = opts.reportInterval ?? DEFAULT_REPORT_INTERVAL
67 this._lastReportAt = 0
68 // cumulative byte accounting so onProgress can drive a
69 // remaining-bytes/ETA bar, not just a chunk count.
70 this._receivedBytes = 0
71 this._totalBytes = 0
72 this._sinceYield = 0 // event-loop yield counter (see YIELD_EVERY)
73
74 this._peers = new Set()
75 this._started = false
76 this._settingUp = false
77 this._finalizing = false
78 this._done = false
79 this._chunks = null // [{ hash, offset, length }]
80 this._needed = new Set() // indices not yet written
81 this._inflight = new Map() // index → peer (one peer per chunk)
82 this._peerInflight = new Map() // peer → count
83 this._received = new Map() // peer → chunks accepted (multi-source proof)
84
85 this._resolve = null
86 this._reject = null
87 this._onEnd = opts.onEnd || null
88 this._contentHash = opts.contentHash || null // enables incremental whole-file verify
89 this._startedAt = Date.now()
90 // idle timeout — re-armed on each progress signal (see _armIdleTimer).
91 this._idleTimeout = opts.timeout || DEFAULT_IDLE_TIMEOUT
92 this._timer = null
93 this._armIdleTimer()
94 }
95
96 promise () {
97 return new Promise((resolve, reject) => { this._resolve = resolve; this._reject = reject })
98 }
99
100 // (Re)arm the no-progress watchdog. Called at construction and on every
101 // forward-progress signal so only a genuine stall — no accepted bytes for the
102 // idle window — aborts the fetch, regardless of total file size.
103 _armIdleTimer () {
104 if (this._done) return
105 clearTimeout(this._timer)
106 this._timer = setTimeout(
107 () => this._fail(new Error('multi-source fetch stalled (no progress for ' + this._idleTimeout + 'ms)')),
108 this._idleTimeout,
109 )
110 }
111
112 _reportEnd (ok, reason) {
113 if (!this._onEnd) return
114 try {
115 this._onEnd({
116 ok,
117 reason,
118 receivedBytes: this._receivedBytes,
119 totalBytes: this._totalBytes,
120 chunksRemaining: this._needed.size,
121 totalChunks: this._chunks ? this._chunks.length : 0,
122 peers: this._peers.size,
123 elapsedMs: Date.now() - this._startedAt,
124 })
125 } catch {}
126 }
127
128 _fail (err) {
129 if (this._done) return
130 this._done = true
131 clearTimeout(this._timer)
132 this._reportEnd(false, err.message)
133 // Release the receiver state at the failure boundary: journal the hash
134 // frontier and close the fd + chunk stash so a never-retried failure parks
135 // nothing (the partial stays on disk for a later resume). Best-effort — the
136 // external-stop path (cancel) does its own pause/cancel before rejecting.
137 if (this._transfer && typeof this._transfer.pause === 'function') {
138 try { Promise.resolve(this._transfer.pause(this.destPath)).catch(() => {}) } catch {}
139 }
140 if (this._reject) this._reject(err)
141 }
142
143 _finish () {
144 if (this._done) return
145 this._done = true
146 clearTimeout(this._timer)
147 this._reportEnd(true, 'complete')
148 if (this._resolve) this._resolve({ chunksPerPeer: this._received, peerCount: this._received.size, size: this._totalBytes })
149 }
150
151 // External stop (pause or discard). The partial is kept or unlinked by
152 // the caller (OverlayProtocol.cancelContent via TransferManager.pause/cancel)
153 // BEFORE this; here we only stop the loop and reject the fetch with ECANCELLED.
154 cancel () {
155 if (this._done) return
156 this._done = true
157 clearTimeout(this._timer)
158 this._reportEnd(false, 'cancelled')
159 const err = new Error('fetch cancelled')
160 err.code = 'ECANCELLED'
161 if (this._reject) this._reject(err)
162 }
163
164 get done () { return this._done }
165
166 // A chunkHashes page arrived while a large map is still
167 // streaming across multiple frames. Receiving it is forward progress, so
168 // re-arm the no-progress watchdog (the full list dispatches via onChunkHashes
169 // only once the final page lands).
170 notePageProgress () { if (!this._done && !this._settingUp) this._armIdleTimer() }
171
172 /** A peer responded with the chunk list. First response starts the transfer. */
173 async onChunkHashes (peer, chunks) {
174 if (this._done) return
175 this._peers.add(peer)
176 if (!this._peerInflight.has(peer)) this._peerInflight.set(peer, 0)
177
178 if (!this._started) {
179 this._started = true
180 // The async startReceive (journal-less resume re-verify) can run for a while
181 // and accepts no chunks; suppress the no-progress watchdog (including later
182 // peers' re-arms) until setup completes, or a concurrent seeder's chunk list
183 // would re-arm it and self-trip a healthy resume.
184 this._settingUp = true
185 clearTimeout(this._timer)
186 // Derive offsets from the ordered chunk lengths.
187 let offset = 0
188 this._chunks = chunks.map((c) => {
189 const entry = { hash: c.hash, offset, length: c.length }
190 offset += c.length
191 return entry
192 })
193 this._totalBytes = offset // file size = sum of chunk lengths
194 let state
195 try {
196 state = await this._transfer.startReceive(
197 this.destPath,
198 { size: offset, chunks: this._chunks, contentHash: this._contentHash },
199 { isCancelled: () => this._done, onVerifyProgress: this._onVerify }
200 )
201 } catch (err) {
202 this._settingUp = false
203 if (this._done) return
204 return this._fail(err)
205 }
206 this._settingUp = false
207 if (this._done) return
208 // resume: only fetch chunks the partial doesn't already hold, and
209 // seed the byte counter so progress/ETA continue from the resumed offset.
210 let have = 0
211 for (let i = 0; i < this._chunks.length; i++) {
212 if (state.received.has(i)) have += this._chunks[i].length
213 else this._needed.add(i)
214 }
215 this._receivedBytes = have
216 if (have > 0) {
217 // report our resume have-bytes so holders' sender-side bars mirror our
218 // true progress, not just the bytes they re-serve. Re-sent periodically as more
219 // chunks land (onChunkData) so late-joining and multi-source holders stay in sync.
220 this._reportHave(Date.now())
221 if (this._onProgress) { try { this._onProgress(this._receivedBytes, this._totalBytes) } catch {} }
222 }
223 this._armIdleTimer()
224 if (this._needed.size === 0) return this._finalize()
225 return this._assign()
226 }
227 if (!this._settingUp) this._armIdleTimer() // a later peer's list is forward progress
228 this._assign()
229 }
230
231 /** A chunk arrived from a peer. Verify + write, then schedule more. */
232 async onChunkData (peer, index, data) {
233 if (this._done) return
234 const res = this._transfer.writeChunk(this.destPath, index, data)
235 // Free the inflight slot regardless — a bad chunk should be re-fetched.
236 if (this._inflight.get(index) === peer) {
237 this._inflight.delete(index)
238 this._peerInflight.set(peer, Math.max(0, (this._peerInflight.get(peer) || 1) - 1))
239 }
240 if (!res.ok) {
241 // A coded failure is a local fs error. A TRANSIENT code (device busy,
242 // fd pressure, interrupted syscall) can succeed on retry — leave the chunk in
243 // `needed` and reassign. Any other coded error (ENOSPC / EACCES / ENOENT / …) is
244 // fatal: fail the whole fetch carrying the code. No code = a hash/length
245 // mismatch — also retried elsewhere.
246 if (res.code && !TRANSIENT_WRITE_CODES.has(res.code)) {
247 const err = new Error('write failed: ' + res.error)
248 err.code = res.code
249 return this._fail(err)
250 }
251 this._assign()
252 return
253 }
254 this._needed.delete(index)
255 this._received.set(peer, (this._received.get(peer) || 0) + 1)
256 this._receivedBytes += (data ? data.length : 0)
257 this._armIdleTimer() // accepted bytes — reset the no-progress watchdog
258 if (this._onProgress) { try { this._onProgress(this._receivedBytes, this._totalBytes) } catch {} }
259 this._maybeReportHave()
260 if (this._needed.size === 0 && this._inflight.size === 0) return this._finalize()
261 // Periodic event-loop yield (see YIELD_EVERY)
262 if (++this._sinceYield >= YIELD_EVERY) {
263 this._sinceYield = 0
264 await yieldToLoop()
265 if (this._done) return
266 }
267 this._assign()
268 }
269
270 // Report our cumulative have-bytes to holders (their sender-side bar). Fired
271 // once at resume, then throttled as chunks land, and once at finalize so the bar settles
272 // at 100% across every holder regardless of who served what.
273 _reportHave (now) {
274 if (!this._onBaseline || this._receivedBytes <= 0) return
275 this._lastReportAt = now
276 try { this._onBaseline(this._receivedBytes) } catch {}
277 }
278
279 _maybeReportHave () {
280 if (!this._onBaseline) return
281 const now = Date.now()
282 if (now - this._lastReportAt >= this._reportInterval) this._reportHave(now)
283 }
284
285 /** A peer went away — return its inflight chunks to the pool. */
286 removePeer (peer) {
287 if (!this._peers.has(peer)) return
288 this._peers.delete(peer)
289 this._peerInflight.delete(peer)
290 for (const [index, p] of this._inflight) {
291 if (p === peer) this._inflight.delete(index) // back to needed (still in _needed)
292 }
293 if (this._done) return
294 if (this._peers.size === 0 && this._needed.size > 0) {
295 return this._fail(new Error('all peers gone with ' + this._needed.size + ' chunk(s) outstanding'))
296 }
297 this._assign()
298 }
299
300 async _finalize () {
301 if (this._finalizing || this._done) return
302 this._finalizing = true
303 this._reportHave(Date.now()) // final have=total so holders' bars settle at 100%
304 clearTimeout(this._timer) // the digest drain can outlast the idle window
305 let fin
306 try { fin = await this._transfer.finalize(this.destPath) }
307 catch (err) { return this._fail(err) }
308 if (!fin.ok) {
309 const err = new Error('finalize failed: ' + fin.error)
310 if (fin.code) err.code = fin.code // preserve EHASHMISMATCH for the caller
311 return this._fail(err)
312 }
313 this._finish()
314 }
315
316 /** Assign needed-but-not-inflight chunks to peers with spare capacity. */
317 _assign () {
318 if (this._done || !this._chunks) return
319 // Per peer, batch the indices we hand it this round.
320 const batches = new Map()
321 for (const peer of this._peers) {
322 let slots = this._cap - (this._peerInflight.get(peer) || 0)
323 if (slots <= 0) continue
324 for (const index of this._needed) {
325 if (slots <= 0) break
326 if (this._inflight.has(index)) continue
327 this._inflight.set(index, peer)
328 this._peerInflight.set(peer, (this._peerInflight.get(peer) || 0) + 1)
329 if (!batches.has(peer)) batches.set(peer, [])
330 batches.get(peer).push(index)
331 slots--
332 }
333 }
334 for (const [peer, indices] of batches) {
335 if (indices.length) this._sendNeed(peer, indices)
336 }
337 }
338}
339
340export { hashChunk }
lib/chunker.js +333 -2
……@@ -1,2 +1,333 @@
1// SHIM: v0.3.0 engine lives in lib/v3/ (Ok mirror). Re-export under our name/subpath.
2export * from './v3/chunker.js'
1/**
2 * FastCDC Content-Defined Chunker
3 *
4 * Gear rolling hash with normalized chunking (Xia et al. 2016).
5 * Produces content-stable chunk boundaries — editing the middle of a file
6 * only changes ~2 chunks regardless of file size.
7 *
8 * v2: Adaptive chunk sizing — tier selected by file size.
9 * All peers MUST use the same tier for a given file to produce matching boundaries.
10 */
11
12import crypto from 'hypercore-crypto'
13
14// ── Tier 0 (default, < 1 MB) ─────────────────────────────────
15export const MIN_SIZE = 4096
16export const AVG_SIZE = 16384
17export const MAX_SIZE = 65536
18
19// ── Adaptive chunk size tiers ─────────────────────────────────
20export const TIERS = [
21 { minSize: 4096, avgSize: 16384, maxSize: 65536 }, // Tier 0: < 1 MB
22 { minSize: 16384, avgSize: 65536, maxSize: 262144 }, // Tier 1: 1 MB - 100 MB
23 { minSize: 65536, avgSize: 262144, maxSize: 1048576 }, // Tier 2: 100 MB - 10 GB
24 { minSize: 262144, avgSize: 1048576, maxSize: 4194304 } // Tier 3: > 10 GB
25]
26
27const TIER_THRESHOLDS = [
28 1048576, // 1 MB
29 104857600, // 100 MB
30 10737418240 // 10 GB
31]
32
33/**
34 * Select the appropriate chunk size tier for a file
35 * @param {number} fileSize - total file size in bytes
36 * @returns {number} tier index (0-3)
37 */
38export function selectTier (fileSize) {
39 for (let i = 0; i < TIER_THRESHOLDS.length; i++) {
40 if (fileSize < TIER_THRESHOLDS[i]) return i
41 }
42 return TIERS.length - 1
43}
44
45/**
46 * Get chunk size parameters for a tier
47 * @param {number} tier - tier index (0-3)
48 * @returns {{ minSize: number, avgSize: number, maxSize: number }}
49 */
50export function getTierParams (tier) {
51 return TIERS[tier] || TIERS[0]
52}
53
54// Gear hash table — 256 random uint32 values
55// MUST be identical across all implementations for chunk boundary compatibility
56const GEAR = new Uint32Array([
57 0x5c95c078, 0x22408989, 0x2d48a214, 0x12842087, 0x530f8afb, 0x474536b9, 0x2963b4f1, 0x44cb738b,
58 0x4ea7403d, 0x4d606b6e, 0x074ec5d3, 0x3af39d18, 0x726c7b74, 0x60b3f158, 0x56a76369, 0x3e8ab850,
59 0x0c215cf6, 0x42ad3c2f, 0x65b26fa7, 0x69a3e894, 0x2a60445d, 0x21b0f756, 0x5caca9af, 0x373b4cf0,
60 0x1a4a56e1, 0x5eaf1e7b, 0x7b5d5c8a, 0x48f5abf4, 0x02a42b68, 0x1f3f1a4e, 0x67e16b96, 0x404c7833,
61 0x6e4e3a3c, 0x4f769e04, 0x0bfef25f, 0x3f08d1ef, 0x3bee8eb5, 0x4302687c, 0x3e9e2964, 0x0fef6b59,
62 0x70b95d2c, 0x0cffb210, 0x2e6c3f2a, 0x6d8f0f67, 0x77a49ba2, 0x35fa6b91, 0x43d2d154, 0x5d4e6c25,
63 0x0e7e5283, 0x5fac3d19, 0x20e34b36, 0x46b6de87, 0x48eed852, 0x63aae4d9, 0x2c65c18b, 0x5e5bc6ad,
64 0x7802c749, 0x44db9a82, 0x50e76e40, 0x532dfa2b, 0x56a17be8, 0x0a81e2e7, 0x5cf2fccc, 0x3ab39d12,
65 0x5c0c0b78, 0x21e8af93, 0x3dca216f, 0x15b4b92e, 0x55445f9c, 0x44f24b2e, 0x2db33f38, 0x444f6ec8,
66 0x4eeaede8, 0x48b27a8d, 0x00b7621d, 0x3c916d41, 0x72f65778, 0x60b1a155, 0x5a946e70, 0x3ea5d857,
67 0x0aca0e86, 0x41b3b22e, 0x6572e2a6, 0x6aa8e89a, 0x2d496f61, 0x23b6f159, 0x5eb7e1a6, 0x312e4fea,
68 0x1baf6dc4, 0x5eb0f27e, 0x7a505c92, 0x49e8abf2, 0x03b12c6e, 0x1e4a7a4d, 0x64e76a98, 0x414c7c39,
69 0x6f4a3f38, 0x4c719e04, 0x0efef963, 0x3d08c1e9, 0x3ce88fba, 0x44026c79, 0x3e1e2469, 0x10ef6858,
70 0x73b55e21, 0x0dcfb614, 0x2b6d3927, 0x6c8e1066, 0x74a09bb5, 0x36fa7690, 0x44d2c158, 0x5e4f6b26,
71 0x0e7e5788, 0x5dad3e1c, 0x22e14a30, 0x47b6dd88, 0x4befd953, 0x64aae5d8, 0x2b65b28a, 0x5f5ac6aa,
72 0x7903c648, 0x43db1a89, 0x50e76f45, 0x542dfb28, 0x55a07cef, 0x0b80e3ea, 0x5ef2f9cf, 0x38b31e15,
73 0x5d0d0a7d, 0x22e9ae91, 0x3ecb216a, 0x16b5b82f, 0x54445e99, 0x45f34c2d, 0x2eb23e3b, 0x454e6dc9,
74 0x4febedef, 0x49b37a88, 0x01b66320, 0x3d926e44, 0x71f75779, 0x61b0a054, 0x5b947f71, 0x3fa4d754,
75 0x0bcb0f83, 0x42b4b32f, 0x6473e3a5, 0x6ba9e999, 0x2e497064, 0x24b7f05a, 0x5fb6e0a7, 0x322f5fe9,
76 0x1cae7ec3, 0x5db0e37f, 0x7b514d91, 0x4ae9abf3, 0x02b22d6b, 0x1f4b7b50, 0x63e66b97, 0x414d7d34,
77 0x6e4f3a3d, 0x4d779f05, 0x0afe0960, 0x3e09d0ee, 0x3de98ebb, 0x43036c7a, 0x3f1f2560, 0x11f06959,
78 0x72b65e22, 0x0dceb711, 0x2a6e3826, 0x6d8f1165, 0x75a19ab4, 0x37fb7793, 0x45d3c057, 0x5f4e6a27,
79 0x0f7f5684, 0x5eae3f1d, 0x23e24b31, 0x48b7de89, 0x4ceed854, 0x63abe5db, 0x2c64b38b, 0x5e5bc7ab,
80 0x7a04c54a, 0x44db1b88, 0x51e86e41, 0x552efb2a, 0x54a17de0, 0x0a81e3e6, 0x5df3f8ce, 0x39b41f14,
81 0x5e0e0b79, 0x23e8af92, 0x3fcc216e, 0x17b4b92d, 0x55456e9d, 0x46f44d2f, 0x2fb33e39, 0x464f6ec7,
82 0x4febe8e9, 0x49b27b8c, 0x02b7631e, 0x3e916e42, 0x73f75677, 0x62b2a256, 0x5c957e72, 0x3ea6d856,
83 0x0cca0f85, 0x43b4b231, 0x6574e2a7, 0x6ca9e89b, 0x2f496e62, 0x25b7f058, 0x5eb7e0a5, 0x332e4eeb,
84 0x1daf6ec5, 0x5eb0e27d, 0x7c515d93, 0x4be8abf1, 0x04b32d6a, 0x204b7a4e, 0x64e76b99, 0x424c7d3a,
85 0x704a3e3b, 0x4e789f06, 0x0bfefa62, 0x3f09d1e8, 0x3ee98fbd, 0x45036d7b, 0x401f2461, 0x12f0685a,
86 0x74b65f23, 0x0ecfb713, 0x2b6d3928, 0x6e901267, 0x76a29bb6, 0x38fc7892, 0x46d3c159, 0x604e6b28,
87 0x107e5785, 0x5fae3e1e, 0x24e24b32, 0x49b8df8a, 0x4defd955, 0x64abe6da, 0x2d65b48c, 0x605bc8ac,
88 0x7b05c64b, 0x45dc1c87, 0x52e96f46, 0x562ffc29, 0x55a27ee1, 0x0b82e4e5, 0x5ef3f7cd, 0x3ab42017
89])
90
91// Precomputed masks per tier (controls cut-point frequency)
92// Below average: larger mask → fewer cuts → chunks grow toward average
93// Above average: smaller mask → more cuts → chunks stay near average
94const TIER_MASKS = TIERS.map(t => ({
95 large: (1 << Math.ceil(Math.log2(t.avgSize))) - 1,
96 small: (1 << Math.floor(Math.log2(t.avgSize))) - 1
97}))
98
99/**
100 * Chunk a buffer using FastCDC with adaptive tier selection
101 *
102 * @param {Buffer} data - Data to chunk
103 * @param {{ tier?: number }} [opts] - Options. tier: force a specific tier (0-3). If omitted, auto-selects based on data.length.
104 * @returns {Array<{ hash: string, offset: number, length: number, data: Buffer }>}
105 */
106export function chunk (data, opts) {
107 if (data.length === 0) return []
108
109 const tier = (opts && typeof opts.tier === 'number') ? opts.tier : selectTier(data.length)
110 const params = TIERS[tier] || TIERS[0]
111 const masks = TIER_MASKS[tier] || TIER_MASKS[0]
112
113 const chunks = []
114 let offset = 0
115
116 while (offset < data.length) {
117 const remaining = data.length - offset
118
119 // If remaining data is smaller than minimum, it's the final chunk
120 if (remaining <= params.minSize) {
121 const slice = data.subarray(offset, offset + remaining)
122 chunks.push({
123 hash: hashChunk(slice),
124 offset,
125 length: remaining,
126 data: slice
127 })
128 break
129 }
130
131 const maxLen = Math.min(remaining, params.maxSize)
132 let hash = 0
133 let i = params.minSize
134
135 // Phase 1: below average — use large mask (fewer cuts)
136 const avgBound = Math.min(params.avgSize, maxLen)
137 while (i < avgBound) {
138 hash = ((hash << 1) + GEAR[data[offset + i]]) >>> 0
139 if ((hash & masks.large) === 0) break
140 i++
141 }
142
143 // Phase 2: above average — use small mask (more cuts)
144 if (i >= avgBound) {
145 while (i < maxLen) {
146 hash = ((hash << 1) + GEAR[data[offset + i]]) >>> 0
147 if ((hash & masks.small) === 0) break
148 i++
149 }
150 }
151
152 const slice = data.subarray(offset, offset + i)
153 chunks.push({
154 hash: hashChunk(slice),
155 offset,
156 length: i,
157 data: slice
158 })
159 offset += i
160 }
161
162 return chunks
163}
164
165/**
166 * Hash a chunk using blake2b-256
167 * @param {Buffer} data
168 * @returns {string} hex hash
169 */
170export function hashChunk (data) {
171 return crypto.data(data).toString('hex')
172}
173
174// ── Streaming primitives ──────────────────────────────────────
175//
176// `chunk(buffer)` requires the entire file in memory and returns chunks
177// whose `.data` is a `Buffer.subarray` of the input — both keep the
178// original buffer pinned until every yielded chunk is GC'd. For
179// multi-GB files (or limited-RAM runtimes) use the streaming versions
180// below: `chunkStream` consumes a Readable and yields chunks one at a
181// time, releasing bytes after each yield.
182
183import sodium from 'sodium-universal'
184import c from 'compact-encoding'
185
186const LEAF_TYPE = Buffer.from([0])
187
188/**
189 * Streaming blake2b-256 hasher matching `hashChunk(buffer)`'s digest
190 * format (hypercore-crypto's `crypto.data` shape:
191 * blake2b(LEAF_TYPE || uint64-LE(size) || data)
192 * ). Lets you hash multi-GB files without buffering them.
193 *
194 * const h = createStreamingHasher({ size }) // size known up front
195 * for await (const buf of stream) h.update(buf)
196 * const oid = h.digest() // == hashChunk(file)
197 *
198 * `size` is required for hash compatibility with `hashChunk` — the
199 * leaf-prefix construction includes the byte length, which is fixed at
200 * stream open. If you don't have the size, omit `size` and you'll get
201 * a plain blake2b digest of the bytes (does NOT match hashChunk).
202 */
203export function createStreamingHasher (opts = {}) {
204 const state = Buffer.alloc(sodium.crypto_generichash_STATEBYTES)
205 let bytes = 0
206 if (opts && opts.restore) {
207 // The saved state already includes the init + leaf/size prefix + every byte
208 // consumed so far, so do not re-init.
209 opts.restore.state.copy(state)
210 bytes = opts.restore.bytes
211 } else {
212 sodium.crypto_generichash_init(state, null, 32)
213 const knownSize = (opts && typeof opts.size === 'number') ? opts.size : null
214 if (knownSize !== null) {
215 sodium.crypto_generichash_update(state, LEAF_TYPE)
216 sodium.crypto_generichash_update(state, c.encode(c.uint64, knownSize))
217 }
218 }
219 return {
220 update (buf) {
221 sodium.crypto_generichash_update(state, buf)
222 bytes += buf.length
223 },
224 digest () {
225 const out = Buffer.alloc(32)
226 sodium.crypto_generichash_final(state, out)
227 return out.toString('hex')
228 },
229 snapshot () { return { state: Buffer.from(state), bytes } },
230 get bytes () { return bytes }
231 }
232}
233
234// Find a FastCDC cut point inside `data` starting at offset 0. Returns
235// the chunk length. Mirrors the inner search loop of `chunk()` but
236// takes pre-resolved params + masks so it works in a streaming loop.
237function findCutPoint (data, params, masks) {
238 const remaining = data.length
239 if (remaining <= params.minSize) return remaining
240
241 const maxLen = Math.min(remaining, params.maxSize)
242 let hash = 0
243 let i = params.minSize
244
245 const avgBound = Math.min(params.avgSize, maxLen)
246 while (i < avgBound) {
247 hash = ((hash << 1) + GEAR[data[i]]) >>> 0
248 if ((hash & masks.large) === 0) return i
249 i++
250 }
251 if (i >= avgBound) {
252 while (i < maxLen) {
253 hash = ((hash << 1) + GEAR[data[i]]) >>> 0
254 if ((hash & masks.small) === 0) return i
255 i++
256 }
257 }
258 return maxLen
259}
260
261// Opt-in memcpy accounting for perf validation. Zero cost when off (one
262// boolean check at the increment sites). Module-level singletons — enabled
263// only by tests + bench/prepare.js, never in library code; NOT
264// concurrency-safe, so do not enable it in production.
265let STATS_ON = false
266export const chunkStats = { concatBytes: 0, copyBytes: 0, blocks: 0, chunks: 0 }
267export function resetChunkStats () { chunkStats.concatBytes = 0; chunkStats.copyBytes = 0; chunkStats.blocks = 0; chunkStats.chunks = 0 }
268export function setChunkStats (on) { STATS_ON = !!on; if (on) resetChunkStats() }
269
270/**
271 * Stream-friendly FastCDC chunker. Consumes an AsyncIterable<Buffer>
272 * (e.g. fs.createReadStream) and yields chunks one at a time. Memory
273 * footprint ≈ 2 × params.maxSize regardless of stream length —
274 * chunks are emitted as bytes arrive, the pending buffer is sliced
275 * past each cut point.
276 *
277 * Each yielded chunk has the same shape as `chunk()` entries:
278 * { hash, offset, length, data }
279 * With the default `copy: true`, `data` is a fresh `Buffer.from(slice)`
280 * detached from the input stream's pool, so callers can store/forward it.
281 * With `copy: false`, `data` is a view backed by the reader's memory —
282 * valid only until the next iteration; the caller MUST consume it
283 * synchronously and not retain it across an await.
284 *
285 * Cut-point determinism: same content + same tier → identical chunk
286 * boundaries as `chunk()`, regardless of input block size. Peers using
287 * either function on the same file produce matching chunk hashes.
288 *
289 * @param {AsyncIterable<Buffer>} readable
290 * @param {{ tier?: number, copy?: boolean }} [opts] tier index (0–3); copy
291 * defaults true. Callers with a known file size should pass `selectTier(size)`.
292 */
293export async function * chunkStream (readable, opts = {}) {
294 const tier = (opts && typeof opts.tier === 'number') ? opts.tier : 0
295 const copy = opts.copy !== false // false → yield views (see JSDoc above)
296 const params = TIERS[tier] || TIERS[0]
297 const masks = TIER_MASKS[tier] || TIER_MASKS[0]
298
299 let pending = Buffer.alloc(0)
300 let absoluteOffset = 0
301
302 // Single emit point for the stats accounting + copy:false decision.
303 const emit = (slice) => {
304 if (STATS_ON) { chunkStats.chunks++; if (copy) chunkStats.copyBytes += slice.length }
305 return { hash: hashChunk(slice), offset: absoluteOffset, length: slice.length, data: copy ? Buffer.from(slice) : slice }
306 }
307
308 for await (const inChunk of readable) {
309 const buf = Buffer.isBuffer(inChunk) ? inChunk : Buffer.from(inChunk)
310 if (STATS_ON) { chunkStats.blocks++; if (pending.length !== 0) chunkStats.concatBytes += pending.length + buf.length }
311 pending = pending.length === 0 ? buf : Buffer.concat([pending, buf])
312
313 // While we have at least maxSize ahead, we can definitively find
314 // the next cut point inside `pending` (the cut is bounded by maxSize).
315 while (pending.length >= params.maxSize) {
316 const cutLen = findCutPoint(pending, params, masks)
317 yield emit(pending.subarray(0, cutLen))
318 absoluteOffset += cutLen
319 pending = pending.subarray(cutLen)
320 }
321 }
322
323 // Stream ended. Process leftover. Same minSize-final-chunk rule as
324 // the buffer-mode `chunk()`.
325 while (pending.length > params.minSize) {
326 const cutLen = findCutPoint(pending, params, masks)
327 yield emit(pending.subarray(0, cutLen))
328 absoluteOffset += cutLen
329 pending = pending.subarray(cutLen)
330 }
331
332 if (pending.length > 0) yield emit(pending)
333}
lib/file-index.js +576 -2
……@@ -1,2 +1,576 @@
1// SHIM: v0.3.0 engine lives in lib/v3/ (Ok mirror). Re-export under our name/subpath.
2export * from './v3/file-index.js'
1/**
2 * FileIndex — Metadata-only file index backed by Hyperbee
3 *
4 * Stores file metadata and chunk maps. Never stores file content.
5 * The filesystem is the source of truth for actual bytes.
6 *
7 * Hyperbee schema:
8 * file:<path> → { contentHash, size, mtime, hashing, executable, linkname }
9 * chunkmap:<path> → [{ hash, offset, length }] (small) | { __paged: N } (large; see below)
10 * chunkmap-oid:<contentHash> → [{ hash, offset, length }] (content-addressed; reused across overlay keys + restarts)
11 * sync:<peerKey>:<path> → { lastSeq, lastHash }
12 * config:sync → { folders: [...] }
13 *
14 * tree:<hex-hash> → { entries: [{kind, exec, name, childHash, size}], size }
15 * treepath:<path> → <hex-hash> (secondary: path → current tree hash)
16 *
17 * Chunk-map paging. A chunk map for a very large file (a 1.1 TB
18 * file at tier 3 ≈ 1M entries ≈ ~120 MB of JSON) does not fit in one Hyperbee
19 * value: Hypercore caps a block at 15 MiB (MAX_SUGGESTED_BLOCK_SIZE) and throws
20 * BAD_ARGUMENT above it. Maps with > CHUNKS_PER_PAGE entries are stored as a
21 * header value { __paged: N } at the base key plus N page values keyed
22 * `<base>\x00<i>` (the NUL byte never occurs in a path or hex hash, so a page key
23 * can't collide with a real one). Small maps stay inline as a plain array —
24 * byte-identical to before, and back-compatible with already-stored maps. The
25 * public API (put/get/del) is unchanged; paging is transparent to callers.
26 */
27
28import Hyperbee from 'hyperbee'
29import ReadyResource from 'ready-resource'
30
31// Entries per chunk-map page. ~116 B/entry worst-case JSON ⇒
32// ≤ ~3.8 MB/page, well under Hypercore's 15 MiB block limit.
33const CHUNKS_PER_PAGE = 32768
34
35// Overlay-path prefix for host-registered, content-addressed serve entries.
36// Hosts that make a file servable purely by content hash should register it at
37// `${CONTENT_REGISTER_PREFIX}${contentHash}` so compact()/evictContent() can
38// recognize (and reclaim) the entry once the hash is no longer served.
39export const CONTENT_REGISTER_PREFIX = '/mir/'
40
41// The content hash an owner-side entry is addressed by (page suffix folded in), or
42// null for entries not keyed by a single content hash (real-path file:/chunkmap:,
43// tree:, sync:, config:). compact() drops the content-addressed entries whose hash is
44// no longer served, and keeps everything else.
45function contentHashOfKey (key) {
46 if (key.startsWith('chunkmap-oid:')) return key.slice('chunkmap-oid:'.length).split('\x00')[0]
47 if (key.startsWith('chunkmap:content:')) return key.slice('chunkmap:content:'.length).split('\x00')[0]
48 if (key.startsWith(`chunkmap:${CONTENT_REGISTER_PREFIX}`)) return key.slice(`chunkmap:${CONTENT_REGISTER_PREFIX}`.length).split('\x00')[0]
49 if (key.startsWith(`file:${CONTENT_REGISTER_PREFIX}`)) return key.slice(`file:${CONTENT_REGISTER_PREFIX}`.length)
50 return null
51}
52
53// v1 keeps the original 'file-index' name so existing stores aren't orphaned on
54// upgrade; compaction moves to file-index-v2, v3, … and purges the predecessor.
55export function indexCoreName (version) {
56 return version === 1 ? 'file-index' : `file-index-v${version}`
57}
58
59export class FileIndex extends ReadyResource {
60 constructor (store, opts = {}) {
61 super()
62 this._store = store
63 this._bee = null
64 this._meta = null
65 this._version = 1
66 this._opts = opts
67 }
68
69 // Local index cores are encrypted at rest when the caller passes an encryptionKey
70 // (opts.encryptionKey); absent, they stay plaintext.
71 _coreOpts (name) {
72 const o = { name, valueEncoding: 'binary' }
73 if (this._opts.encryptionKey) o.encryptionKey = this._opts.encryptionKey
74 return o
75 }
76
77 async _open () {
78 this._meta = new Hyperbee(this._store.get(this._coreOpts('index-meta')), {
79 keyEncoding: 'utf-8',
80 valueEncoding: 'json'
81 })
82 await this._meta.ready()
83 this._version = (await this._meta.get('version'))?.value ?? 1
84 // A prior purge can leave a version's core with a dangling name alias — its data and
85 // by-discovery-key alias were deleted but the by-name alias was not — which throws
86 // STORAGE_EMPTY on open. The index is rebuildable cache, so advance to a fresh core
87 // name and record it rather than failing open.
88 for (let attempts = 0; ; attempts++) {
89 const bee = new Hyperbee(this._store.get(this._coreOpts(indexCoreName(this._version))), {
90 keyEncoding: 'utf-8',
91 valueEncoding: 'json'
92 })
93 try {
94 await bee.ready()
95 this._bee = bee
96 return
97 } catch (err) {
98 if (err.code !== 'STORAGE_EMPTY' || attempts >= 64) throw err
99 await bee.close().catch(() => {})
100 this._version += 1
101 await this._meta.put('version', this._version)
102 }
103 }
104 }
105
106 async _close () {
107 if (this._bee) await this._bee.close()
108 if (this._meta) await this._meta.close()
109 }
110
111 get bee () { return this._bee }
112
113 get version () { return this._version }
114
115 get cores () { return [this._bee?.core, this._meta?.core].filter(Boolean) }
116
117 // Reclaim the append-only index: stream the still-served entries into a fresh
118 // versioned core, flip the version pointer, and return the old core so the caller
119 // can clear+purge it (the only way to return an append-only bee's disk to the OS).
120 // Content-addressed entries (maps + content-register entries) whose hash isServed() is false
121 // are dropped; real-path, tree, sync and config entries are always kept.
122 async compact ({ isServed }) {
123 // Skip the rewrite entirely when nothing is droppable — otherwise a compaction of
124 // an already-clean index just churns (a fresh version core + a version-marker
125 // append), which grows the index without reclaiming anything.
126 let droppable = false
127 for await (const { key } of this._bee.createReadStream()) {
128 const hash = contentHashOfKey(key)
129 if (hash !== null && !isServed(hash)) { droppable = true; break }
130 }
131 if (!droppable) return null
132
133 const next = this._version + 1
134 const dst = new Hyperbee(this._store.get(this._coreOpts(indexCoreName(next))), {
135 keyEncoding: 'utf-8',
136 valueEncoding: 'json'
137 })
138 await dst.ready()
139 let batch = dst.batch()
140 let pending = 0
141 for await (const { key, value } of this._bee.createReadStream()) {
142 const hash = contentHashOfKey(key)
143 if (hash !== null && !isServed(hash)) continue
144 await batch.put(key, value)
145 if (++pending >= 500) { await batch.flush(); batch = dst.batch(); pending = 0 }
146 }
147 await batch.flush()
148
149 const oldCore = this._bee.core
150 await this._meta.put('version', next)
151 this._bee = dst
152 this._version = next
153 return oldCore // left open; caller clears + purges it
154 }
155
156 // ── File entries ────────────────────────────────────────────
157
158 /**
159 * Store or update a file entry (metadata only, no content)
160 * @param {string} path - file path
161 * @param {{ contentHash: string|null, size: number, mtime: number, hashing?: boolean, executable?: boolean, linkname?: string|null }} meta
162 */
163 async putFile (path, meta) {
164 await this._bee.put(`file:${path}`, {
165 contentHash: meta.contentHash || null,
166 size: meta.size,
167 mtime: meta.mtime,
168 hashing: meta.hashing || false,
169 executable: meta.executable || false,
170 linkname: meta.linkname || null
171 })
172 }
173
174 /**
175 * Get file metadata
176 * @param {string} path
177 * @returns {{ contentHash: string|null, size: number, mtime: number, hashing: boolean, executable: boolean, linkname: string|null } | null}
178 */
179 async getFile (path) {
180 const entry = await this._bee.get(`file:${path}`)
181 return entry ? entry.value : null
182 }
183
184 /**
185 * Check if file exists in the index
186 * @param {string} path
187 * @returns {boolean}
188 */
189 async hasFile (path) {
190 const entry = await this._bee.get(`file:${path}`)
191 return entry !== null
192 }
193
194 /**
195 * Remove a file entry and its chunk map
196 * @param {string} path
197 */
198 async delFile (path) {
199 await this._bee.del(`file:${path}`)
200 await this._delPagedValue(`chunkmap:${path}`)
201 }
202
203 /**
204 * List files under a directory prefix
205 * @param {string} [dir=''] - directory prefix
206 * @returns {Array<{ path: string, contentHash: string|null, size: number, mtime: number }>}
207 */
208 async listFiles (dir = '') {
209 const prefix = dir ? `file:${dir}` : 'file:'
210 const files = []
211
212 for await (const entry of this._bee.createReadStream({ gt: prefix, lt: prefix + '\xff' })) {
213 files.push({
214 path: entry.key.slice('file:'.length),
215 ...entry.value
216 })
217 }
218
219 return files
220 }
221
222 // ── Index meta (from hyper-overlay 0.2.14; carried in v3) ───
223 // Small key/value bag on the main bee under `meta:` — used by the fast
224 // manifest-diff boot path (e.g. hyper-deploy). compact() keeps these
225 // (contentHashOfKey returns null for `meta:` → not droppable).
226
227 async putIndexMeta (key, value) {
228 await this._bee.put(`meta:${key}`, value)
229 }
230
231 async getIndexMeta (key) {
232 const entry = await this._bee.get(`meta:${key}`)
233 return entry ? entry.value : null
234 }
235
236 // ── Chunk maps ──────────────────────────────────────────────
237
238 /**
239 * Store a chunk map for a file (hash + offset + length per chunk, no data).
240 * Large maps are paged transparently (see the header note) so they never exceed the
241 * Hypercore block limit.
242 * @param {string} path - file path
243 * @param {Array<{ hash: string, offset: number, length: number }>} chunks
244 */
245 async putChunkMap (path, chunks) {
246 await this._putPagedValue(`chunkmap:${path}`, chunks)
247 }
248
249 /**
250 * Get the chunk map for a file
251 * @param {string} path
252 * @returns {Array<{ hash: string, offset: number, length: number }> | null}
253 */
254 async getChunkMap (path) {
255 return this._getPagedValue(`chunkmap:${path}`)
256 }
257
258 /**
259 * Check if a chunk map exists for a file
260 * @param {string} path
261 * @returns {boolean}
262 */
263 async hasChunkMap (path) {
264 const entry = await this._bee.get(`chunkmap:${path}`)
265 return entry !== null
266 }
267
268 /**
269 * Remove a chunk map
270 * @param {string} path
271 */
272 async delChunkMap (path) {
273 await this._delPagedValue(`chunkmap:${path}`)
274 }
275
276 // Paged-value storage. Small maps stay inline as a plain array
277 // (unchanged on disk, back-compatible); maps over CHUNKS_PER_PAGE entries are
278 // split into a { __paged: N } header + N `<base>\x00<i>` page values, written
279 // in one Hyperbee batch so the whole map commits atomically. Stale pages from
280 // a previously-larger map at the same key are deleted in the same batch.
281 async _putPagedValue (baseKey, chunks) {
282 const prevPages = await this._pagedCount(baseKey)
283
284 if (chunks.length <= CHUNKS_PER_PAGE) {
285 if (prevPages === 0) {
286 await this._bee.put(baseKey, chunks)
287 return
288 }
289 await this._batched(async (b) => {
290 await b.put(baseKey, chunks)
291 for (let i = 0; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`)
292 })
293 return
294 }
295
296 const pageCount = Math.ceil(chunks.length / CHUNKS_PER_PAGE)
297 await this._batched(async (b) => {
298 await b.put(baseKey, { __paged: pageCount })
299 for (let i = 0; i < pageCount; i++) {
300 await b.put(`${baseKey}\x00${i}`, chunks.slice(i * CHUNKS_PER_PAGE, (i + 1) * CHUNKS_PER_PAGE))
301 }
302 for (let i = pageCount; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`)
303 })
304 }
305
306 async _getPagedValue (baseKey) {
307 const entry = await this._bee.get(baseKey)
308 if (!entry) return null
309 if (Array.isArray(entry.value)) return entry.value // inline / legacy
310 const pageCount = entry.value.__paged
311 if (!pageCount) return null
312 const chunks = []
313 for (let i = 0; i < pageCount; i++) {
314 const page = await this._bee.get(`${baseKey}\x00${i}`)
315 // The header + all pages commit in one atomic batch, so a missing or non-array
316 // page means the value is corrupt/incomplete. Return null — a clean "miss" so the
317 // caller re-chunks from the source file — never a silently truncated chunk map.
318 if (!page || !Array.isArray(page.value)) return null
319 for (const c of page.value) chunks.push(c)
320 }
321 return chunks
322 }
323
324 async _delPagedValue (baseKey) {
325 const prevPages = await this._pagedCount(baseKey)
326 if (prevPages === 0) {
327 await this._bee.del(baseKey)
328 return
329 }
330 await this._batched(async (b) => {
331 await b.del(baseKey)
332 for (let i = 0; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`)
333 })
334 }
335
336 // Page count of an existing paged value, or 0 if the key is absent or inline.
337 async _pagedCount (baseKey) {
338 const entry = await this._bee.get(baseKey)
339 if (!entry || Array.isArray(entry.value)) return 0
340 return entry.value.__paged || 0
341 }
342
343 // Run ops in a Hyperbee batch, closing it on error (flush releases the lock
344 // itself, even on append failure, via _appendBatch's finally).
345 async _batched (fn) {
346 const b = this._bee.batch()
347 try {
348 await fn(b)
349 await b.flush()
350 } catch (err) {
351 try { await b.close() } catch {}
352 throw err
353 }
354 }
355
356 // Content-addressed chunk map. FastCDC is deterministic on bytes, so a
357 // given content hash always chunks identically regardless of which overlay path
358 // it was registered under. Keying by content hash lets the publish-time chunking
359 // be reused at serve time (a synthetic 'content:<hash>' key) and across restarts,
360 // instead of re-reading the whole file to re-chunk it before the first byte ships.
361 async putChunkMapByHash (contentHash, chunks) {
362 await this._putPagedValue(`chunkmap-oid:${contentHash}`, chunks)
363 }
364
365 async getChunkMapByHash (contentHash) {
366 return this._getPagedValue(`chunkmap-oid:${contentHash}`)
367 }
368
369 async hasChunkMapByHash (contentHash) {
370 return (await this._bee.get(`chunkmap-oid:${contentHash}`)) !== null
371 }
372
373 async delChunkMapByHash (contentHash) {
374 await this._delPagedValue(`chunkmap-oid:${contentHash}`)
375 }
376
377 // Evict every durable entry a served content hash leaves behind: the content-
378 // addressed map plus the deterministic owner-side synthetic paths (the serve
379 // copy at content:<hash> and the register entry under CONTENT_REGISTER_PREFIX).
380 // Peer-path-keyed receiver entries are unknowable from the hash — the host's
381 // own index sweep handles those.
382 async evictContent (contentHash) {
383 await this.delChunkMapByHash(contentHash)
384 await this.delChunkMap(`content:${contentHash}`)
385 await this.delFile(`${CONTENT_REGISTER_PREFIX}${contentHash}`)
386 }
387
388 // ── Sync state ──────────────────────────────────────────────
389
390 /**
391 * Store sync state for a peer + file
392 * @param {string} peerKey - hex peer key
393 * @param {string} path - file path
394 * @param {{ lastSeq: number, lastHash: string }} state
395 */
396 async putSyncState (peerKey, path, state) {
397 await this._bee.put(`sync:${peerKey}:${path}`, {
398 lastSeq: state.lastSeq,
399 lastHash: state.lastHash
400 })
401 }
402
403 /**
404 * Get sync state for a peer + file
405 * @param {string} peerKey
406 * @param {string} path
407 * @returns {{ lastSeq: number, lastHash: string } | null}
408 */
409 async getSyncState (peerKey, path) {
410 const entry = await this._bee.get(`sync:${peerKey}:${path}`)
411 return entry ? entry.value : null
412 }
413
414 /**
415 * Remove sync state for a peer + file
416 * @param {string} peerKey
417 * @param {string} path
418 */
419 async delSyncState (peerKey, path) {
420 await this._bee.del(`sync:${peerKey}:${path}`)
421 }
422
423 /**
424 * List all sync states for a peer
425 * @param {string} peerKey
426 * @returns {Array<{ path: string, lastSeq: number, lastHash: string }>}
427 */
428 async listSyncStates (peerKey) {
429 const prefix = `sync:${peerKey}:`
430 const states = []
431
432 for await (const entry of this._bee.createReadStream({ gt: prefix, lt: prefix + '\xff' })) {
433 states.push({
434 path: entry.key.slice(prefix.length),
435 ...entry.value
436 })
437 }
438
439 return states
440 }
441
442 // ── Sync config ─────────────────────────────────────────────
443
444 /**
445 * Store sync configuration
446 * @param {{ folders: Array<{ path: string, strategy: string, quota?: string|null, peers?: string[] }> }} config
447 */
448 async putSyncConfig (config) {
449 await this._bee.put('config:sync', config)
450 }
451
452 /**
453 * Get sync configuration
454 * @returns {{ folders: Array } | null}
455 */
456 async getSyncConfig () {
457 const entry = await this._bee.get('config:sync')
458 return entry ? entry.value : null
459 }
460
461 // ── Trees ───────────────────────────────────────────────────
462
463 /**
464 * Store a tree by its content-addressed hash.
465 * Same hash put twice is a no-op (content-addressed — bytes can't differ).
466 *
467 * @param {string} hash - hex-encoded blake2b-256 tree hash
468 * @param {{ entries: Array, size: number }} tree
469 */
470 async putTree (hash, tree) {
471 await this._bee.put(`tree:${hash}`, {
472 entries: tree.entries,
473 size: tree.size
474 })
475 }
476
477 /**
478 * Get a tree by its hash.
479 * @param {string} hash
480 * @returns {{ entries: Array, size: number } | null}
481 */
482 async getTree (hash) {
483 const entry = await this._bee.get(`tree:${hash}`)
484 return entry ? entry.value : null
485 }
486
487 /**
488 * Check if a tree is present.
489 * @param {string} hash
490 * @returns {boolean}
491 */
492 async hasTree (hash) {
493 const entry = await this._bee.get(`tree:${hash}`)
494 return entry !== null
495 }
496
497 /**
498 * Remove a tree. Only safe when no treepath entries reference this hash.
499 * @param {string} hash
500 */
501 async delTree (hash) {
502 await this._bee.del(`tree:${hash}`)
503 }
504
505 /**
506 * Record the current tree hash for a path (secondary index).
507 * @param {string} path
508 * @param {string} hash
509 */
510 async putTreePath (path, hash) {
511 await this._bee.put(`treepath:${path}`, hash)
512 }
513
514 /**
515 * Look up the current tree hash for a path.
516 * @param {string} path
517 * @returns {string|null}
518 */
519 async getTreePath (path) {
520 const entry = await this._bee.get(`treepath:${path}`)
521 return entry ? entry.value : null
522 }
523
524 /**
525 * Remove the treepath entry. Does NOT delete the underlying tree — other
526 * paths may still reference the same hash.
527 * @param {string} path
528 */
529 async delTreePath (path) {
530 await this._bee.del(`treepath:${path}`)
531 }
532
533 /**
534 * List all stored tree hashes.
535 * @param {object} [opts]
536 * @param {number} [opts.limit] - optional cap
537 * @returns {Array<{ hash: string, entryCount: number, size: number }>}
538 */
539 async listTrees (opts = {}) {
540 const trees = []
541 for await (const entry of this._bee.createReadStream({ gt: 'tree:', lt: 'tree:\xff' })) {
542 trees.push({
543 hash: entry.key.slice('tree:'.length),
544 entryCount: entry.value.entries.length,
545 size: entry.value.size
546 })
547 if (opts.limit && trees.length >= opts.limit) break
548 }
549 return trees
550 }
551
552 // ── Stats ───────────────────────────────────────────────────
553
554 /**
555 * Get index statistics
556 * @returns {{ fileCount: number, chunkMapCount: number, syncStateCount: number, treeCount: number, treePathCount: number }}
557 */
558 async stats () {
559 let fileCount = 0
560 let chunkMapCount = 0
561 let syncStateCount = 0
562 let treeCount = 0
563 let treePathCount = 0
564
565 for await (const entry of this._bee.createReadStream()) {
566 if (entry.key.startsWith('file:')) fileCount++
567 // Count one per file — skip the `\x00`-suffixed page values.
568 else if (entry.key.startsWith('chunkmap:') && !entry.key.includes('\x00')) chunkMapCount++
569 else if (entry.key.startsWith('sync:')) syncStateCount++
570 else if (entry.key.startsWith('tree:')) treeCount++
571 else if (entry.key.startsWith('treepath:')) treePathCount++
572 }
573
574 return { fileCount, chunkMapCount, syncStateCount, treeCount, treePathCount }
575 }
576}
lib/fuse.js +0 -202
……@@ -1,202 +0,0 @@
1/**
2 * FUSE Mount — expose overlay as a native filesystem
3 *
4 * Uses fuse-native (N-API, prebuilds for Linux).
5 * Maps FUSE operations to HyperOverlay adapter interface.
6 *
7 * Usage:
8 * import { mount } from 'hyper-overlay/fuse'
9 * const handle = await mount(overlay, '/mnt/overlay', { readonly: true })
10 * // ... files visible in /mnt/overlay/ ...
11 * await handle.unmount()
12 */
13
14// fuse-native is a Node.js N-API addon — does not work on Bare runtime.
15// This module loads lazily so importing hyper-overlay/fuse doesn't crash on Bare.
16let Fuse = null
17try {
18 const mod = await import('fuse-native')
19 Fuse = mod.default || mod
20} catch {
21 // fuse-native not available (Bare runtime or not installed)
22}
23
24const DIR_MODE = 0o40755
25const FILE_MODE = 0o100644
26const FILE_MODE_RW = 0o100666
27
28/**
29 * Mount an overlay at a local path
30 *
31 * @param {HyperOverlay} overlay
32 * @param {string} mountpoint - local filesystem path
33 * @param {{ readonly?: boolean, mkdir?: boolean }} [opts]
34 * @returns {Promise<{ unmount: () => Promise<void> }>}
35 */
36export async function mount (overlay, mountpoint, opts = {}) {
37 if (!Fuse) throw new Error('FUSE not available — fuse-native requires Node.js (not Bare runtime)')
38 const readonly = opts.readonly !== false
39 const uid = typeof process.getuid === 'function' ? process.getuid() : 0
40 const gid = typeof process.getgid === 'function' ? process.getgid() : 0
41 const writeBuffers = new Map() // fd → { path, parts }
42 let nextFd = 100
43
44 function makeStat (entry) {
45 const isDir = entry.type === 'dir'
46 return {
47 mtime: new Date(entry.mtime || Date.now()),
48 atime: new Date(entry.mtime || Date.now()),
49 ctime: new Date(entry.mtime || Date.now()),
50 nlink: isDir ? 2 : 1,
51 size: entry.size || 0,
52 mode: isDir ? DIR_MODE : (readonly ? FILE_MODE : FILE_MODE_RW),
53 uid,
54 gid
55 }
56 }
57
58 const ops = {
59 getattr (path, cb) {
60 overlay.stat(path).then(entry => {
61 if (!entry) return cb(Fuse.ENOENT)
62 cb(0, makeStat(entry))
63 }).catch(() => cb(Fuse.EIO))
64 },
65
66 readdir (path, cb) {
67 overlay.readdir(path).then(names => {
68 cb(0, names)
69 }).catch(() => cb(Fuse.EIO))
70 },
71
72 open (path, flags, cb) {
73 overlay.stat(path).then(entry => {
74 if (!entry || entry.type === 'dir') return cb(Fuse.ENOENT)
75 const fd = nextFd++
76 cb(0, fd)
77 }).catch(() => cb(Fuse.EIO))
78 },
79
80 read (path, fd, buf, len, pos, cb) {
81 overlay.read(path, pos, len).then(data => {
82 if (!data) return cb(0)
83 data.copy(buf)
84 cb(data.length)
85 }).catch(() => cb(Fuse.EIO))
86 },
87
88 release (path, fd, cb) {
89 // Finalize buffered write if any
90 const wb = writeBuffers.get(fd)
91 if (wb) {
92 const data = Buffer.concat(wb.parts)
93 writeBuffers.delete(fd)
94 overlay.put(wb.path, data).then(() => cb(0)).catch(() => cb(Fuse.EIO))
95 return
96 }
97 cb(0)
98 },
99
100 statfs (path, cb) {
101 overlay.stats().then(s => {
102 cb(0, {
103 bsize: 16384,
104 frsize: 16384,
105 blocks: Math.max(s.totalSize / 16384, 1) | 0,
106 bfree: 1000000,
107 bavail: 1000000,
108 files: s.chunkCount + s.manifestCount,
109 ffree: 1000000,
110 favail: 1000000,
111 fsid: 0,
112 flag: readonly ? 1 : 0,
113 namemax: 255
114 })
115 }).catch(() => cb(Fuse.EIO))
116 }
117 }
118
119 // Write operations (only if not readonly)
120 if (!readonly) {
121 ops.create = function (path, mode, cb) {
122 const fd = nextFd++
123 writeBuffers.set(fd, { path, parts: [], size: 0 })
124 // Create an empty entry so getattr works immediately
125 overlay.put(path, Buffer.alloc(0)).then(() => cb(0, fd)).catch(() => cb(Fuse.EIO))
126 }
127
128 ops.write = function (path, fd, buf, len, pos, cb) {
129 const wb = writeBuffers.get(fd)
130 if (!wb) return cb(Fuse.EIO)
131 // Store a copy of the buffer slice (fuse-native reuses the buffer)
132 wb.parts.push({ data: Buffer.from(buf.subarray(0, len)), pos })
133 wb.size = Math.max(wb.size, pos + len)
134 cb(len)
135 }
136
137 ops.truncate = function (path, size, cb) {
138 if (size === 0) {
139 overlay.put(path, Buffer.alloc(0)).then(() => cb(0)).catch(() => cb(Fuse.EIO))
140 return
141 }
142 overlay.read(path, 0, size).then(data => {
143 if (!data) return cb(Fuse.ENOENT)
144 return overlay.put(path, data).then(() => cb(0))
145 }).catch(() => cb(Fuse.EIO))
146 }
147
148 ops.unlink = function (path, cb) {
149 overlay.del(path).then(() => cb(0)).catch(() => cb(Fuse.EIO))
150 }
151
152 ops.mkdir = function (path, mode, cb) {
153 overlay.put(path + '/.keep', Buffer.alloc(0)).then(() => cb(0)).catch(() => cb(Fuse.EIO))
154 }
155
156 ops.rmdir = function (path, cb) {
157 overlay.del(path + '/.keep').then(() => cb(0)).catch(() => cb(Fuse.EIO))
158 }
159
160 // Override release to assemble scattered writes
161 const origRelease = ops.release
162 ops.release = function (path, fd, cb) {
163 const wb = writeBuffers.get(fd)
164 if (wb && wb.parts.length > 0) {
165 // Assemble the final buffer from positional writes
166 const buf = Buffer.alloc(wb.size)
167 for (const p of wb.parts) p.data.copy(buf, p.pos)
168 writeBuffers.delete(fd)
169 overlay.put(wb.path, buf).then(() => cb(0)).catch(() => cb(Fuse.EIO))
170 return
171 }
172 if (wb) writeBuffers.delete(fd)
173 cb(0)
174 }
175 }
176
177 const fuse = new Fuse(mountpoint, ops, {
178 force: true,
179 mkdir: opts.mkdir !== false,
180 autoUnmount: true
181 })
182
183 await new Promise((resolve, reject) => {
184 fuse.mount(err => {
185 if (err) return reject(err)
186 resolve()
187 })
188 })
189
190 return {
191 fuse,
192 mountpoint,
193 async unmount () {
194 await new Promise((resolve, reject) => {
195 fuse.unmount(err => {
196 if (err) return reject(err)
197 resolve()
198 })
199 })
200 }
201 }
202}
lib/http.js +0 -168
……@@ -1,168 +0,0 @@
1/**
2 * HTTP Chunk Endpoint — fallback for HTTPS clones
3 *
4 * Serves chunks and manifests over HTTP when Hyperswarm is unavailable.
5 * Immutable content-addressed data = aggressive caching.
6 *
7 * Routes:
8 * GET {prefix}/chunks/{hash} → raw chunk bytes
9 * GET {prefix}/manifests/{hash} → compressed manifest
10 * GET {prefix}/status → JSON stats
11 * POST {prefix}/upload[/path] → disk-spool + putFile (requires opts.uploadSpoolDir)
12 */
13
14import fs from '#fs'
15import path from '#path'
16import crypto from 'hypercore-crypto'
17import { pipelinePromise } from 'streamx'
18import { createStreamingHasher } from './chunker.js'
19
20/**
21 * Mount overlay HTTP endpoints on a server
22 *
23 * @param {ChunkRegistry} registry
24 * @param {Object} httpServer - bare-http1 or Node http server
25 * @param {string} prefix - URL prefix (e.g., '/overlay')
26 * @param {HyperOverlay} [overlay] - full overlay instance (needed for upload)
27 * @param {Object} [opts]
28 * @param {string} [opts.uploadSpoolDir] - persistent dir for upload spool
29 * files. Required if uploads are to be served. Bytes live here forever
30 * (chunk-registry stores type:'local' pointers into this dir).
31 */
32export function serveOverlay (registry, httpServer, prefix = '/overlay', overlay = null, opts = {}) {
33 const routes = {
34 chunks: `${prefix}/chunks/`,
35 manifests: `${prefix}/manifests/`,
36 status: `${prefix}/status`,
37 upload: `${prefix}/upload`
38 }
39 const spoolDir = opts.uploadSpoolDir || null
40
41 httpServer.on('request', async (req, res) => {
42 const url = req.url
43
44 // ── Upload (disk-spool + putFile) ──────────────────────
45 // Spools the request body to a persistent dir, hashes it, renames
46 // to <spoolDir>/<oid>, then registers chunks as type:'local'
47 // pointers. RAM stays bounded at one chunk; corestore growth is
48 // metadata-only (~30 KB regardless of input size).
49 //
50 // The spool dir IS the canonical storage for the bytes — chunk
51 // pointers reference it, so files cannot be deleted until a
52 // future GC pass walks file: entries and unlinks unreferenced
53 // oids.
54 if (req.method === 'POST' && url.startsWith(routes.upload) && overlay) {
55 if (!spoolDir) {
56 res.writeHead(503, { 'Content-Type': 'application/json' })
57 res.end(JSON.stringify({ error: 'uploads disabled — serveOverlay() requires opts.uploadSpoolDir' }))
58 return
59 }
60
61 const filePath = url.slice(routes.upload.length) || '/' + Date.now()
62 const tmpPath = path.join(spoolDir, '.tmp-' + crypto.randomBytes(8).toString('hex'))
63 let cleanupTmp = true
64 try {
65 await pipelinePromise(req, fs.createWriteStream(tmpPath))
66 const size = fs.statSync(tmpPath).size
67 if (size === 0) {
68 res.writeHead(400, { 'Content-Type': 'application/json' })
69 res.end(JSON.stringify({ error: 'Empty body' }))
70 return
71 }
72
73 const hasher = createStreamingHasher({ size })
74 for await (const buf of fs.createReadStream(tmpPath)) hasher.update(buf)
75 const oid = hasher.digest()
76
77 const finalPath = path.join(spoolDir, oid)
78 if (fs.existsSync(finalPath)) {
79 fs.unlinkSync(tmpPath)
80 } else {
81 fs.renameSync(tmpPath, finalPath)
82 }
83 cleanupTmp = false
84
85 const result = await overlay.putFile(filePath, finalPath)
86 res.writeHead(200, { 'Content-Type': 'application/json' })
87 res.end(JSON.stringify({
88 oid: result.oid,
89 size: result.size,
90 chunkCount: result.chunkCount,
91 path: filePath,
92 pointer: result.pointer
93 }))
94 } catch (err) {
95 res.writeHead(500, { 'Content-Type': 'application/json' })
96 res.end(JSON.stringify({ error: 'Upload failed' }))
97 } finally {
98 if (cleanupTmp) try { fs.unlinkSync(tmpPath) } catch {}
99 }
100 return
101 }
102
103 // ── GET /chunks/:hash ───────────────────────────────────
104 if (url.startsWith(routes.chunks)) {
105 const hash = url.slice(routes.chunks.length).split('?')[0]
106 if (!hash || !/^[0-9a-f]{64}$/.test(hash)) {
107 res.writeHead(400, { 'Content-Type': 'text/plain' })
108 res.end('Invalid chunk hash')
109 return
110 }
111
112 // getChunkData handles type:'cache' (inline base64), type:'local'
113 // (file path + offset), and returns null otherwise. Returning null
114 // here lets us 404 cleanly without inlining source-shape logic.
115 const data = await registry.getChunkData(hash)
116 if (!data) {
117 res.writeHead(404, { 'Content-Type': 'text/plain' })
118 res.end('Chunk not found')
119 return
120 }
121
122 res.writeHead(200, {
123 'Content-Type': 'application/octet-stream',
124 'ETag': `"${hash}"`,
125 'Cache-Control': 'public, immutable, max-age=31536000'
126 })
127 res.end(data)
128 return
129 }
130
131 // ── GET /manifests/:hash ────────────────────────────────
132 if (url.startsWith(routes.manifests)) {
133 const hash = url.slice(routes.manifests.length).split('?')[0]
134 if (!hash || !/^[0-9a-f]{64}$/.test(hash)) {
135 res.writeHead(400, { 'Content-Type': 'text/plain' })
136 res.end('Invalid manifest hash')
137 return
138 }
139
140 const data = await registry.getManifest(hash)
141 if (!data) {
142 res.writeHead(404, { 'Content-Type': 'text/plain' })
143 res.end('Manifest not found')
144 return
145 }
146
147 res.writeHead(200, {
148 'Content-Type': 'application/octet-stream',
149 'ETag': `"${hash}"`,
150 'Cache-Control': 'public, immutable, max-age=31536000'
151 })
152 res.end(data)
153 return
154 }
155
156 // ── GET /status ─────────────────────────────────────────
157 if (url === routes.status || url === routes.status + '/') {
158 const stats = await registry.stats()
159 const json = JSON.stringify(stats)
160
161 res.writeHead(200, { 'Content-Type': 'application/json' })
162 res.end(json)
163 return
164 }
165
166 // Not an overlay route — pass through
167 })
168}
lib/messages-v2.js +0 -2
……@@ -1,2 +0,0 @@
1// SHIM: v0.3.0 engine lives in lib/v3/ (Ok mirror). Re-export under our name/subpath.
2export * from './v3/messages.js'
lib/messages.js +334 -114
……@@ -1,224 +1,444 @@
11/**
2 * Wire Protocol Messages for hyper-overlay/alpha
2 * Wire Protocol Messages for hyper-overlay/v2
33 *
4 * 9 message types with compact-encoding codecs.
5 * Each has preencode/encode/decode methods.
4 * 13 message types for sync + transfer + trees (0.5a extension):
5 * 0-2: Sync messages (state exchange, file offers/requests)
6 * 3-8: Transfer messages (chunk hashes, data, completion)
7 * 9-11: Tree messages (tree-request, tree-response, content-request)
8 * 12: transfer-control (downloader→holder pause/stop notice)
9 *
10 * Messages 9-12 are backward-compatible additions — peers without
11 * this code simply don't register the corresponding protomux slots, and
12 * senders see no receiver for those messages. File-level sync (0-8)
13 * keeps working unchanged.
14 *
15 * All use compact-encoding codecs with preencode/encode/decode.
616 */
717
818import c from 'compact-encoding'
919
10// Message 0: Handshake (exchanged on channel open)
20// ── Helpers ───────────────────────────────────────────────────
21
22function toBuffer32 (hashOrBuf) {
23 if (Buffer.isBuffer(hashOrBuf)) return hashOrBuf
24 return Buffer.from(hashOrBuf, 'hex')
25}
26
27// Encode/decode an array of { hash (32-byte hex), length (uint) }
28const chunkInfoArray = {
29 preencode (state, arr) {
30 c.uint32.preencode(state, arr.length)
31 for (const item of arr) {
32 state.end += 32
33 c.uint.preencode(state, item.length)
34 }
35 },
36 encode (state, arr) {
37 c.uint32.encode(state, arr.length)
38 for (const item of arr) {
39 toBuffer32(item.hash).copy(state.buffer, state.start, 0, 32)
40 state.start += 32
41 c.uint.encode(state, item.length)
42 }
43 },
44 decode (state) {
45 const len = c.uint32.decode(state)
46 const arr = []
47 for (let i = 0; i < len; i++) {
48 const hash = state.buffer.subarray(state.start, state.start + 32).toString('hex')
49 state.start += 32
50 const length = c.uint.decode(state)
51 arr.push({ hash, length })
52 }
53 return arr
54 }
55}
56
57// Encode/decode an array of uints
58const uintArray = {
59 preencode (state, arr) {
60 c.uint32.preencode(state, arr.length)
61 for (const v of arr) c.uint.preencode(state, v)
62 },
63 encode (state, arr) {
64 c.uint32.encode(state, arr.length)
65 for (const v of arr) c.uint.encode(state, v)
66 },
67 decode (state) {
68 const len = c.uint32.decode(state)
69 const arr = []
70 for (let i = 0; i < len; i++) arr.push(c.uint.decode(state))
71 return arr
72 }
73}
74
75// ── Handshake ─────────────────────────────────────────────────
76
1177export const handshake = {
1278 preencode (state, m) {
1379 c.uint8.preencode(state, m.version)
14 c.uint32.preencode(state, m.manifestCount)
1580 c.uint8.preencode(state, m.capabilities)
1681 },
1782 encode (state, m) {
1883 c.uint8.encode(state, m.version)
19 c.uint32.encode(state, m.manifestCount)
2084 c.uint8.encode(state, m.capabilities)
2185 },
2286 decode (state) {
2387 return {
2488 version: c.uint8.decode(state),
25 manifestCount: c.uint32.decode(state),
2689 capabilities: c.uint8.decode(state)
2790 }
2891 }
2992}
3093
31// Helper: array of fixed-size 32-byte hashes
32const hashArray = {
33 preencode (state, arr) {
34 c.uint32.preencode(state, arr.length)
35 state.end += arr.length * 32
94// ── Sync Messages (0-2) ──────────────────────────────────────
95
96// Message 0: sync-state — exchange last-synced sequence numbers
97export const syncState = {
98 preencode (state, m) {
99 c.buffer.preencode(state, toBuffer32(m.feedKey))
100 c.uint.preencode(state, m.localSeq)
101 c.uint.preencode(state, m.remoteSeq)
36102 },
37 encode (state, arr) {
38 c.uint32.encode(state, arr.length)
39 for (const hash of arr) {
40 const buf = typeof hash === 'string' ? Buffer.from(hash, 'hex') : hash
41 buf.copy(state.buffer, state.start, 0, 32)
42 state.start += 32
43 }
103 encode (state, m) {
104 c.buffer.encode(state, toBuffer32(m.feedKey))
105 c.uint.encode(state, m.localSeq)
106 c.uint.encode(state, m.remoteSeq)
44107 },
45108 decode (state) {
46 const len = c.uint32.decode(state)
47 const arr = []
48 for (let i = 0; i < len; i++) {
49 arr.push(state.buffer.subarray(state.start, state.start + 32).toString('hex'))
50 state.start += 32
109 return {
110 feedKey: c.buffer.decode(state).toString('hex'),
111 localSeq: c.uint.decode(state),
112 remoteSeq: c.uint.decode(state)
51113 }
52 return arr
53114 }
54115}
55116
56// Message 1: manifest-have (announce available manifests)
57export const manifestHave = {
58 preencode (state, m) { hashArray.preencode(state, m.hashes) },
59 encode (state, m) { hashArray.encode(state, m.hashes) },
60 decode (state) { return { hashes: hashArray.decode(state) } }
61}
62
63// Message 2: manifest-request
64export const manifestRequest = {
117// Message 1: file-offer — announce a file change
118export const fileOffer = {
65119 preencode (state, m) {
66 c.buffer.preencode(state, toBuffer32(m.hash))
67 c.uint.preencode(state, m.requestId)
120 c.string.preencode(state, m.path)
121 c.buffer.preencode(state, toBuffer32(m.contentHash))
122 c.uint.preencode(state, m.size)
123 c.uint.preencode(state, m.mtime)
124 c.uint8.preencode(state, m.op)
68125 },
69126 encode (state, m) {
70 c.buffer.encode(state, toBuffer32(m.hash))
71 c.uint.encode(state, m.requestId)
127 c.string.encode(state, m.path)
128 c.buffer.encode(state, toBuffer32(m.contentHash))
129 c.uint.encode(state, m.size)
130 c.uint.encode(state, m.mtime)
131 c.uint8.encode(state, m.op)
72132 },
73133 decode (state) {
74134 return {
75 hash: c.buffer.decode(state).toString('hex'),
76 requestId: c.uint.decode(state)
135 path: c.string.decode(state),
136 contentHash: c.buffer.decode(state).toString('hex'),
137 size: c.uint.decode(state),
138 mtime: c.uint.decode(state),
139 op: c.uint8.decode(state)
77140 }
78141 }
79142}
80143
81// Message 3: manifest-response
82export const manifestResponse = {
144// Message 2: file-request — request a file transfer
145export const fileRequest = {
83146 preencode (state, m) {
84 c.buffer.preencode(state, toBuffer32(m.hash))
85 c.uint.preencode(state, m.requestId)
86 c.buffer.preencode(state, m.data || Buffer.alloc(0))
87 c.bool.preencode(state, m.found)
147 c.string.preencode(state, m.path)
148 c.buffer.preencode(state, toBuffer32(m.contentHash))
149 c.buffer.preencode(state, m.chunksHave || Buffer.alloc(0))
88150 },
89151 encode (state, m) {
90 c.buffer.encode(state, toBuffer32(m.hash))
91 c.uint.encode(state, m.requestId)
92 c.buffer.encode(state, m.data || Buffer.alloc(0))
93 c.bool.encode(state, m.found)
152 c.string.encode(state, m.path)
153 c.buffer.encode(state, toBuffer32(m.contentHash))
154 c.buffer.encode(state, m.chunksHave || Buffer.alloc(0))
94155 },
95156 decode (state) {
96157 return {
97 hash: c.buffer.decode(state).toString('hex'),
98 requestId: c.uint.decode(state),
99 data: c.buffer.decode(state),
100 found: c.bool.decode(state)
158 path: c.string.decode(state),
159 contentHash: c.buffer.decode(state).toString('hex'),
160 chunksHave: c.buffer.decode(state)
101161 }
102162 }
103163}
104164
105// Message 4: chunk-bitfield (per-manifest availability)
106export const chunkBitfield = {
165// ── Transfer Messages (3-8) ──────────────────────────────────
166
167// Message 3: chunk-hashes — chunk hash list for a requested file
168//
169// `more` (paging flag) is appended last. A very large file's
170// chunk list serializes past the 16 MiB-1 Noise frame limit
171// (@hyperswarm/secret-stream MAX_ATOMIC_WRITE), so the protocol splits it into
172// several chunkHashes frames: every page but the last sets more:1, the final
173// page sets more:0. The receiver concatenates pages (in arrival order, keyed by
174// path) before dispatching. Appended last so a pre-paging peer that omits it
175// decodes more:0 — i.e. as a single, complete page (back-compatible).
176export const chunkHashes = {
107177 preencode (state, m) {
108 c.buffer.preencode(state, toBuffer32(m.manifestHash))
109 c.buffer.preencode(state, m.bitfield)
178 c.string.preencode(state, m.path)
179 c.uint8.preencode(state, m.tier)
180 chunkInfoArray.preencode(state, m.chunks)
181 c.uint8.preencode(state, m.more || 0)
110182 },
111183 encode (state, m) {
112 c.buffer.encode(state, toBuffer32(m.manifestHash))
113 c.buffer.encode(state, m.bitfield)
184 c.string.encode(state, m.path)
185 c.uint8.encode(state, m.tier)
186 chunkInfoArray.encode(state, m.chunks)
187 c.uint8.encode(state, m.more || 0)
188 },
189 decode (state) {
190 const path = c.string.decode(state)
191 const tier = c.uint8.decode(state)
192 const chunks = chunkInfoArray.decode(state)
193 // A frame from a pre-paging peer has no trailing byte → more:0.
194 const more = state.start < state.end ? c.uint8.decode(state) : 0
195 return { path, tier, chunks, more }
196 }
197}
198
199// Message 4: chunk-need — tell sender which chunks to send
200export const chunkNeed = {
201 preencode (state, m) {
202 c.string.preencode(state, m.path)
203 uintArray.preencode(state, m.indices)
204 },
205 encode (state, m) {
206 c.string.encode(state, m.path)
207 uintArray.encode(state, m.indices)
114208 },
115209 decode (state) {
116210 return {
117 manifestHash: c.buffer.decode(state).toString('hex'),
118 bitfield: c.buffer.decode(state)
211 path: c.string.decode(state),
212 indices: uintArray.decode(state)
119213 }
120214 }
121215}
122216
123// Message 5: chunk-request
124export const chunkRequest = {
217// Message 5: chunk-data — send chunk bytes
218export const chunkData = {
125219 preencode (state, m) {
126 c.buffer.preencode(state, toBuffer32(m.manifestHash))
220 c.string.preencode(state, m.path)
127221 c.uint.preencode(state, m.index)
128 c.uint.preencode(state, m.requestId)
129 c.uint.preencode(state, m.priority)
222 c.buffer.preencode(state, m.data)
130223 },
131224 encode (state, m) {
132 c.buffer.encode(state, toBuffer32(m.manifestHash))
225 c.string.encode(state, m.path)
133226 c.uint.encode(state, m.index)
134 c.uint.encode(state, m.requestId)
135 c.uint.encode(state, m.priority)
227 c.buffer.encode(state, m.data)
136228 },
137229 decode (state) {
138230 return {
139 manifestHash: c.buffer.decode(state).toString('hex'),
231 path: c.string.decode(state),
140232 index: c.uint.decode(state),
141 requestId: c.uint.decode(state),
142 priority: c.uint.decode(state)
233 data: c.buffer.decode(state)
143234 }
144235 }
145236}
146237
147// Message 6: chunk-response
148export const chunkResponse = {
238// Message 6: chunk-cancel — cancel a pending transfer
239export const chunkCancel = {
240 preencode (state, m) { c.string.preencode(state, m.path) },
241 encode (state, m) { c.string.encode(state, m.path) },
242 decode (state) { return { path: c.string.decode(state) } }
243}
244
245// Message 7: transfer-complete — confirm file fully received
246export const transferComplete = {
149247 preencode (state, m) {
150 c.uint.preencode(state, m.requestId)
151 c.buffer.preencode(state, m.data || Buffer.alloc(0))
152 c.bool.preencode(state, m.found)
248 c.string.preencode(state, m.path)
249 c.buffer.preencode(state, toBuffer32(m.contentHash))
153250 },
154251 encode (state, m) {
155 c.uint.encode(state, m.requestId)
156 c.buffer.encode(state, m.data || Buffer.alloc(0))
157 c.bool.encode(state, m.found)
252 c.string.encode(state, m.path)
253 c.buffer.encode(state, toBuffer32(m.contentHash))
158254 },
159255 decode (state) {
160256 return {
161 requestId: c.uint.decode(state),
162 data: c.buffer.decode(state),
163 found: c.bool.decode(state)
257 path: c.string.decode(state),
258 contentHash: c.buffer.decode(state).toString('hex')
164259 }
165260 }
166261}
167262
168// Message 7: chunk-cancel
169export const chunkCancel = {
170 preencode (state, m) { c.uint.preencode(state, m.requestId) },
171 encode (state, m) { c.uint.encode(state, m.requestId) },
172 decode (state) { return { requestId: c.uint.decode(state) } }
263// Message 8: conflict — notify peer of a detected conflict
264export const conflict = {
265 preencode (state, m) {
266 c.string.preencode(state, m.path)
267 c.buffer.preencode(state, toBuffer32(m.myHash))
268 c.buffer.preencode(state, toBuffer32(m.theirHash))
269 c.buffer.preencode(state, toBuffer32(m.ancestorHash || Buffer.alloc(32)))
270 },
271 encode (state, m) {
272 c.string.encode(state, m.path)
273 c.buffer.encode(state, toBuffer32(m.myHash))
274 c.buffer.encode(state, toBuffer32(m.theirHash))
275 c.buffer.encode(state, toBuffer32(m.ancestorHash || Buffer.alloc(32)))
276 },
277 decode (state) {
278 return {
279 path: c.string.decode(state),
280 myHash: c.buffer.decode(state).toString('hex'),
281 theirHash: c.buffer.decode(state).toString('hex'),
282 ancestorHash: c.buffer.decode(state).toString('hex')
283 }
284 }
285}
286
287// ── Tree Messages (9-11) — 0.5a extension ────────────────────
288
289// Encode/decode an array of tree entries: { kind, exec, name, childHash, size }
290const treeEntryArray = {
291 preencode (state, arr) {
292 c.uint32.preencode(state, arr.length)
293 for (const e of arr) {
294 c.uint8.preencode(state, e.kind)
295 c.uint8.preencode(state, e.exec || 0)
296 c.string.preencode(state, e.name)
297 state.end += 32 // childHash fixed32
298 c.uint.preencode(state, e.size || 0)
299 }
300 },
301 encode (state, arr) {
302 c.uint32.encode(state, arr.length)
303 for (const e of arr) {
304 c.uint8.encode(state, e.kind)
305 c.uint8.encode(state, e.exec || 0)
306 c.string.encode(state, e.name)
307 toBuffer32(e.childHash).copy(state.buffer, state.start, 0, 32)
308 state.start += 32
309 c.uint.encode(state, e.size || 0)
310 }
311 },
312 decode (state) {
313 const len = c.uint32.decode(state)
314 const arr = []
315 for (let i = 0; i < len; i++) {
316 const kind = c.uint8.decode(state)
317 const exec = c.uint8.decode(state)
318 const name = c.string.decode(state)
319 const childHash = state.buffer.subarray(state.start, state.start + 32).toString('hex')
320 state.start += 32
321 const size = c.uint.decode(state)
322 arr.push({ kind, exec, name, childHash, size })
323 }
324 return arr
325 }
173326}
174327
175// Message 9: chunk-request-by-hash
328// Message 9: tree-request — ask peer for a tree by its content hash
176329//
177// Content-addressed chunk fetch — ask a peer directly for a chunk by its
178// content hash, without going through a manifest lookup. The peer replies
179// using the existing chunkResponse message (message 6).
330// `nonce` (per-request id) is appended last; the holder echoes it in every
331// treeResponse page so the requester can drop pages from a superseded attempt.
332// A pre-nonce peer omits it and decodes 0, which the requester treats as
333// "unverifiable".
334export const treeRequest = {
335 preencode (state, m) {
336 c.buffer.preencode(state, toBuffer32(m.hash))
337 c.uint.preencode(state, m.nonce || 0)
338 },
339 encode (state, m) {
340 c.buffer.encode(state, toBuffer32(m.hash))
341 c.uint.encode(state, m.nonce || 0)
342 },
343 decode (state) {
344 const hash = c.buffer.decode(state).toString('hex')
345 const nonce = state.start < state.end ? c.uint.decode(state) : 0
346 return { hash, nonce }
347 }
348}
349
350// Message 10: tree-response — return a tree's entries to the requester
180351//
181// Used for small (<1MB) content-addressed blobs like profile pictures and
182// issue attachments, where the caller knows the chunk hash out-of-band.
183export const requestChunkByHash = {
352// `more` (paging flag) mirrors chunkHashes: a tree too large for one 16 MiB-1
353// Noise frame is split into several frames, every page but the last more:1.
354// `nonce` echoes the requester's per-request id so pages from a superseded
355// attempt are dropped. Both are appended last; a pre-paging peer omits them and
356// decodes 0.
357export const treeResponse = {
184358 preencode (state, m) {
185359 c.buffer.preencode(state, toBuffer32(m.hash))
186 c.uint.preencode(state, m.requestId)
360 treeEntryArray.preencode(state, m.entries)
361 c.uint8.preencode(state, m.more || 0)
362 c.uint.preencode(state, m.nonce || 0)
187363 },
188364 encode (state, m) {
189365 c.buffer.encode(state, toBuffer32(m.hash))
190 c.uint.encode(state, m.requestId)
366 treeEntryArray.encode(state, m.entries)
367 c.uint8.encode(state, m.more || 0)
368 c.uint.encode(state, m.nonce || 0)
369 },
370 decode (state) {
371 const hash = c.buffer.decode(state).toString('hex')
372 const entries = treeEntryArray.decode(state)
373 const more = state.start < state.end ? c.uint8.decode(state) : 0
374 const nonce = state.start < state.end ? c.uint.decode(state) : 0
375 return { hash, entries, more, nonce }
376 }
377}
378
379// Message 11: content-request — fetch a file by its content hash (not path)
380// Sender locates any local file matching the hash and serves its chunks.
381// `from` carries the requester's profile-key (hex) so the
382// holder's serveAuthorizer can authenticate the asker. Appended last → an
383// overlay peer that predates this field decodes '' (empty), never crashes.
384export const contentRequest = {
385 preencode (state, m) {
386 c.buffer.preencode(state, toBuffer32(m.contentHash))
387 c.buffer.preencode(state, m.chunksHave || Buffer.alloc(0))
388 c.string.preencode(state, m.from || '')
389 },
390 encode (state, m) {
391 c.buffer.encode(state, toBuffer32(m.contentHash))
392 c.buffer.encode(state, m.chunksHave || Buffer.alloc(0))
393 c.string.encode(state, m.from || '')
191394 },
192395 decode (state) {
193396 return {
194 hash: c.buffer.decode(state).toString('hex'),
195 requestId: c.uint.decode(state)
397 contentHash: c.buffer.decode(state).toString('hex'),
398 chunksHave: c.buffer.decode(state),
399 from: c.string.decode(state)
196400 }
197401 }
198402}
199403
200// Message 8: chunk-have (incremental availability update)
201export const chunkHave = {
404// Message 12: transfer-control — downloader→holder pause/stop notice for a
405// content-addressed fetch. state: 0 = paused, 1 = stopped. Appended last so a
406// holder that predates this never registers slot 12 and silently ignores it.
407export const transferControl = {
202408 preencode (state, m) {
203 c.buffer.preencode(state, toBuffer32(m.manifestHash))
204 c.uint.preencode(state, m.startIndex)
205 c.buffer.preencode(state, m.bitfield)
409 c.buffer.preencode(state, toBuffer32(m.contentHash))
410 c.uint8.preencode(state, m.state)
206411 },
207412 encode (state, m) {
208 c.buffer.encode(state, toBuffer32(m.manifestHash))
209 c.uint.encode(state, m.startIndex)
210 c.buffer.encode(state, m.bitfield)
413 c.buffer.encode(state, toBuffer32(m.contentHash))
414 c.uint8.encode(state, m.state)
211415 },
212416 decode (state) {
213417 return {
214 manifestHash: c.buffer.decode(state).toString('hex'),
215 startIndex: c.uint.decode(state),
216 bitfield: c.buffer.decode(state)
418 contentHash: c.buffer.decode(state).toString('hex'),
419 state: c.uint8.decode(state)
217420 }
218421 }
219422}
220423
221function toBuffer32 (hashOrBuf) {
222 if (Buffer.isBuffer(hashOrBuf)) return hashOrBuf
223 return Buffer.from(hashOrBuf, 'hex')
424// Message 13: transfer-progress — downloader→holder one-shot have-baseline for a
425// resumed content-addressed fetch. `have` = bytes the downloader already holds on
426// disk at resume, so the holder's "who is downloading" bar reflects the downloader's
427// TRUE completion, not only the bytes it re-serves. Appended last so a holder that
428// predates this never registers slot 13 and silently ignores it.
429export const transferProgress = {
430 preencode (state, m) {
431 c.buffer.preencode(state, toBuffer32(m.contentHash))
432 c.uint.preencode(state, m.have || 0)
433 },
434 encode (state, m) {
435 c.buffer.encode(state, toBuffer32(m.contentHash))
436 c.uint.encode(state, m.have || 0)
437 },
438 decode (state) {
439 return {
440 contentHash: c.buffer.decode(state).toString('hex'),
441 have: c.uint.decode(state)
442 }
443 }
224444}
lib/overlay-v2.js +0 -2
……@@ -1,2 +0,0 @@
1// SHIM -> lib/v3 engine. HyperOverlay(v3) exposed under our HyperOverlayV2 name.
2export { HyperOverlay as HyperOverlayV2, HyperOverlay } from './v3/overlay.js'
lib/overlay.js +417 -0
……@@ -0,0 +1,417 @@
1/**
2 * HyperOverlay — content-addressed file serving over the V2 protocol.
3 *
4 * A thin facade over the V2 stack (FileIndex + SyncEngine + TransferManager +
5 * OverlayProtocol) for the "serve a file to whoever asks, fetch a file from
6 * whoever has it" use case — distinct from V1's local chunk-registry blobs.
7 *
8 * Model (the deliberate post-Hyperdrive design): the canonical bytes are a
9 * REAL FILE ON DISK. The index holds only metadata + chunk maps (no blob
10 * bytes), so there is no hypercore storage balloon. A peer fetches a file by
11 * its content hash; chunks are read from the source file on demand, streamed,
12 * hash-verified, and written to the requester's disk.
13 *
14 * `autoSync` is OFF: this is on-demand content fetch (requestContent →
15 * _onContentRequest), NOT Syncthing-style folder push. Registering a file
16 * makes it servable; nothing is offered to peers unsolicited.
17 *
18 * Requester flow: pre-register a synthetic `content:<hash>` → destPath
19 * mapping, requestContent,
20 * await the hash-verified transferComplete, walk peers on timeout/mismatch.
21 */
22
23import ReadyResource from 'ready-resource'
24import fs from 'bare-fs'
25import path from 'bare-path'
26import os from 'bare-os'
27import crypto from 'hypercore-crypto'
28
29import { FileIndex } from './file-index.js'
30import { SyncEngine } from './sync-engine.js'
31import { TransferManager } from './transfer.js'
32import { OverlayProtocol } from './protocol.js'
33import { createStreamingHasher } from './chunker.js'
34
35const DEFAULT_PER_PEER_TIMEOUT = 30000
36const DEFAULT_IDLE_TIMEOUT = 10000
37
38// Stream-verify a file against its overlay content hash WITHOUT buffering
39// it — a readFileSync of a multi-GB blob OOMs the process. Matches
40// crypto.data(buffer): blake2b(LEAF_TYPE || uint64-LE(size) || bytes), which is
41// exactly createStreamingHasher({ size }). Returns the size so callers can re-seed
42// without a second stat.
43async function verifyOnDisk (filePath, contentHash) {
44 let size
45 try { size = fs.statSync(filePath).size } catch { return { ok: false, size: 0 } }
46 const h = createStreamingHasher({ size })
47 try {
48 await new Promise((resolve, reject) => {
49 const rs = fs.createReadStream(filePath)
50 rs.on('data', (c) => h.update(c))
51 rs.on('end', resolve)
52 rs.on('error', reject)
53 })
54 } catch { return { ok: false, size } }
55 return { ok: h.digest() === contentHash, size }
56}
57
58export class HyperOverlay extends ReadyResource {
59 constructor (corestore, opts = {}) {
60 super()
61 this._corestore = corestore.namespace(opts.namespace || 'overlay-v2')
62 this._destDir = opts.destDir || path.join(os.tmpdir(), 'hyper-overlay-v2-fetch')
63 this._journalDir = opts.journalDir || null
64 this._syncBaseDirs = opts.syncBaseDirs || []
65 this._perPeerTimeout = opts.perPeerTimeout || DEFAULT_PER_PEER_TIMEOUT
66 this._idleTimeout = opts.idleTimeout || DEFAULT_IDLE_TIMEOUT
67
68 // Serve-authorization + identity opts threaded to OverlayProtocol.
69 // serveAuthorizer gates every inbound content-request; localProfileKey stamps
70 // outbound content-requests so the remote's gate can authenticate the asker.
71 this._serveAuthorizer = opts.serveAuthorizer || null
72 this._localProfileKey = opts.localProfileKey || null
73
74 // At-rest key for the local index cores (file-index/index-meta/sync-feed).
75 this._indexEncryptionKey = opts.indexEncryptionKey || null
76
77 // serve-side download telemetry (sender-side download indicator).
78 this._onServeStart = opts.onServeStart || null
79 this._onChunkServe = opts.onChunkServe || null
80 this._onServeEnd = opts.onServeEnd || null
81 this._onServeControl = opts.onServeControl || null
82 this._onServeProgress = opts.onServeProgress || null
83
84 // Shared maps the protocol reads: overlayPath → diskPath (for serving an
85 // offered/known path), contentHash → diskPath (fast path for content
86 // requests). Both populated by registerFile.
87 this._filePaths = new Map()
88 this._contentHashPaths = new Map()
89
90 // contentHash → pending fetch { destPath, finish, timers, bytesReceived }
91 this._pendingContent = new Map()
92
93 // Built lazily on first use (Hyperdrive-style: a drive that never touches
94 // its blobs core never opens one). A daemon that never registers, serves,
95 // or connects a peer for attachments pays nothing — no feed, no index.
96 this._index = null
97 this._sync = null
98 this._transfer = null
99 this._protocol = null
100 this._stackPromise = null
101 }
102
103 // The protocol must exist synchronously when attachProtocol is called —
104 // protomux will NOT pair a channel created after the remote opened theirs
105 // (verified). So the stack is built at ready time. _ensure stays as the
106 // idempotent builder (used by register/fetch, and as a safety net).
107 async _open () { await this._ensure() }
108
109 async _close () {
110 // Only tear down what was actually built.
111 if (this._protocol) this._protocol.destroy()
112 if (this._sync) await this._sync.close()
113 if (this._index) await this._index.close()
114 }
115
116 // Lazily build the V2 stack. Idempotent + concurrency-safe (a single
117 // in-flight promise is shared). No-op cost once built.
118 _ensure () {
119 if (this._protocol) return Promise.resolve()
120 if (this._stackPromise) return this._stackPromise
121 this._stackPromise = (async () => {
122 const index = new FileIndex(this._corestore, { encryptionKey: this._indexEncryptionKey })
123 await index.ready()
124 const sync = new SyncEngine(index, this._corestore, { encryptionKey: this._indexEncryptionKey })
125 await sync.ready()
126 const transfer = new TransferManager(index, { journalDir: this._journalDir })
127 const protocol = new OverlayProtocol(sync, transfer, {
128 filePaths: this._filePaths,
129 contentHashPaths: this._contentHashPaths,
130 syncBaseDirs: this._syncBaseDirs,
131 autoSync: false,
132 serveAuthorizer: this._serveAuthorizer, // serve gate
133 localProfileKey: this._localProfileKey, // outbound identity
134 onSynced: (info) => this._onSynced(info),
135 onChunkProgress: (info) => this._onChunkProgress(info),
136 onServeStart: this._onServeStart,
137 onChunkServe: this._onChunkServe,
138 onServeEnd: this._onServeEnd,
139 onServeControl: this._onServeControl,
140 onServeProgress: this._onServeProgress
141 })
142 try { fs.mkdirSync(this._destDir, { recursive: true }) } catch {}
143 if (this._journalDir) { try { fs.mkdirSync(this._journalDir, { recursive: true }) } catch {} }
144 this._index = index
145 this._sync = sync
146 this._transfer = transfer
147 this._protocol = protocol
148 })()
149 return this._stackPromise
150 }
151
152 // The host's swarm transport must call this synchronously per connection. The
153 // channel MUST be created synchronously — protomux will not pair a channel
154 // opened after the remote's (verified). The stack is built at ready, so
155 // this._protocol is always present by the time connections arrive.
156 attachProtocol (mux) {
157 if (this.closing || this.closed || !this._protocol) return
158 return this._protocol.attach(mux)
159 }
160
161 get peerCount () { return this._protocol ? this._protocol.peerCount : 0 }
162
163 // The overlay's local-only cores (file-index + version marker + sync feed), so the
164 // leftover scan can treat them as wanted and the storage breakdown can size them.
165 localCores () {
166 const cores = this._index ? this._index.cores : []
167 if (this._sync?.feed) cores.push(this._sync.feed)
168 return cores
169 }
170
171 cleanJournals (maxAge) { try { return this._transfer?.cleanJournals(maxAge) || [] } catch { return [] } }
172
173 /**
174 * Make an on-disk file servable by its content hash.
175 * @param {string} overlayPath logical path, e.g. '/attachments/<oid>'
176 * @param {string} diskPath absolute path to the canonical bytes on disk
177 * @param {object} [meta]
178 * meta.contentHash preferred — the oid the caller already computed
179 * meta.size file size (avoids an extra stat if known)
180 * @returns {Promise<{ contentHash, size } | null>} null if the source vanished
181 */
182 async registerFile (overlayPath, diskPath, meta = {}) {
183 await this._ensure()
184 // Source can vanish between prepare and register (a file moved out of a
185 // shared folder mid-scan). Nothing to serve — bail with null instead of throwing
186 // so a routine move doesn't crash the publish path.
187 let stat
188 try {
189 stat = fs.statSync(diskPath)
190 } catch {
191 return null
192 }
193 const size = typeof meta.size === 'number' ? meta.size : stat.size
194 const contentHash = meta.contentHash || await hashFileOnDisk(diskPath, size)
195
196 await this._index.putFile(overlayPath, { contentHash, size, mtime: stat.mtimeMs })
197 this._filePaths.set(overlayPath, diskPath)
198 this._contentHashPaths.set(contentHash, diskPath)
199 // Build (and cache) the chunk map so a content request can stream
200 // immediately without re-chunking under the request. meta.prepare:false
201 // skips this eager full second read of the file — the file is already servable
202 // via the maps above and the serve path (_onContentRequest) chunks lazily on
203 // first request; publishers that already built the chunk map in one pass
204 // (prepareForServe) pass false so registration doesn't re-read the file.
205 if (meta.prepare !== false) await this._transfer.prepareFile(diskPath, overlayPath)
206 // Record in the local change feed (used by the protocol's markSynced
207 // bookkeeping; with autoSync off it is not broadcast).
208 await this._sync.logPut(overlayPath, { contentHash, size, mtime: stat.mtimeMs })
209 return { contentHash, size }
210 }
211
212 // One-pass hash + chunk-map build for the publish path: streams the file
213 // ONCE, persists the content-addressed chunk map (durable in FileIndex), and
214 // returns the content hash. Replaces a hash-only pass plus a lazy fetch-time
215 // re-chunk with a single read, so the first fetcher never waits on chunk indexing.
216 async prepareForServe (diskPath, opts = {}) {
217 await this._ensure()
218 // The overlay path below is a throwaway key — byHashOnly persists ONLY the
219 // content-addressed chunk map, nothing under this path.
220 const prepared = await this._transfer.prepareFile(diskPath, '/mir-prep' + diskPath, { onProgress: opts.onProgress, byHashOnly: true, signal: opts.signal })
221 return prepared ? { contentHash: prepared.contentHash, size: prepared.size } : null
222 }
223
224 async evictContent (contentHash) {
225 await this._ensure()
226 return this._index.evictContent(contentHash)
227 }
228
229 async compactIndex (opts) {
230 await this._ensure()
231 return this._index.compact(opts)
232 }
233
234 /**
235 * Fetch a file by its content hash. Serves from the local disk copy if we
236 * have it; otherwise pulls from a connected peer that does (hash-verified).
237 * @param {string} contentHash
238 * @param {object} [opts] opts.timeout (idle), opts.destPath
239 * @returns {Promise<{ destPath: string, local: boolean, size: number } | null>}
240 */
241 async fetchFile (contentHash, opts = {}) {
242 await this._ensure()
243
244 const localDisk = this._contentHashPaths.get(contentHash)
245 if (localDisk) {
246 const v = await verifyOnDisk(localDisk, contentHash)
247 if (v.ok) return { destPath: localDisk, local: true, size: v.size }
248 }
249
250 // Wait briefly for at least one peer — a connection may be mid-handshake
251 // (lazy attach) or just forming when the fetch is issued. Without this a
252 // fetch issued the instant a peer connects would spuriously 404.
253 const peerWaitMs = opts.peerWaitMs || 3000
254 for (let waited = 0; this._protocol._peers.size === 0 && waited < peerWaitMs; waited += 100) {
255 await new Promise(r => setTimeout(r, 100))
256 }
257 if (this._protocol._peers.size === 0) {
258 // No fetchContent will run, so drop any cancel recorded during the wait — a
259 // stale marker would otherwise cancel the next fetch of the same content.
260 this._protocol.clearCancelPending(contentHash)
261 return null
262 }
263
264 // Multi-source: fetch chunks in parallel from ALL connected peers that
265 // have the file (torrent-style). The scheduler dedups + fails over.
266 const peers = [...this._protocol._peers.values()]
267 // opts.mirrorSpool: store the verified blob CONTENT-ADDRESSED at
268 // <mirrorSpool>/<contentHash> rather than a throwaway destDir name. This is
269 // the mirror path — a node with no working tree of its own fetches files
270 // into a content-addressed spool so it durably re-serves them (via the
271 // <syncBaseDir>/<contentHash> resolution in _onContentRequest).
272 let destPath
273 if (opts.destPath) {
274 destPath = opts.destPath
275 } else if (opts.mirrorSpool) {
276 try { fs.mkdirSync(opts.mirrorSpool, { recursive: true }) } catch {}
277 destPath = path.join(opts.mirrorSpool, contentHash)
278 } else {
279 destPath = path.join(
280 this._destDir,
281 'blob-' + contentHash + '-' + Date.now() + '-' + Math.random().toString(36).slice(2, 8)
282 )
283 }
284 // Whole-file integrity is verified INCREMENTALLY during the transfer (the
285 // scheduler passes contentHash to TransferManager, which hashes the file in
286 // offset order as chunks land and rejects in finalize on a mismatch). So the
287 // assembled file is already verified here — no trailing whole-file re-read.
288 let result
289 try {
290 result = await this._protocol.fetchContent(contentHash, peers, {
291 destPath,
292 timeout: opts.timeout,
293 onProgress: opts.onProgress || (() => {}), // forward bytes/total
294 onVerify: opts.onVerify, // resume re-verify fraction
295 onEnd: opts.onEnd // terminal diagnostic (reason + bytes/chunks)
296 })
297 } catch (err) {
298 // An integrity failure or an explicit cancel/pause is distinct from a
299 // no-holder / stall — surface it so the caller doesn't treat it as a failure.
300 if (err?.code === 'EHASHMISMATCH' || err?.code === 'ECANCELLED') throw err
301 // A local I/O error (full disk / read-only / permission / vanished
302 // mount) is not a no-holder — surface it so the consumer can pause rather than
303 // retry forever.
304 if (err?.code === 'ENOSPC' || err?.code === 'EACCES' || err?.code === 'EROFS' || err?.code === 'EPERM' || err?.code === 'ENOENT') throw err
305 return null
306 }
307 const size = result?.size ?? fs.statSync(destPath).size
308 // Seeding multiplication: register the verified blob locally so this node
309 // now SERVES these chunks to other peers too. A file held by N peers then
310 // gets faster as more peers fetch it (torrent-style). Best-effort — the
311 // fetch still succeeds even if re-registration fails. Skipped if a peer
312 // raced us to register the same hash (registerFile is idempotent-safe).
313 // reSeed:false — opt-out for consumers that should hold nothing.
314 if (opts.reSeed !== false && !this._contentHashPaths.has(contentHash)) {
315 try {
316 await this.registerFile('/attachments/' + contentHash, destPath, { contentHash, size })
317 } catch {}
318 }
319 return { destPath, local: false, size }
320 }
321
322 // Stop an in-flight fetchFile for this content hash. opts.discardPartial
323 // true unlinks the partial (cancel), false keeps it for resume (pause). The
324 // pending fetchFile rejects with ECANCELLED, which the caller treats as not-a-failure.
325 cancelFetch (contentHash, opts = {}) {
326 return this._protocol ? this._protocol.cancelContent(contentHash, opts) : false
327 }
328
329 // Tell holders we stopped pulling this hash when there's no in-flight fetch
330 // to tear down (e.g. discarding an already-paused transfer), so their indicator clears.
331 notifyTransferStopped (contentHash) {
332 return this._protocol ? this._protocol.sendStopControl(contentHash) : false
333 }
334
335 _destPathFor (contentHash) {
336 return path.join(this._destDir, 'blob-' + contentHash)
337 }
338
339 _fetchFromPeer (peer, contentHash, opts = {}) {
340 const syntheticPath = 'content:' + contentHash
341 const destPath = path.join(
342 this._destDir,
343 'blob-' + contentHash + '-' + Date.now() + '-' + Math.random().toString(36).slice(2, 8)
344 )
345 // The protocol's chunk handlers write to whatever _filePaths maps the
346 // synthetic path to. Per-attempt destPath so concurrent peer tries don't
347 // clobber one another.
348 this._filePaths.set(syntheticPath, destPath)
349
350 return new Promise((resolve, reject) => {
351 let settled = false
352 const finish = (err, result) => {
353 if (settled) return
354 settled = true
355 clearTimeout(timer)
356 clearTimeout(pending.idleTimer)
357 this._pendingContent.delete(contentHash)
358 if (err) reject(err); else resolve(result)
359 }
360 const timer = setTimeout(() => {
361 finish(new Error('content request timed out: ' + contentHash.slice(0, 12)))
362 }, opts.timeout || this._perPeerTimeout)
363 const idleFire = () => finish(new Error('peer went silent mid-stream: ' + contentHash.slice(0, 12)))
364 const pending = {
365 syntheticPath, destPath, finish, peer, contentHash,
366 bytesReceived: 0,
367 idleTimer: setTimeout(idleFire, this._idleTimeout),
368 idleFire
369 }
370 this._pendingContent.set(contentHash, pending)
371 this._protocol.requestContent(peer, contentHash, null)
372 })
373 }
374
375 _onChunkProgress (info) {
376 if (!info || !info.path || !info.path.startsWith('content:')) return
377 const contentHash = info.path.slice('content:'.length)
378 const pending = this._pendingContent.get(contentHash)
379 if (!pending) return
380 pending.bytesReceived += (info.bytes || 0)
381 clearTimeout(pending.idleTimer)
382 pending.idleTimer = setTimeout(pending.idleFire, this._idleTimeout)
383 }
384
385 // The V2 chunk flow does NOT verify the reassembled file against the
386 // advertised hash (that's the caller's job in this protocol). We do it here
387 // so fetchFile never returns un-verified bytes.
388 _onSynced (info) {
389 if (!info || info.direction !== 'pull') return
390 if (!info.path || !info.path.startsWith('content:')) return
391 const contentHash = info.path.slice('content:'.length)
392 const pending = this._pendingContent.get(contentHash)
393 if (!pending) return
394 try {
395 const data = fs.readFileSync(pending.destPath)
396 const actual = crypto.data(data).toString('hex')
397 if (actual !== contentHash) {
398 pending.finish(Object.assign(
399 new Error('content-hash mismatch: expected ' + contentHash.slice(0, 12) + '…, got ' + actual.slice(0, 12) + '…'),
400 { code: 'EHASHMISMATCH', expected: contentHash, actual }
401 ))
402 return
403 }
404 pending.finish(null, data)
405 } catch (err) {
406 pending.finish(err)
407 }
408 }
409}
410
411// Stream a file through the chunker's hasher to get its oid without buffering
412// the whole file in memory (matches /api/upload's createStreamingHasher oid).
413async function hashFileOnDisk (diskPath, size) {
414 const hasher = createStreamingHasher({ size })
415 for await (const buf of fs.createReadStream(diskPath)) hasher.update(buf)
416 return hasher.digest()
417}
lib/pairing.js +0 -203
……@@ -1,203 +0,0 @@
1/**
2 * Device Pairing — blind-pairing based device linking for hyper-overlay
3 *
4 * Follows the same pattern as pear-git's device-linker and repo-linker.
5 * Pairing config stored in devices.json (no Corestore lock needed).
6 *
7 * Flow:
8 * Device A: overlay invite → gives hex code to user
9 * Device B: overlay join <code> → pairs with A
10 * Both devices store shared topic key → auto-connect on every startup
11 */
12
13import BlindPairing from 'blind-pairing'
14import crypto from 'hypercore-crypto'
15import fs from '#fs'
16import path from '#path'
17
18// ── Device storage (plain JSON, no Corestore) ─────────────────
19
20function devicesPath (storagePath) {
21 return path.join(storagePath, 'devices.json')
22}
23
24export function readDevices (storagePath) {
25 try {
26 return JSON.parse(fs.readFileSync(devicesPath(storagePath), 'utf-8'))
27 } catch {
28 return []
29 }
30}
31
32export function writeDevices (storagePath, devices) {
33 fs.mkdirSync(storagePath, { recursive: true })
34 fs.writeFileSync(devicesPath(storagePath), JSON.stringify(devices, null, 2))
35}
36
37export function addDevice (storagePath, name, sharedKey) {
38 const keyHex = sharedKey.toString('hex')
39 const devices = readDevices(storagePath)
40 if (devices.find(d => d.key === keyHex)) return
41 devices.push({ name, key: keyHex, pairedAt: Date.now() })
42 writeDevices(storagePath, devices)
43}
44
45// ── Invite creation ───────────────────────────────────────────
46
47/**
48 * Create a pairing invite (member/creator side).
49 * Follows pear-git's pattern: keypair for discovery + seed for signing.
50 *
51 * @returns {{ inviteHex, sharedTopic, pairingKey, invitePublicKey, discoveryKey, inviteKeyPair }}
52 */
53export function createInvite () {
54 // The shared topic is what both devices will join on Hyperswarm after pairing
55 const sharedTopic = crypto.randomBytes(32)
56
57 // Fresh keypair for DHT discovery
58 const pairingKey = crypto.keyPair()
59 const inviteResult = BlindPairing.createInvite(pairingKey.publicKey)
60 const { invite, publicKey: invitePublicKey, discoveryKey, seed: inviteSeed } = inviteResult
61
62 // Derive signing keypair from invite seed (for signed confirm)
63 const inviteKeyPair = crypto.keyPair(inviteSeed)
64
65 return {
66 inviteHex: invite.toString('hex'),
67 sharedTopic,
68 pairingKey,
69 invitePublicKey,
70 discoveryKey,
71 inviteKeyPair
72 }
73}
74
75// ── Member side (invite creator, listens for candidate) ───────
76
77/**
78 * Listen for a pairing candidate.
79 *
80 * @param {Hyperswarm} swarm
81 * @param {string} storagePath
82 * @param {Object} inviteData - from createInvite()
83 * @returns {Promise<{ peerName: string, sharedTopic: Buffer }>}
84 */
85export function listenForPair (swarm, storagePath, inviteData) {
86 const bp = new BlindPairing(swarm, { poll: 5000 })
87
88 // Payload: the shared topic key that both devices will use
89 const payload = Buffer.from(JSON.stringify({
90 topic: inviteData.sharedTopic.toString('hex')
91 }), 'utf-8')
92
93 const peerConnected = new Promise((resolve, reject) => {
94 const timeout = setTimeout(() => {
95 reject(new Error('Timeout waiting for peer (5 min)'))
96 }, 5 * 60 * 1000)
97
98 bp.addMember({
99 discoveryKey: inviteData.discoveryKey,
100 onadd (request) {
101 clearTimeout(timeout)
102 try {
103 // Decrypt candidate's userData
104 const userData = request.open(inviteData.invitePublicKey)
105 const peerName = (userData && userData.length > 0)
106 ? userData.toString('utf-8')
107 : 'unknown'
108
109 // Confirm pairing — send signed payload with shared topic
110 request.confirm({
111 key: inviteData.pairingKey.publicKey,
112 encryptionKey: null,
113 additional: {
114 data: payload,
115 signature: crypto.sign(payload, inviteData.inviteKeyPair.secretKey)
116 }
117 })
118
119 // Store paired device
120 addDevice(storagePath, peerName, inviteData.sharedTopic)
121
122 resolve({ peerName, sharedTopic: inviteData.sharedTopic, bp })
123 } catch (err) {
124 reject(err)
125 }
126 }
127 })
128 })
129
130 return { peerConnected, bp }
131}
132
133// ── Candidate side (invite joiner) ────────────────────────────
134
135/**
136 * Join using a pairing invite code.
137 * No Corestore needed — just Hyperswarm + JSON file.
138 *
139 * @param {Hyperswarm} swarm
140 * @param {string} storagePath
141 * @param {string} inviteHex
142 * @param {string} deviceName
143 * @returns {Promise<{ sharedTopic: Buffer }>}
144 */
145export async function joinWithInvite (swarm, storagePath, inviteHex, deviceName) {
146 const inviteBuf = Buffer.from(inviteHex, 'hex')
147 const bp = new BlindPairing(swarm, { poll: 5000 })
148
149 const result = await new Promise((resolve, reject) => {
150 const timeout = setTimeout(() => {
151 reject(new Error('Timeout waiting for source device (5 min)'))
152 }, 5 * 60 * 1000)
153
154 bp.addCandidate({
155 invite: inviteBuf,
156 userData: Buffer.from(deviceName, 'utf-8'),
157 onadd (auth) {
158 clearTimeout(timeout)
159 try {
160 if (!auth.data || auth.data.length === 0) {
161 reject(new Error('No payload received from source'))
162 return
163 }
164 const payload = JSON.parse(auth.data.toString('utf-8'))
165 resolve(payload)
166 } catch (err) {
167 reject(err)
168 }
169 }
170 })
171 })
172
173 const sharedTopic = Buffer.from(result.topic, 'hex')
174
175 // Store paired device
176 addDevice(storagePath, 'remote', sharedTopic)
177
178 await bp.close()
179
180 return { sharedTopic }
181}
182
183// ── Auto-rejoin paired devices ────────────────────────────────
184
185/**
186 * Join swarm topics for all paired devices.
187 * @param {Hyperswarm} swarm
188 * @param {string} storagePath
189 * @returns {number} number of topics joined
190 */
191export async function joinPairedDevices (swarm, storagePath) {
192 const devices = readDevices(storagePath)
193 let count = 0
194
195 for (const device of devices) {
196 const topic = Buffer.from(device.key, 'hex')
197 swarm.join(topic, { server: true, client: true })
198 count++
199 }
200
201 if (count > 0) await swarm.flush()
202 return count
203}
lib/protocol-v2.js +0 -2
……@@ -1,2 +0,0 @@
1// SHIM -> lib/v3 engine. OverlayProtocol(v3) exposed under our OverlayProtocolV2 name.
2export { OverlayProtocol, OverlayProtocol as OverlayProtocolV2 } from './v3/protocol.js'
lib/protocol.js +853 -277
……@@ -1,39 +1,211 @@
11/**
2 * OverlayProtocol — Protomux channel for P2P chunk exchange
2 * OverlayProtocol — Protomux channel for sync + transfer
33 *
4 * Runs as 'hyper-overlay/alpha' channel on replicationSwarm connections.
5 * Two-level availability: manifest hash set exchange → per-manifest bitfield.
4 * Sync approach: each side sends its local feed entries directly via
5 * file-offer messages. No Hypercore feed replication needed — the feed
6 * entries are small JSON objects sent over our own Protomux channel.
67 *
7 * Pattern follows pear-git's git-sync-channel.js:
8 * - mux.pair() registration before createChannel
9 * - handshake exchanges version + manifest count
10 * - manifestHave after open
11 * - chunkBitfield per shared manifest
12 * - chunkRequest/chunkResponse for transfer
8 * Flow:
9 * 1. Channel opens → both send sync-state (feed length)
10 * 2. Side with more entries sends file-offers for each change
11 * 3. Receiver auto-requests files it doesn't have
12 * 4. Sender chunks on demand → streams chunks → receiver writes to disk
1313 */
1414
15import Protomux from 'protomux'
15import path from 'bare-path'
1616import * as messages from './messages.js'
17import { hashChunk } from './chunker.js'
18import { decode as decodeManifest } from './manifest.js'
19
20const PROTOCOL = 'hyper-overlay/alpha'
17import { hashChunk, selectTier } from './chunker.js'
18import { ChunkScheduler } from './chunk-scheduler.js'
19import fs from 'bare-fs'
20
21const PROTOCOL = 'hyper-overlay/v2'
22const VERSION = 2
23const CAP_LOCAL_FILES = 0x01
24const CAP_ADAPTIVE_CHUNKS = 0x02
25
26// A file's chunk list, shipped in one frame, can exceed the
27// 16 MiB-1 Noise transport limit (@hyperswarm/secret-stream MAX_ATOMIC_WRITE):
28// a 1.25 TB file at tier 3 is ~1.2M entries ≈ ~44 MB, which the secret-stream
29// rejects ("Message is too large for an atomic write"), dropping the connection
30// before a byte ships. So chunkHashes is paged into frames of ≤ this many
31// entries. Each wire entry is 32 B (hash) + ≤5 B (varint length) ≈ 37 B, so
32// 100k entries ≈ 3.7 MB/frame — well under the limit even after Protomux's
33// ≤8 MiB send-batching.
34const MAX_CHUNKS_PER_MSG = 100000
35
36// Tree entries are variable length (name is a var-string), unlike the fixed-width
37// chunkHashes entries, so treeResponse pages by an encoded-byte budget rather than
38// a fixed count. 4 MiB keeps each frame well under the atomic-write limit even
39// after Protomux's ≤8 MiB send-batching.
40const MAX_TREE_BYTES_PER_MSG = 4 * 1024 * 1024
41
42// Defensive ceiling on the bytes a requester will accumulate across a paged
43// treeResponse. A single-directory tree is orders of magnitude smaller; this only
44// bounds memory so a peer streaming pages endlessly rejects the request instead
45// of OOMing.
46const MAX_TREE_RESPONSE_BYTES = 256 * 1024 * 1024
47
48// transfer-control (message 12) states, downloader→holder.
49const CONTROL_PAUSED = 0
50const CONTROL_STOPPED = 1
51
52// Abandon a serve loop whose stream neither drains nor closes within
53// this window (a wedged-but-TCP-alive peer); the receiver re-requests, so it self-heals.
54const DRAIN_TIMEOUT_MS = 60000
2155
2256export class OverlayProtocol {
23 constructor (registry, opts = {}) {
24 this._registry = registry
57 constructor (syncEngine, transferManager, opts = {}) {
58 this._syncEngine = syncEngine
59 this._transferManager = transferManager
60 this._corestore = opts.corestore || null
2561 this._overlayKey = opts.key || null
26 this._peers = new Map() // mux → peer state
27 this._nextRequestId = 1
28 this._pendingRequests = new Map() // requestId → { resolve, reject, timeout }
29 this._requestTimeout = opts.requestTimeout || 30000
62 this._peers = new Map()
63 this._filePaths = opts.filePaths || new Map()
64 this._syncBaseDirs = opts.syncBaseDirs || []
65 this._defaultStrategy = opts.defaultStrategy || 'prompt'
66 this._autoSync = opts.autoSync !== false
67
68 this._conflictCb = opts.onConflict || null
69 this._syncedCb = opts.onSynced || null
70 this._fileOfferCb = opts.onFileOffer || null
71 // Called for every chunk that arrives on this peer. Lets callers
72 // (e.g. NetworkFetcher) implement silent-peer watchdogs by resetting
73 // an idle timer on every chunk. Receives { path, index, bytes, peer }.
74 this._chunkProgressCb = opts.onChunkProgress || null
75 // serve-side telemetry, symmetric to onChunkProgress — fired when WE
76 // serve a peer (the sender-side download indicator). onServeStart({ path, peer,
77 // from, total }) once a peer is authorized to fetch; onChunkServe({ path, peer,
78 // from, bytes }) per served chunk; onServeEnd({ path, peer, from }) on close.
79 this._serveStartCb = opts.onServeStart || null
80 this._chunkServeCb = opts.onChunkServe || null
81 this._serveEndCb = opts.onServeEnd || null
82 // A downloader paused/stopped a hash WE serve. { path, peer, from, state }.
83 this._serveControlCb = opts.onServeControl || null
84 // A downloader reported its on-disk have-bytes for a hash WE serve (resume
85 // baseline). { path, peer, from, have }.
86 this._serveProgressCb = opts.onServeProgress || null
87
88 // 0.5a: content-hash → local disk path, for responding to content-requests
89 this._contentHashPaths = opts.contentHashPaths || new Map()
90 // 0.5a: tree-request timeout (ms)
91 this._treeRequestTimeout = opts.treeRequestTimeout || 30000
92 // Cap on bytes accumulated across a paged treeResponse.
93 this._maxTreeResponseBytes = opts.maxTreeResponseBytes || MAX_TREE_RESPONSE_BYTES
94 // Multi-source: synthetic path 'content:<hash>' → ChunkScheduler. When set,
95 // _onChunkHashes/_onChunkData delegate to the scheduler (parallel fetch
96 // across peers) instead of the legacy single-peer auto-flow.
97 this._schedulers = new Map()
98 // 'content:<hash>' cancelled before its scheduler existed (the caller's
99 // peer-wait window) — fetchContent honors it the moment it creates the scheduler.
100 this._cancelPending = new Set()
101
102 // serve authorization. _serveAuthorizer(peer, from, hash) →
103 // boolean gates every inbound content-request; deny is a silent drop
104 // (indistinguishable from "I don't hold it" — no membership oracle).
105 // _localProfileKey is stamped on outbound content-requests as msg.from.
106 this._serveAuthorizer = opts.serveAuthorizer || null
107 this._localProfileKey = opts.localProfileKey || null
30108 }
31109
32110 /**
33 * Attach to a Protomux instance (from a replicationSwarm connection)
34 * @param {Protomux} mux
35 * @returns {Object} peer handle
111 * Multi-source fetch of a file by content hash from several peers in
112 * parallel (torrent-style). The caller pre-registers _filePaths['content:'
113 * + contentHash] = destPath. Resolves when the file is assembled + verified.
114 *
115 * @param {string} contentHash
116 * @param {Array} peers peer objects to fan the request to
117 * @param {object} [opts] { destPath, cap, timeout, onProgress }
118 * @returns {Promise<void>}
36119 */
120 fetchContent (contentHash, peers, opts = {}) {
121 const p = 'content:' + contentHash
122 // Content-addressed dedup: a concurrent fetch of the SAME hash (a
123 // different file/transfer with identical bytes) joins the in-flight one instead
124 // of failing — await its verified bytes, then copy to our destPath. If the leader
125 // was cancelled, re-issue our own fetch (the scheduler entry is already gone).
126 const inflight = this._schedulers.get(p)
127 if (inflight) {
128 return inflight.shared.then(
129 (res) => {
130 if (opts.destPath && opts.destPath !== inflight.destPath) {
131 try { fs.copyFileSync(inflight.destPath, opts.destPath) } catch (err) { throw err }
132 }
133 return res
134 },
135 (err) => { if (err?.code === 'ECANCELLED') return this.fetchContent(contentHash, peers, opts); throw err },
136 )
137 }
138 const sched = new ChunkScheduler({
139 path: p,
140 destPath: opts.destPath,
141 transfer: this._transferManager,
142 sendNeed: (peer, indices) => peer.msgs.chunkNeed.send({ path: p, indices }),
143 cap: opts.cap,
144 timeout: opts.timeout,
145 onProgress: opts.onProgress,
146 onVerify: opts.onVerify,
147 onEnd: opts.onEnd,
148 onBaseline: (have) => this.sendTransferProgress(contentHash, have),
149 contentHash // verify the whole-file hash incrementally during the transfer
150 })
151 this._schedulers.set(p, sched)
152 // shared promise so joiners (above) can await the same completion —
153 // calling sched.promise() twice would overwrite its resolve/reject handlers.
154 sched.shared = sched.promise().finally(() => this._schedulers.delete(p))
155 // honor a cancel/pause that arrived before this scheduler existed.
156 if (this._cancelPending.delete(p)) { sched.cancel(); return sched.shared }
157 for (const peer of peers) this.requestContent(peer, contentHash, null)
158 return sched.shared
159 }
160
161 // Stop an in-flight multi-source fetch. discardPartial:true unlinks the
162 // partial (cancel); false keeps it for resume (pause). The scheduler's promise
163 // rejects with ECANCELLED and its .finally() removes it from _schedulers. When no
164 // scheduler exists yet (the caller is still in its peer-wait window), remember the
165 // intent so fetchContent cancels the scheduler the instant it is created.
166 async cancelContent (contentHash, { discardPartial = false, signal = true } = {}) {
167 const p = 'content:' + contentHash
168 const sched = this._schedulers.get(p)
169 // No scheduler → the fetch never sent a content-request, so no holder has a serve
170 // row for us; skip the (wasted) broadcast and just record the pre-scheduler cancel.
171 if (!sched) { this._cancelPending.add(p); return false }
172 // Tell holders we paused (keep partial) or stopped (discard) BEFORE local teardown,
173 // so their sender-side indicator reacts now, not after the idle sweep. signal:false
174 // suppresses it (a supersede is a restart, not a user stop).
175 if (signal) this._sendTransferControl(contentHash, discardPartial ? CONTROL_STOPPED : CONTROL_PAUSED)
176 // Stop accepting chunks first so the pause hash-drain converges on a stable frontier.
177 sched.cancel()
178 if (discardPartial) this._transferManager.cancel(sched.destPath)
179 else await this._transferManager.pause(sched.destPath)
180 return true
181 }
182
183 // Tell holders we stopped pulling a hash whose local fetch is already gone (e.g.
184 // discarding an already-paused transfer) — no scheduler to tear down here.
185 sendStopControl (contentHash) { this._sendTransferControl(contentHash, CONTROL_STOPPED) }
186
187 // Best-effort broadcast to all connected holders. A holder with no serve-ledger entry
188 // for us (or no slot for this message) just drops it; wrapped per-peer so a closing
189 // channel or an old peer can't throw.
190 _broadcast (msgName, payload) {
191 for (const [, peer] of this._peers) {
192 try { peer.msgs[msgName]?.send(payload) } catch {}
193 }
194 }
195
196 _sendTransferControl (contentHash, state) { this._broadcast('transferControl', { contentHash, state }) }
197
198 // Tell holders our current on-disk have-bytes for a hash we're (re)fetching, so their
199 // sender-side bar mirrors our TRUE progress rather than only the bytes they re-serve.
200 sendTransferProgress (contentHash, have) { this._broadcast('transferProgress', { contentHash, have }) }
201
202 // Drop a cancel recorded before a scheduler existed when the fetch is
203 // abandoned without ever reaching fetchContent (no peer connected) — otherwise the
204 // marker would cancel the next fetch of the same content.
205 clearCancelPending (contentHash) { this._cancelPending.delete('content:' + contentHash) }
206
207 setContentHashPaths (map) { this._contentHashPaths = map }
208
37209 attach (mux) {
38210 if (this._peers.has(mux)) return this._peers.get(mux)
39211
……@@ -42,98 +214,68 @@ export class OverlayProtocol {
42214 const channel = mux.createChannel({
43215 protocol: PROTOCOL,
44216 id: this._overlayKey || null,
45
46217 onopen () {
47218 const peer = self._peers.get(mux)
48219 if (peer) self._onOpen(peer)
49220 },
50
51221 onclose () {
52 self._onClose(mux)
222 const peer = self._peers.get(mux)
223 self._peers.delete(mux)
224 // Failover: let any active multi-source fetch reassign this peer's
225 // inflight chunks to the remaining peers.
226 if (peer) for (const sched of self._schedulers.values()) sched.removePeer(peer)
227 // Fail in-flight tree requests now, so a mid-stream disconnect rejects
228 // promptly instead of stranding the request (and its buffered pages)
229 // until the idle timeout fires.
230 if (peer) self._failPendingTrees(peer, new Error('channel closed'))
231 // tell the sender-side indicator this peer is gone for every file
232 // it was pulling, so a disconnect mid-download clears its row promptly.
233 if (peer && self._serveEndCb) {
234 for (const [synthPath, from] of peer.authorizedServe) {
235 try { self._serveEndCb({ path: synthPath, peer, from }) } catch {}
236 }
237 }
53238 }
54239 })
55240
56241 if (!channel) return null
57242
58243 const peer = {
59 mux,
60 channel,
61 remoteManifests: new Set(),
62 msgs: {
63 handshake: null,
64 manifestHave: null,
65 manifestRequest: null,
66 manifestResponse: null,
67 chunkBitfield: null,
68 chunkRequest: null,
69 chunkResponse: null,
70 chunkCancel: null,
71 chunkHave: null
72 }
244 mux, channel, remoteFeedKey: null, remoteSeq: 0, msgs: {},
245 pendingTrees: new Map(), // hash → { resolve, reject, timer, timeout, nonce, pages, pageBytes }
246 // synthetic serve-paths this peer was authorized for via a
247 // GATED _onContentRequest, mapped to the requester profileKey (msg.from).
248 // _onChunkNeed serves bytes only for paths in here, so a peer cannot pull
249 // bytes by sending chunkNeed/fileRequest directly for a registered path it
250 // was never authorized to fetch; the value flows the requester identity to
251 // the serve telemetry.
252 authorizedServe: new Map()
73253 }
74254
75 // Register message handlers (order matches message indices 0-8)
76 peer.msgs.handshake = channel.addMessage({
77 encoding: messages.handshake,
78 onmessage: (msg) => {} // handled via channel.handshake
79 })
80
81 peer.msgs.manifestHave = channel.addMessage({
82 encoding: messages.manifestHave,
83 onmessage: (msg) => self._onManifestHave(peer, msg)
84 })
85
86 peer.msgs.manifestRequest = channel.addMessage({
87 encoding: messages.manifestRequest,
88 onmessage: (msg) => self._onManifestRequest(peer, msg)
89 })
90
91 peer.msgs.manifestResponse = channel.addMessage({
92 encoding: messages.manifestResponse,
93 onmessage: (msg) => self._onManifestResponse(peer, msg)
94 })
95
96 peer.msgs.chunkBitfield = channel.addMessage({
97 encoding: messages.chunkBitfield,
98 onmessage: (msg) => self._onChunkBitfield(peer, msg)
99 })
100
101 peer.msgs.chunkRequest = channel.addMessage({
102 encoding: messages.chunkRequest,
103 onmessage: (msg) => self._onChunkRequest(peer, msg)
104 })
105
106 peer.msgs.chunkResponse = channel.addMessage({
107 encoding: messages.chunkResponse,
108 onmessage: (msg) => self._onChunkResponse(peer, msg)
109 })
110
111 peer.msgs.chunkCancel = channel.addMessage({
112 encoding: messages.chunkCancel,
113 onmessage: (msg) => self._onChunkCancel(peer, msg)
114 })
115
116 peer.msgs.chunkHave = channel.addMessage({
117 encoding: messages.chunkHave,
118 onmessage: (msg) => self._onChunkHave(peer, msg)
119 })
120
121 // Message 9: content-addressed chunk fetch (by hash, no manifest).
122 // Peer replies via the existing chunkResponse (message 6).
123 peer.msgs.requestChunkByHash = channel.addMessage({
124 encoding: messages.requestChunkByHash,
125 onmessage: (msg) => self._onRequestChunkByHash(peer, msg)
126 })
255 peer.msgs.syncState = channel.addMessage({ encoding: messages.syncState, onmessage: (msg) => self._onSyncState(peer, msg) })
256 peer.msgs.fileOffer = channel.addMessage({ encoding: messages.fileOffer, onmessage: (msg) => self._onFileOffer(peer, msg) })
257 peer.msgs.fileRequest = channel.addMessage({ encoding: messages.fileRequest, onmessage: (msg) => self._onFileRequest(peer, msg) })
258 peer.msgs.chunkHashes = channel.addMessage({ encoding: messages.chunkHashes, onmessage: (msg) => self._onChunkHashes(peer, msg) })
259 peer.msgs.chunkNeed = channel.addMessage({ encoding: messages.chunkNeed, onmessage: (msg) => self._onChunkNeed(peer, msg) })
260 peer.msgs.chunkData = channel.addMessage({ encoding: messages.chunkData, onmessage: (msg) => self._onChunkData(peer, msg) })
261 peer.msgs.chunkCancel = channel.addMessage({ encoding: messages.chunkCancel, onmessage: (msg) => self._onChunkCancel(peer, msg) })
262 peer.msgs.transferComplete = channel.addMessage({ encoding: messages.transferComplete, onmessage: (msg) => self._onTransferComplete(peer, msg) })
263 peer.msgs.conflict = channel.addMessage({ encoding: messages.conflict, onmessage: (msg) => self._onConflictMsg(peer, msg) })
264
265 // 0.5a — tree messages (backward-compatible additions)
266 peer.msgs.treeRequest = channel.addMessage({ encoding: messages.treeRequest, onmessage: (msg) => self._onTreeRequest(peer, msg) })
267 peer.msgs.treeResponse = channel.addMessage({ encoding: messages.treeResponse, onmessage: (msg) => self._onTreeResponse(peer, msg) })
268 peer.msgs.contentRequest = channel.addMessage({ encoding: messages.contentRequest, onmessage: (msg) => self._onContentRequest(peer, msg) })
269 // Appended-last slots: ids 0-11 stay fixed for peers without 12/13, which just
270 // never register these channels and ignore the frames.
271 peer.msgs.transferControl = channel.addMessage({ encoding: messages.transferControl, onmessage: (msg) => self._onTransferControl(peer, msg) })
272 peer.msgs.transferProgress = channel.addMessage({ encoding: messages.transferProgress, onmessage: (msg) => self._onTransferProgress(peer, msg) })
127273
128274 this._peers.set(mux, peer)
129 channel.open()
130
275 channel.open({ version: VERSION, capabilities: CAP_LOCAL_FILES | CAP_ADAPTIVE_CHUNKS })
131276 return peer
132277 }
133278
134 /**
135 * Detach from a Protomux instance
136 */
137279 detach (mux) {
138280 const peer = this._peers.get(mux)
139281 if (!peer) return
……@@ -141,258 +283,692 @@ export class OverlayProtocol {
141283 this._peers.delete(mux)
142284 }
143285
144 /**
145 * Request a chunk from any connected peer
146 * @param {string} manifestHash
147 * @param {number} index
148 * @returns {Promise<Buffer|null>}
149 */
150 async requestChunk (manifestHash, index) {
151 // Find a peer that has this manifest
152 for (const [, peer] of this._peers) {
153 if (!peer.remoteManifests.has(manifestHash)) continue
154
155 const requestId = this._nextRequestId++
156 const promise = new Promise((resolve, reject) => {
157 const timeout = setTimeout(() => {
158 this._pendingRequests.delete(requestId)
159 reject(new Error(`Chunk request ${requestId} timed out`))
160 }, this._requestTimeout)
161
162 this._pendingRequests.set(requestId, { resolve, reject, timeout })
163 })
286 setFilePaths (map) { this._filePaths = map }
164287
165 peer.msgs.chunkRequest.send({
166 manifestHash,
167 index,
168 requestId,
169 priority: 1
170 })
171
172 return promise
288 broadcastOffer (overlayPath, meta, op) {
289 const msg = {
290 path: overlayPath,
291 contentHash: meta.contentHash || '0'.repeat(64),
292 size: meta.size || 0,
293 mtime: meta.mtime || 0,
294 op
173295 }
296 for (const [, peer] of this._peers) {
297 peer.msgs.fileOffer.send(msg)
298 }
299 }
174300
175 return null // No peer has this manifest
301 requestFile (peer, overlayPath, contentHash, chunksHave) {
302 peer.msgs.fileRequest.send({
303 path: overlayPath,
304 contentHash,
305 chunksHave: chunksHave || null
306 })
176307 }
177308
178309 /**
179 * Request a chunk directly by its content hash, no manifest required.
180 * Asks every connected peer in turn until one responds with the bytes.
310 * 0.5a: Request a tree by its hash. Returns a promise resolving to
311 * the entries array (or [] if the peer doesn't have it).
181312 *
182 * @param {string} hash - blake2b chunk hash (hex)
183 * @returns {Promise<Buffer|null>}
313 * @param {object} peer
314 * @param {string} hash - hex tree hash
315 * @param {object} [opts]
316 * @param {number} [opts.timeout] - override default timeout (ms)
317 * @returns {Promise<Array<{kind, exec, name, childHash, size}>>}
184318 */
185 async requestChunkByHash (hash) {
186 for (const [, peer] of this._peers) {
187 const requestId = this._nextRequestId++
188 const promise = new Promise((resolve, reject) => {
189 const timeout = setTimeout(() => {
190 this._pendingRequests.delete(requestId)
191 reject(new Error(`Chunk-by-hash request ${requestId} timed out`))
192 }, this._requestTimeout)
193
194 this._pendingRequests.set(requestId, { resolve, reject, timeout })
195 })
196
197 peer.msgs.requestChunkByHash.send({ hash, requestId })
319 requestTree (peer, hash, opts = {}) {
320 return new Promise((resolve, reject) => {
321 // If a prior request is already in-flight for the same hash, reject
322 // this caller rather than silently overwrite.
323 if (peer.pendingTrees.has(hash)) {
324 return reject(new Error('tree request already pending for hash: ' + hash))
325 }
326 // Per-request nonce (per-peer monotonic, non-zero) lets the requester drop
327 // pages echoed for a superseded attempt. pages/pageBytes buffer this
328 // request's own paged response, so a fresh attempt never inherits an old one.
329 const nonce = peer._treeReqSeq = (peer._treeReqSeq || 0) + 1
330 const timeout = opts.timeout || this._treeRequestTimeout
331 const pending = { resolve, reject, timer: null, timeout, nonce, pages: null, pageBytes: 0 }
332 this._armTreeTimeout(peer, hash, pending)
333 peer.pendingTrees.set(hash, pending)
334 peer.msgs.treeRequest.send({ hash, nonce })
335 })
336 }
198337
199 let result
200 try {
201 result = await promise
202 } catch {
203 continue // timeout — try next peer
338 // Arm (or re-arm) a pending tree request's idle timeout. Every buffered page
339 // re-arms it (a large tree on a slow link mustn't time out mid-stream); the
340 // request still settles because the accumulator is byte-capped in _onTreeResponse.
341 _armTreeTimeout (peer, hash, pending) {
342 pending.timer = setTimeout(() => {
343 if (peer.pendingTrees.get(hash) === pending) {
344 peer.pendingTrees.delete(hash)
345 pending.reject(new Error('tree request timed out: ' + hash))
204346 }
347 }, pending.timeout)
348 }
205349
206 if (result) return result
350 // Reject every in-flight tree request on this peer (channel close / destroy),
351 // clearing timers so the promise fails promptly and its buffered pages release.
352 _failPendingTrees (peer, err) {
353 for (const [hash, pending] of peer.pendingTrees) {
354 clearTimeout(pending.timer)
355 peer.pendingTrees.delete(hash)
356 pending.reject(err)
207357 }
208
209 return null
210358 }
211359
212360 /**
213 * Request a manifest from any connected peer
214 * @param {string} hash
215 * @returns {Promise<Buffer|null>}
361 * 0.5a: Request a file by its content hash. Peer looks up any file
362 * matching that hash locally and streams chunks via the existing flow
363 * (chunkHashes → chunkNeed → chunkData). Returns immediately; chunks
364 * arrive asynchronously.
365 *
366 * Note: receiver must pre-register a target path mapping via
367 * _filePaths keyed by the contentHash before calling, OR this method
368 * is a no-op on the receive side. Full wiring deferred to 0.5c.
216369 */
217 async requestManifest (hash) {
370 requestContent (peer, contentHash, chunksHave) {
371 peer.msgs.contentRequest.send({
372 contentHash,
373 chunksHave: chunksHave || null,
374 from: this._localProfileKey || '' // authenticate the asker to the serve gate
375 })
376 }
377
378 get peerCount () { return this._peers.size }
379
380 destroy () {
218381 for (const [, peer] of this._peers) {
219 if (!peer.remoteManifests.has(hash)) continue
382 this._failPendingTrees(peer, new Error('protocol destroyed'))
383 peer.channel.close()
384 }
385 this._peers.clear()
386 }
220387
221 const requestId = this._nextRequestId++
222 const promise = new Promise((resolve, reject) => {
223 const timeout = setTimeout(() => {
224 this._pendingRequests.delete(requestId)
225 reject(new Error(`Manifest request ${requestId} timed out`))
226 }, this._requestTimeout)
388 // ── Handlers ────────────────────────────────────────────────
227389
228 this._pendingRequests.set(requestId, { resolve, reject, timeout })
229 })
390 _onOpen (peer) {
391 peer.msgs.syncState.send({
392 feedKey: this._syncEngine.feedKey,
393 localSeq: this._syncEngine.feed.length,
394 remoteSeq: 0
395 })
396 }
397
398 async _onSyncState (peer, msg) {
399 peer.remoteFeedKey = msg.feedKey
400 peer.remoteSeq = msg.localSeq
230401
231 peer.msgs.manifestRequest.send({ hash, requestId })
232 return promise
402 if (!this._autoSync) return
403
404 try {
405 await this._offerLocalFiles(peer)
406 } catch {
407 // ignore — the HyperOverlay facade runs with autoSync off, so this path is dormant
233408 }
409 }
234410
235 return null
411 async _onFileOffer (peer, msg) {
412 // In gated mode the overlay flow is contentRequest→scheduler ONLY; gated
413 // consumers never act on path-based file-offers (autoSync is off). Ignoring
414 // them closes the offer→requestFile→legacy-receive path a peer could drive.
415 if (this._serveAuthorizer) return
416 if (this._fileOfferCb) this._fileOfferCb(peer, msg)
417
418 if (msg.op === 0) {
419 const local = await this._syncEngine._fileIndex.getFile(msg.path)
420 if (!local || local.contentHash !== msg.contentHash) {
421 const diskPath = this._resolveFilePath(msg.path)
422 if (diskPath) {
423 this._filePaths.set(msg.path, diskPath)
424 this.requestFile(peer, msg.path, msg.contentHash)
425 }
426 }
427 }
236428 }
237429
238 // ── Internal handlers ───────────────────────────────────────
430 async _onFileRequest (peer, msg) {
431 // path-based serve is UNGATED and would bypass the serve gate (the only
432 // gated entry point is _onContentRequest). Gated consumers never send
433 // fileRequest, so refuse it outright in gated mode.
434 if (this._serveAuthorizer) return
435 const diskPath = this._filePaths.get(msg.path)
436 if (!diskPath) return
239437
240 async _onOpen (peer) {
241 // Send our manifest list
242 const stats = await this._registry.stats()
243 const manifests = []
438 const prepared = await this._transferManager.prepareFile(diskPath, msg.path)
439 if (!prepared) return
244440
245 for await (const entry of this._registry._bee.createReadStream({
246 gt: 'manifest:', lt: 'manifest:\xff'
247 })) {
248 manifests.push(entry.key.slice('manifest:'.length))
441 if (!await this._transferManager._fileIndex.hasChunkMap(msg.path)) {
442 await this._transferManager._fileIndex.putChunkMap(msg.path, prepared.chunks)
249443 }
250444
251 if (manifests.length > 0) {
252 peer.msgs.manifestHave.send({ hashes: manifests })
445 this._sendChunkHashes(peer, msg.path, prepared.tier, prepared.chunks)
446 }
447
448 // Send a file's chunk list, paged so no single frame exceeds
449 // the Noise transport's 16 MiB-1 atomic-write limit. A list that fits in one
450 // frame ships as one (more:0) — identical on the wire to the pre-paging format
451 // bar a trailing 0 byte. _onChunkHashes reassembles the pages on the receiver.
452 _sendChunkHashes (peer, filePath, tier, chunks) {
453 const total = chunks.length
454 if (total <= MAX_CHUNKS_PER_MSG) {
455 peer.msgs.chunkHashes.send({ path: filePath, tier, chunks, more: 0 })
456 return
457 }
458 for (let i = 0; i < total; i += MAX_CHUNKS_PER_MSG) {
459 const more = i + MAX_CHUNKS_PER_MSG < total ? 1 : 0
460 peer.msgs.chunkHashes.send({ path: filePath, tier, chunks: chunks.slice(i, i + MAX_CHUNKS_PER_MSG), more })
253461 }
254462 }
255463
256 _onClose (mux) {
257 this._peers.delete(mux)
464 // Reassemble a paged chunk list. Returns the full list once
465 // the final page (more:0) arrives, or null while pages are still pending. A
466 // lone complete frame passes straight through (no copy) — the small-file norm.
467 // Pages for a path arrive in order on this channel, so arrival-order
468 // concatenation is correct even when several files interleave on it.
469 _reassembleChunkHashes (peer, msg) {
470 const buffered = peer._chunkHashPages && peer._chunkHashPages.get(msg.path)
471 if (!buffered && !msg.more) return msg.chunks
472 if (!peer._chunkHashPages) peer._chunkHashPages = new Map()
473 let acc = buffered
474 if (!acc) { acc = []; peer._chunkHashPages.set(msg.path, acc) }
475 for (const ch of msg.chunks) acc.push(ch)
476 if (msg.more) return null
477 peer._chunkHashPages.delete(msg.path)
478 return acc
258479 }
259480
260 _onManifestHave (peer, msg) {
261 for (const hash of msg.hashes) {
262 peer.remoteManifests.add(hash)
263 }
481 // Upper-bound wire size of one tree entry: kind(1)+exec(1) + string(len-prefix ≤4
482 // + utf8 name bytes) + childHash(32) + size varint(≤9, c.uint is 9 bytes for a
483 // >4 GiB size).
484 _treeEntryWireSize (e) {
485 return 1 + 1 + 4 + Buffer.byteLength(e.name || '', 'utf8') + 32 + 9
264486 }
265487
266 async _onManifestRequest (peer, msg) {
267 const data = await this._registry.getManifest(msg.hash)
268 peer.msgs.manifestResponse.send({
269 hash: msg.hash,
270 requestId: msg.requestId,
271 data: data || Buffer.alloc(0),
272 found: data !== null
273 })
488 // Send a tree's entries, paged so no single frame exceeds the Noise atomic-write
489 // limit; every page but the last is more:1. A tree that fits in one frame ships
490 // as a single more:0 frame. `nonce` echoes the requester's per-request id.
491 _sendTreeResponse (peer, hash, entries, nonce) {
492 let page = []
493 let bytes = 40 // hash(32) + uint32 count prefix + more/nonce + slack
494 for (const e of entries) {
495 const sz = this._treeEntryWireSize(e)
496 if (page.length && bytes + sz > MAX_TREE_BYTES_PER_MSG) {
497 peer.msgs.treeResponse.send({ hash, entries: page, more: 1, nonce })
498 page = []
499 bytes = 40
500 }
501 page.push(e)
502 bytes += sz
503 }
504 peer.msgs.treeResponse.send({ hash, entries: page, more: 0, nonce })
274505 }
275506
276 _onManifestResponse (peer, msg) {
277 const pending = this._pendingRequests.get(msg.requestId)
278 if (!pending) return
507 async _onChunkHashes (peer, msg) {
508 // Reassemble paged chunkHashes frames before dispatching.
509 const chunks = this._reassembleChunkHashes(peer, msg)
510 if (chunks === null) {
511 // A buffered page is forward progress — keep the scheduler's no-progress
512 // watchdog from tripping while a large map streams across several frames.
513 const pending = this._schedulers.get(msg.path)
514 if (pending && pending.notePageProgress) pending.notePageProgress()
515 return
516 }
517 // Multi-source: if a scheduler owns this path, let it distribute chunk
518 // requests across peers instead of the legacy single-peer auto-flow.
519 const sched = this._schedulers.get(msg.path)
520 if (sched) { Promise.resolve(sched.onChunkHashes(peer, chunks)).catch(() => {}); return }
521 // the legacy single-peer receive below writes incoming bytes
522 // to _filePaths.get(path) and finalizes a rename OVER it. For a path that
523 // maps to one of OUR own files (e.g. a content-register entry), an
524 // unsolicited chunkHashes would let a peer overwrite the owner's source
525 // file. Gated consumers only ever receive via a scheduler (the branch
526 // above), so refuse any other receive.
527 if (this._serveAuthorizer) return
528
529 const diskPath = this._filePaths.get(msg.path)
530 if (!diskPath) return
531
532 // Empty-file fast path: a 0-byte file has no chunks to fetch, but we
533 // still need to materialize it on disk and fire the same completion
534 // flow as _onChunkData's "transfer complete" branch — otherwise the
535 // requester (e.g. NetworkFetcher.getContent) never resolves and times
536 // out. Empty files are common in practice: .keep, __init__.py, build
537 // stamps, touch'd markers. Bug symptom before this: node-gyp's
538 // postinstall.stamp (0 bytes, hash 5187b7a8…) hung every install.
539 if (chunks.length === 0) {
540 try {
541 fs.mkdirSync(path.dirname(diskPath), { recursive: true })
542 fs.writeFileSync(diskPath, Buffer.alloc(0))
543 } catch {
544 return
545 }
546 const contentHash = hashChunk(Buffer.alloc(0))
547 const stat = fs.statSync(diskPath)
548 await this._syncEngine._fileIndex.putFile(msg.path, {
549 contentHash, size: 0, mtime: stat.mtimeMs
550 })
551 await this._syncEngine.logPut(msg.path, {
552 contentHash, size: 0, mtime: stat.mtimeMs
553 })
554 if (peer.remoteFeedKey) {
555 await this._syncEngine.markSynced(peer.remoteFeedKey, msg.path, contentHash, this._syncEngine.feed.length - 1)
556 }
557 peer.msgs.transferComplete.send({ path: msg.path, contentHash })
558 if (this._syncedCb) {
559 this._syncedCb({ path: msg.path, contentHash, size: 0, direction: 'pull' })
560 }
561 return
562 }
563
564 const needed = this._transferManager.computeNeeded(chunks, new Set())
565 if (needed.length === 0) return
279566
280 clearTimeout(pending.timeout)
281 this._pendingRequests.delete(msg.requestId)
567 let size = 0
568 for (const c of chunks) size += c.length
282569
283 if (msg.found && msg.data) {
284 pending.resolve(msg.data)
285 } else {
286 pending.resolve(null)
570 // startReceive can throw (fd open, partial recovery) — drop this transfer
571 // instead of rejecting protomux's onmessage promise, which would destroy the
572 // whole Noise stream.
573 try {
574 await this._transferManager.startReceive(diskPath, {
575 size,
576 chunks: chunks.map((c, i) => {
577 let offset = 0
578 for (let j = 0; j < i; j++) offset += chunks[j].length
579 return { hash: c.hash, offset, length: c.length }
580 })
581 })
582 } catch {
583 return
287584 }
585
586 peer.msgs.chunkNeed.send({ path: msg.path, indices: needed })
288587 }
289588
290 _onChunkBitfield (peer, msg) {
291 if (!peer.bitfields) peer.bitfields = new Map()
292 peer.bitfields.set(msg.manifestHash, msg.bitfield)
589 async _onChunkNeed (peer, msg) {
590 // Serve bytes ONLY for a synthetic path THIS peer was
591 // authorized for via a gated _onContentRequest. Blocks a peer pulling bytes
592 // by sending chunkNeed directly for a registered path (e.g. a content-
593 // register entry) without passing the serve gate.
594 if (this._serveAuthorizer && !peer.authorizedServe.has(msg.path)) return
595 const diskPath = this._filePaths.get(msg.path)
596 if (!diskPath) return
597
598 const fileIndex = this._transferManager._fileIndex
599 const chunkMap = msg.path.startsWith('content:')
600 ? await fileIndex.getChunkMapByHash(msg.path.slice('content:'.length))
601 : await fileIndex.getChunkMap(msg.path)
602 if (!chunkMap) return
603
604 // Single-flight serve pump per peer (mirrors hypercore's `receiverBusy`).
605 // Enqueue this request; if no pump is running for this peer, drain the queue
606 // one job at a time. protomux dispatches every frame of a socket read
607 // synchronously, so two chunkNeed frames from the same peer would otherwise
608 // start rival serve loops — interleaving sends and admitting N chunks per
609 // drain instead of honouring backpressure once (the discipline our 0.2.13
610 // serialized pump had, which Ok's inline loop dropped).
611 if (!peer._serveQueue) peer._serveQueue = []
612 peer._serveQueue.push({ path: msg.path, diskPath, chunkMap, indices: msg.indices })
613 if (peer._serving) return
614 peer._serving = true
615 try {
616 while (peer._serveQueue.length) {
617 if (peer.channel && peer.channel.closed) { peer._serveQueue.length = 0; break }
618 try { await this._serveJob(peer, peer._serveQueue.shift()) } catch {}
619 }
620 } finally {
621 peer._serving = false
622 }
623 }
624
625 // Serve one chunkNeed job: stream the requested chunk indices, honouring
626 // stream backpressure — stop producing when send() reports the write buffer is
627 // full and resume on drain, so sibling protocols multiplexed on the same Noise
628 // stream (Corestore replication, app handshakes; protomux MAX_BACKLOG is
629 // Infinity) aren't starved. Runs under the per-peer single-flight guard in
630 // _onChunkNeed, so exactly one job streams at a time per peer.
631 async _serveJob (peer, job) {
632 const { path, diskPath, chunkMap, indices } = job
633 for (let i = 0; i < indices.length; i++) {
634 const index = indices[i]
635 if (index >= chunkMap.length) continue
636 const c = chunkMap[index]
637 const data = this._transferManager.readChunk(diskPath, c.offset, c.length)
638 if (!data) continue
639 const flushed = peer.msgs.chunkData.send({ path, index, data })
640 if (this._chunkServeCb) {
641 try { this._chunkServeCb({ path, index, bytes: data.length, peer, from: peer.authorizedServe.get(path) || null }) } catch {}
642 }
643 if (flushed === false && i < indices.length - 1) {
644 const alive = await this._waitForDrain(peer)
645 if (!alive) return
646 if (this._serveAuthorizer && !peer.authorizedServe.has(path)) return
647 }
648 }
293649 }
294650
295 async _onChunkRequest (peer, msg) {
296 const notFound = () => peer.msgs.chunkResponse.send({
297 requestId: msg.requestId,
298 data: Buffer.alloc(0),
299 found: false
651 // One shared drain waiter per peer so concurrent serve loops (the
652 // same peer pulling several files) don't each attach listeners and trip the stream's
653 // max-listeners warning. Resolves true on drain, false if the channel closes first.
654 _waitForDrain (peer) {
655 const channel = peer.channel
656 const stream = peer.mux && peer.mux.stream
657 if (!stream || !channel || channel.closed) return Promise.resolve(false)
658 if (channel.drained) return Promise.resolve(true)
659 if (peer._drainWait) return peer._drainWait
660 peer._drainWait = new Promise((resolve) => {
661 let timer = null
662 const done = (alive) => {
663 if (timer) clearTimeout(timer)
664 stream.removeListener('drain', onDrain)
665 stream.removeListener('close', onClose)
666 peer._drainWait = null
667 resolve(alive)
668 }
669 const onDrain = () => done(!channel.closed)
670 const onClose = () => done(false)
671 timer = setTimeout(() => done(false), DRAIN_TIMEOUT_MS)
672 stream.on('drain', onDrain)
673 stream.on('close', onClose)
300674 })
675 return peer._drainWait
676 }
301677
302 // Resolve chunk hash from manifest + index
303 const manifestData = await this._registry.getManifest(msg.manifestHash)
304 if (!manifestData) return notFound()
678 async _onChunkData (peer, msg) {
679 // Silent-peer watchdog hook: fire BEFORE any processing so a slow disk
680 // doesn't get blamed for a peer-side stall.
681 if (this._chunkProgressCb) {
682 try {
683 this._chunkProgressCb({
684 path: msg.path,
685 index: msg.index,
686 bytes: msg.data ? msg.data.length : 0,
687 peer
688 })
689 } catch {}
690 }
305691
306 let chunkHash
307 try {
308 const manifest = decodeManifest(manifestData)
309 if (msg.index >= manifest.chunks.length) return notFound()
310 chunkHash = manifest.chunks[msg.index].hash
311 } catch {
312 return notFound()
692 // Multi-source: scheduler owns the write + reschedule + finalize.
693 const sched = this._schedulers.get(msg.path)
694 if (sched) { Promise.resolve(sched.onChunkData(peer, msg.index, msg.data)).catch(() => {}); return }
695 // refuse the legacy single-peer receive (writes+renames over
696 // _filePaths.get(path), i.e. potentially our own source file). Gated
697 // consumers only receive via a scheduler (above).
698 if (this._serveAuthorizer) return
699
700 const diskPath = this._filePaths.get(msg.path)
701 if (!diskPath) return
702
703 const result = this._transferManager.writeChunk(diskPath, msg.index, msg.data)
704
705 if (result.ok && this._transferManager.isComplete(diskPath)) {
706 const fin = await this._transferManager.finalize(diskPath)
707 if (fin.ok) {
708 const fileData = fs.readFileSync(diskPath)
709 const contentHash = hashChunk(fileData)
710 const stat = fs.statSync(diskPath)
711
712 await this._syncEngine._fileIndex.putFile(msg.path, {
713 contentHash, size: stat.size, mtime: stat.mtimeMs
714 })
715 await this._syncEngine.logPut(msg.path, {
716 contentHash, size: stat.size, mtime: stat.mtimeMs
717 })
718
719 if (peer.remoteFeedKey) {
720 await this._syncEngine.markSynced(peer.remoteFeedKey, msg.path, contentHash, this._syncEngine.feed.length - 1)
721 }
722
723 peer.msgs.transferComplete.send({ path: msg.path, contentHash })
724 if (this._syncedCb) {
725 this._syncedCb({ path: msg.path, contentHash, size: stat.size, direction: 'pull' })
726 }
727 }
313728 }
729 }
314730
315 // Fetch chunk data by its content hash
316 const data = await this._registry.getChunkData(chunkHash)
317 if (!data) return notFound()
731 _onChunkCancel (peer, msg) {
732 const diskPath = this._filePaths.get(msg.path)
733 if (diskPath) this._transferManager.cancel(diskPath)
734 }
318735
319 peer.msgs.chunkResponse.send({
320 requestId: msg.requestId,
321 data,
322 found: true
323 })
736 // Resolve a serve-message's downloader identity from the authenticated record we kept at
737 // serve start (peer.authorizedServe), NOT the message — so a peer can only affect its own
738 // ledger row, never spoof another. Returns null `from` when the peer was never authorized.
739 _authorizedFrom (peer, contentHash) {
740 const synthPath = 'content:' + contentHash
741 return { synthPath, from: peer.authorizedServe.get(synthPath) || null }
324742 }
325743
326 _onChunkResponse (peer, msg) {
327 const pending = this._pendingRequests.get(msg.requestId)
328 if (!pending) return
744 // A downloader told us it paused/stopped pulling a hash WE serve.
745 _onTransferControl (peer, msg) {
746 if (!this._serveControlCb) return
747 const { synthPath, from } = this._authorizedFrom(peer, msg.contentHash)
748 if (!from) return
749 const state = msg.state === CONTROL_STOPPED ? 'stopped' : 'paused'
750 try { this._serveControlCb({ path: synthPath, peer, from, state }) } catch {}
751 }
329752
330 clearTimeout(pending.timeout)
331 this._pendingRequests.delete(msg.requestId)
753 // A downloader reported its on-disk have-bytes for a hash WE serve.
754 _onTransferProgress (peer, msg) {
755 if (!this._serveProgressCb) return
756 const { synthPath, from } = this._authorizedFrom(peer, msg.contentHash)
757 if (!from) return
758 try { this._serveProgressCb({ path: synthPath, peer, from, have: msg.have || 0 }) } catch {}
759 }
332760
333 if (msg.found && msg.data) {
334 pending.resolve(msg.data)
335 } else {
336 pending.resolve(null)
761 async _onTransferComplete (peer, msg) {
762 if (peer.remoteFeedKey) {
763 await this._syncEngine.markSynced(peer.remoteFeedKey, msg.path, msg.contentHash, this._syncEngine.feed.length - 1)
764 }
765 if (this._syncedCb) {
766 this._syncedCb({ path: msg.path, contentHash: msg.contentHash, direction: 'push' })
337767 }
338768 }
339769
340 _onChunkCancel (peer, msg) {
341 // Peer cancelled a request — nothing to do on our side
770 _onConflictMsg (peer, msg) {
771 if (this._conflictCb) this._conflictCb(msg, peer)
342772 }
343773
344 async _onRequestChunkByHash (peer, msg) {
345 const notFound = () => peer.msgs.chunkResponse.send({
346 requestId: msg.requestId,
347 data: Buffer.alloc(0),
348 found: false
349 })
774 // ── Tree handlers (0.5a) ────────────────────────────────────
350775
351 const data = await this._registry.getChunkData(msg.hash)
352 if (!data) return notFound()
776 async _onTreeRequest (peer, msg) {
777 // ungated metadata oracle (returns fileIndex tree entries to
778 // any peer). Gated consumers never use tree-requests; refuse in gated mode.
779 if (this._serveAuthorizer) return
780 let entries = []
781 try {
782 const fileIndex = this._syncEngine && this._syncEngine._fileIndex
783 if (fileIndex && typeof fileIndex.getTree === 'function') {
784 const tree = await fileIndex.getTree(msg.hash)
785 if (tree && Array.isArray(tree.entries)) entries = tree.entries
786 }
787 } catch {
788 // fall through — reply with empty entries on lookup failure
789 }
790 this._sendTreeResponse(peer, msg.hash, entries, msg.nonce || 0)
791 }
353792
354 peer.msgs.chunkResponse.send({
355 requestId: msg.requestId,
356 data,
357 found: true
358 })
793 _onTreeResponse (peer, msg) {
794 const pending = peer.pendingTrees.get(msg.hash)
795 if (!pending) return // no request in flight — drop
796 // Drop a page echoing a superseded request's nonce (a stale page from a
797 // timed-out attempt must not settle a later same-hash retry). nonce 0 = a
798 // pre-nonce holder that can't echo it, so fall through (unverifiable,
799 // best-effort).
800 if (msg.nonce && msg.nonce !== pending.nonce) return
801 const settle = (fn, arg) => { clearTimeout(pending.timer); peer.pendingTrees.delete(msg.hash); fn(arg) }
802 // Lone complete frame (the small-tree norm) resolves with no accumulation.
803 if (!pending.pages && !msg.more) return settle(pending.resolve, msg.entries)
804 // Accumulate into THIS request's own buffer, bounding total bytes so a peer
805 // streaming pages endlessly rejects the request instead of growing memory
806 // without limit.
807 if (!pending.pages) pending.pages = []
808 for (const e of msg.entries) { pending.pages.push(e); pending.pageBytes += this._treeEntryWireSize(e) }
809 if (pending.pageBytes > this._maxTreeResponseBytes) {
810 return settle(pending.reject, new Error('tree response too large: ' + msg.hash))
811 }
812 if (msg.more) {
813 // Forward progress — re-arm the idle timeout so a legitimately large
814 // multi-page tree on a slow link doesn't time out mid-stream.
815 clearTimeout(pending.timer)
816 this._armTreeTimeout(peer, msg.hash, pending)
817 return
818 }
819 settle(pending.resolve, pending.pages)
359820 }
360821
361 _onChunkHave (peer, msg) {
362 if (!peer.bitfields) peer.bitfields = new Map()
363 // Merge incremental bitfield update into existing
364 const existing = peer.bitfields.get(msg.manifestHash)
365 if (existing && msg.bitfield) {
366 // Overlay new bits starting at startIndex
367 const merged = Buffer.from(existing)
368 const startByte = Math.floor(msg.startIndex / 8)
369 msg.bitfield.copy(merged, startByte, 0, Math.min(msg.bitfield.length, merged.length - startByte))
370 peer.bitfields.set(msg.manifestHash, merged)
371 } else if (msg.bitfield) {
372 peer.bitfields.set(msg.manifestHash, msg.bitfield)
822 async _onContentRequest (peer, msg) {
823 // serve gate. BEFORE any path resolution or streaming,
824 // ask the injected authorizer whether this peer may receive this hash.
825 // Deny == silent return (same observable behavior as "I don't hold it"
826 // below), so a non-member cannot distinguish denial from absence.
827 if (this._serveAuthorizer) {
828 let ok = false
829 try { ok = await this._serveAuthorizer(peer, msg.from || null, msg.contentHash) } catch { ok = false }
830 if (!ok) return
831 }
832
833 // Resolve msg.contentHash → an on-disk path. Three lookup paths, in order:
834 // 1. Fast path: _contentHashPaths map (warm — just-registered / consumer-set)
835 // 2. Content-addressed spool (RESTART-DURABLE): <syncBaseDir>/<contentHash>
836 // 3. Slow path: scan fileIndex for any file:* entry with that hash
837 let diskPath = this._contentHashPaths.get(msg.contentHash)
838 if (diskPath && !fileExists(diskPath)) diskPath = null
839
840 if (!diskPath) {
841 // Content-addressed spool fallback. The canonical bytes are stored at
842 // <syncBaseDir>/<contentHash> (fetchFile's mirrorSpool option writes them
843 // there). The in-memory map is empty after a restart and FileIndex
844 // persists no disk path, so resolve the hash straight from the
845 // content-addressed spool — a pure function of the hash + dir, like a
846 // git object store's blobs/<hash>. This is what lets a node keep serving
847 // its spooled files across restarts with no rehydration step.
848 for (const baseDir of this._syncBaseDirs) {
849 const candidate = path.join(baseDir, msg.contentHash)
850 if (fileExists(candidate)) {
851 diskPath = candidate
852 this._contentHashPaths.set(msg.contentHash, candidate) // warm the map
853 break
854 }
855 }
856 }
857
858 if (!diskPath) {
859 // Scan fileIndex for a file whose contentHash matches.
860 const fileIndex = this._syncEngine && this._syncEngine._fileIndex
861 if (fileIndex && typeof fileIndex.listFiles === 'function') {
862 try {
863 const files = await fileIndex.listFiles()
864 for (const f of files) {
865 if (f.contentHash === msg.contentHash) {
866 diskPath = this._filePaths.get(f.path)
867 if ((!diskPath || !fileExists(diskPath)) && f.path) diskPath = f.path
868 break
869 }
870 }
871 } catch {
872 // fall through
873 }
874 }
875 }
876
877 if (!diskPath) {
878 // Don't have it. Silently drop; requester will time out.
879 return
880 }
881
882 // Use a SYNTHETIC path keyed by contentHash so the requester can
883 // pre-register a destPath and the chunk flow routes correctly on
884 // both sides. Temporarily populate _filePaths so _onChunkNeed +
885 // chunkmap lookups resolve under this synthetic key.
886 const syntheticPath = 'content:' + msg.contentHash
887 this._filePaths.set(syntheticPath, diskPath)
888 peer.authorizedServe.set(syntheticPath, msg.from || null) // _onChunkNeed will now serve this path to THIS peer
889
890 // The chunk map is content-addressed — identical bytes always chunk
891 // identically. Reuse one computed at publish (or a prior serve, or before a
892 // restart) instead of re-reading the whole file to re-chunk it: re-chunking a
893 // multi-GB file adds a full-file-read of latency before the first byte ships.
894 const fileIndex = this._transferManager._fileIndex
895 let chunks = await fileIndex.getChunkMapByHash(msg.contentHash)
896 let tier
897 if (chunks) {
898 let size = 0
899 try { size = fs.statSync(diskPath).size } catch {}
900 tier = selectTier(size)
901 } else {
902 const prepared = await this._transferManager.prepareFile(diskPath, syntheticPath, { byHashOnly: true })
903 if (!prepared) {
904 this._filePaths.delete(syntheticPath)
905 peer.authorizedServe.delete(syntheticPath) // keep the authorize map in lockstep with _filePaths
906 return
907 }
908 chunks = prepared.chunks
909 tier = prepared.tier
910 // prepareFile persists the hash-keyed map only for large files; ensure it exists
911 // for small ones too, since _onChunkNeed resolves content:<hash> by that map.
912 if (!await fileIndex.hasChunkMapByHash(msg.contentHash)) await fileIndex.putChunkMapByHash(msg.contentHash, chunks)
913 }
914 this._sendChunkHashes(peer, syntheticPath, tier, chunks)
915 if (this._serveStartCb) {
916 let total = 0
917 for (const c of chunks) total += c.length || 0
918 try { this._serveStartCb({ path: syntheticPath, peer, from: msg.from || null, total }) } catch {}
373919 }
374920 }
375921
922 // ── Sync logic ──────────────────────────────────────────────
923
376924 /**
377 * Get connected peer count
925 * Offer all local files to a peer.
926 * Reads from the local feed and sends file-offers for each entry.
927 * The peer will request any files it doesn't have.
378928 */
379 get peerCount () {
380 return this._peers.size
929 async _offerLocalFiles (peer) {
930 const feed = this._syncEngine.feed
931 if (feed.length === 0) return
932
933 for (let i = 0; i < feed.length; i++) {
934 try {
935 const entry = await feed.get(i)
936 if (!entry) continue
937
938 // Register the file path mapping so _onFileRequest can find it
939 if (entry.op === 0) {
940 const diskPath = this._resolveFilePath(entry.path)
941 if (diskPath) this._filePaths.set(entry.path, diskPath)
942 }
943
944 peer.msgs.fileOffer.send({
945 path: entry.path,
946 contentHash: entry.hash || '0'.repeat(64),
947 size: entry.size || 0,
948 mtime: entry.mtime || 0,
949 op: entry.op
950 })
951 } catch {
952 // skip unreadable entries
953 }
954 }
381955 }
382956
383957 /**
384 * Destroy all connections
958 * Resolve an overlay path to a disk path.
385959 */
386 destroy () {
387 for (const [mux, peer] of this._peers) {
388 peer.channel.close()
389 }
390 this._peers.clear()
391
392 for (const [, req] of this._pendingRequests) {
393 clearTimeout(req.timeout)
394 req.reject(new Error('Protocol destroyed'))
960 _resolveFilePath (overlayPath) {
961 if (this._filePaths.has(overlayPath)) return this._filePaths.get(overlayPath)
962 for (const baseDir of this._syncBaseDirs) {
963 return path.join(baseDir, overlayPath)
395964 }
396 this._pendingRequests.clear()
965 return null
397966 }
398967}
968
969// True iff `p` is an existing regular file. Used to validate a resolved disk path
970// before serving (a cold-restart map entry or an overlay-path masquerading as a
971// disk path won't exist) and to probe the content-addressed spool.
972function fileExists (p) {
973 try { return fs.statSync(p).isFile() } catch { return false }
974}
lib/sync-engine.js +364 -2
……@@ -1,2 +1,364 @@
1// SHIM: v0.3.0 engine lives in lib/v3/ (Ok mirror). Re-export under our name/subpath.
2export * from './v3/sync-engine.js'
1/**
2 * SyncEngine — Feed-based change tracking and conflict detection
3 *
4 * Each peer has a Hypercore feed logging file change operations.
5 * When peers connect, feeds replicate automatically (Hypercore handles this).
6 * The sync engine diffs feeds since the last-synced sequence number
7 * and detects conflicts using three-value comparison:
8 *
9 * my current hash vs their current hash vs last-synced hash (common ancestor)
10 *
11 * Conflict resolution is configurable per sync folder:
12 * prompt | newest-wins | keep-both | skip | mine-wins | theirs-wins
13 */
14
15import ReadyResource from 'ready-resource'
16
17// Operation types written to the change feed
18const OP_PUT = 0
19const OP_DEL = 1
20
21export const STRATEGIES = ['prompt', 'newest-wins', 'keep-both', 'skip', 'mine-wins', 'theirs-wins']
22
23export class SyncEngine extends ReadyResource {
24 constructor (fileIndex, corestore, opts = {}) {
25 super()
26 this._fileIndex = fileIndex
27 this._corestore = corestore
28 this._feed = null
29 this._peerFeeds = new Map() // peerKey hex → Hypercore
30 this._conflicts = [] // unresolved conflicts
31 this._opts = opts
32 }
33
34 async _open () {
35 // Local change feed — append-only log of file operations. Encrypted at rest when
36 // the caller passes opts.encryptionKey (peer feeds, opened by key in addPeerFeed,
37 // stay plaintext — their key is not ours to encrypt under).
38 const opts = { name: 'sync-feed', valueEncoding: 'json' }
39 if (this._opts.encryptionKey) opts.encryptionKey = this._opts.encryptionKey
40 this._feed = this._corestore.get(opts)
41 await this._feed.ready()
42 }
43
44 async _close () {
45 // Peer feeds are managed by corestore, no explicit close needed
46 }
47
48 /**
49 * The local change feed (Hypercore)
50 */
51 get feed () { return this._feed }
52
53 /**
54 * The local feed's public key as hex
55 */
56 get feedKey () { return this._feed.key.toString('hex') }
57
58 /**
59 * Current unresolved conflicts
60 */
61 get conflicts () { return this._conflicts }
62
63 // ── Change logging ──────────────────────────────────────────
64
65 /**
66 * Log a file put/update operation
67 * @param {string} path - file path
68 * @param {{ contentHash: string, size: number, mtime: number }} meta
69 */
70 async logPut (path, meta) {
71 await this._feed.append({
72 op: OP_PUT,
73 path,
74 hash: meta.contentHash,
75 size: meta.size,
76 mtime: meta.mtime,
77 timestamp: Date.now()
78 })
79 }
80
81 /**
82 * Log a file deletion
83 * @param {string} path - file path
84 */
85 async logDel (path) {
86 await this._feed.append({
87 op: OP_DEL,
88 path,
89 hash: null,
90 size: 0,
91 mtime: 0,
92 timestamp: Date.now()
93 })
94 }
95
96 // ── Peer feeds ──────────────────────────────────────────────
97
98 /**
99 * Add a remote peer's feed for syncing
100 * @param {Buffer|string} key - peer's feed public key
101 * @returns {Hypercore} the peer feed
102 */
103 addPeerFeed (key) {
104 const keyHex = typeof key === 'string' ? key : key.toString('hex')
105 if (this._peerFeeds.has(keyHex)) return this._peerFeeds.get(keyHex)
106
107 const feed = this._corestore.get({ key: typeof key === 'string' ? Buffer.from(key, 'hex') : key, valueEncoding: 'json' })
108 this._peerFeeds.set(keyHex, feed)
109 return feed
110 }
111
112 /**
113 * Get a peer feed by key
114 * @param {string} keyHex
115 * @returns {Hypercore|null}
116 */
117 getPeerFeed (keyHex) {
118 return this._peerFeeds.get(keyHex) || null
119 }
120
121 // ── Feed diffing ────────────────────────────────────────────
122
123 /**
124 * Get new operations from a feed since a given sequence number
125 * @param {Hypercore} feed
126 * @param {number} sinceSeq - start reading from this seq (exclusive)
127 * @returns {Array<{ seq: number, op: number, path: string, hash: string|null, size: number, mtime: number }>}
128 */
129 async getChanges (feed, sinceSeq) {
130 await feed.ready()
131 await feed.update()
132
133 const changes = []
134 const start = sinceSeq + 1
135 const end = feed.length
136
137 for (let i = start; i < end; i++) {
138 const entry = await feed.get(i)
139 changes.push({ seq: i, ...entry })
140 }
141
142 return changes
143 }
144
145 /**
146 * Build a map of latest state per path from a list of changes.
147 * Later entries override earlier ones for the same path.
148 * @param {Array} changes
149 * @returns {Map<string, { seq, op, hash, size, mtime }>}
150 */
151 buildChangeMap (changes) {
152 const map = new Map()
153 for (const c of changes) {
154 map.set(c.path, { seq: c.seq, op: c.op, hash: c.hash, size: c.size, mtime: c.mtime })
155 }
156 return map
157 }
158
159 // ── Sync diff ───────────────────────────────────────────────
160
161 /**
162 * Compute the sync diff between local and a remote peer.
163 * Returns actions to take: pull, push, conflict, or skip.
164 *
165 * @param {string} peerKey - hex key of the remote peer's feed
166 * @returns {Array<{ path, action, local?, remote?, ancestor? }>}
167 * action: 'pull' | 'push' | 'conflict' | 'skip' | 'delete-local' | 'delete-remote'
168 */
169 async diff (peerKey) {
170 const peerFeed = this._peerFeeds.get(peerKey)
171 if (!peerFeed) throw new Error('Unknown peer: ' + peerKey)
172
173 await peerFeed.ready()
174 await peerFeed.update()
175
176 // Get last-synced state for all files with this peer
177 const syncStates = await this._fileIndex.listSyncStates(peerKey)
178 const syncMap = new Map()
179 for (const s of syncStates) {
180 syncMap.set(s.path, s)
181 }
182
183 // Get my changes since last sync
184 const myLastSeq = this._getMaxSyncedSeq(syncStates, 'local')
185 const myChanges = await this.getChanges(this._feed, -1) // read all, filter by syncMap
186 const myMap = this.buildChangeMap(myChanges)
187
188 // Get their changes since last sync
189 const theirChanges = await this.getChanges(peerFeed, -1) // read all
190 const theirMap = this.buildChangeMap(theirChanges)
191
192 // Collect all paths mentioned by either side
193 const allPaths = new Set([...myMap.keys(), ...theirMap.keys()])
194
195 const actions = []
196
197 for (const path of allPaths) {
198 const mine = myMap.get(path)
199 const theirs = theirMap.get(path)
200 const ancestor = syncMap.get(path)
201
202 const myHash = mine ? mine.hash : null
203 const theirHash = theirs ? theirs.hash : null
204 const ancestorHash = ancestor ? ancestor.lastHash : null
205
206 // Both have it and hashes match — in sync
207 if (myHash && theirHash && myHash === theirHash) {
208 actions.push({ path, action: 'skip', local: mine, remote: theirs })
209 continue
210 }
211
212 // Only they have it (or they changed, I haven't)
213 if (theirHash && !mine) {
214 actions.push({ path, action: 'pull', remote: theirs })
215 continue
216 }
217
218 // Only I have it (or I changed, they haven't)
219 if (myHash && !theirs) {
220 actions.push({ path, action: 'push', local: mine })
221 continue
222 }
223
224 // Both null — both deleted
225 if (!myHash && !theirHash) {
226 actions.push({ path, action: 'skip' })
227 continue
228 }
229
230 // One deleted, one has content
231 if (mine && mine.op === OP_DEL && theirHash) {
232 // I deleted, they still have it
233 if (ancestorHash && ancestorHash === theirHash) {
234 // They haven't changed since ancestor — propagate my delete
235 actions.push({ path, action: 'delete-remote', local: mine, remote: theirs })
236 } else {
237 // They changed it after ancestor — conflict
238 actions.push({ path, action: 'conflict', local: mine, remote: theirs, ancestor })
239 }
240 continue
241 }
242
243 if (theirs && theirs.op === OP_DEL && myHash) {
244 // They deleted, I still have it
245 if (ancestorHash && ancestorHash === myHash) {
246 // I haven't changed since ancestor — accept their delete
247 actions.push({ path, action: 'delete-local', local: mine, remote: theirs })
248 } else {
249 // I changed it after ancestor — conflict
250 actions.push({ path, action: 'conflict', local: mine, remote: theirs, ancestor })
251 }
252 continue
253 }
254
255 // Both have different content — use ancestor to determine conflict
256 if (myHash && theirHash && myHash !== theirHash) {
257 if (ancestorHash === myHash) {
258 // Only they changed — pull
259 actions.push({ path, action: 'pull', local: mine, remote: theirs })
260 } else if (ancestorHash === theirHash) {
261 // Only I changed — push
262 actions.push({ path, action: 'push', local: mine, remote: theirs })
263 } else {
264 // Both changed — conflict
265 actions.push({ path, action: 'conflict', local: mine, remote: theirs, ancestor })
266 }
267 continue
268 }
269
270 // Fallback — shouldn't reach here
271 actions.push({ path, action: 'skip', local: mine, remote: theirs })
272 }
273
274 return actions
275 }
276
277 // ── Conflict resolution ─────────────────────────────────────
278
279 /**
280 * Resolve a conflict using a strategy
281 * @param {{ path, local, remote, ancestor }} conflict
282 * @param {string} strategy - one of STRATEGIES
283 * @returns {{ action: 'pull'|'push'|'keep-both'|'skip', winner?: 'local'|'remote' }}
284 */
285 resolveConflict (conflict, strategy) {
286 switch (strategy) {
287 case 'mine-wins':
288 return { action: 'push', winner: 'local' }
289
290 case 'theirs-wins':
291 return { action: 'pull', winner: 'remote' }
292
293 case 'newest-wins': {
294 const mySeq = conflict.local ? conflict.local.seq : -1
295 const theirSeq = conflict.remote ? conflict.remote.seq : -1
296 if (mySeq >= theirSeq) {
297 return { action: 'push', winner: 'local' }
298 }
299 return { action: 'pull', winner: 'remote' }
300 }
301
302 case 'keep-both':
303 return { action: 'keep-both' }
304
305 case 'skip':
306 return { action: 'skip' }
307
308 case 'prompt':
309 default:
310 // Add to conflict queue for user resolution
311 this._conflicts.push(conflict)
312 return { action: 'skip' }
313 }
314 }
315
316 /**
317 * Resolve a queued conflict by user choice
318 * @param {string} path
319 * @param {string} choice - 'overwrite' | 'keep-both' | 'skip'
320 * @returns {{ action: string } | null}
321 */
322 resolveQueued (path, choice) {
323 const idx = this._conflicts.findIndex(c => c.path === path)
324 if (idx === -1) return null
325
326 const conflict = this._conflicts.splice(idx, 1)[0]
327
328 switch (choice) {
329 case 'overwrite':
330 return { action: 'pull', winner: 'remote', conflict }
331 case 'keep-mine':
332 return { action: 'push', winner: 'local', conflict }
333 case 'keep-both':
334 return { action: 'keep-both', conflict }
335 case 'skip':
336 default:
337 return { action: 'skip', conflict }
338 }
339 }
340
341 /**
342 * Update sync state after a successful sync operation
343 * @param {string} peerKey
344 * @param {string} path
345 * @param {string} hash - the hash both sides now agree on
346 * @param {number} seq - the sequence number of the operation
347 */
348 async markSynced (peerKey, path, hash, seq) {
349 await this._fileIndex.putSyncState(peerKey, path, {
350 lastSeq: seq,
351 lastHash: hash
352 })
353 }
354
355 // ── Internal ────────────────────────────────────────────────
356
357 _getMaxSyncedSeq (syncStates, _side) {
358 let max = -1
359 for (const s of syncStates) {
360 if (s.lastSeq > max) max = s.lastSeq
361 }
362 return max
363 }
364}
lib/transfer.js +808 -2
……@@ -1,2 +1,808 @@
1// SHIM: v0.3.0 engine lives in lib/v3/ (Ok mirror). Re-export under our name/subpath.
2export * from './v3/transfer.js'
1/**
2 * TransferManager — On-demand chunking and streaming file transfer
3 *
4 * Sender side:
5 * 1. Read file from disk
6 * 2. Chunk on demand (adaptive FastCDC)
7 * 3. Send chunk hashes → peer says which it needs → stream those chunks
8 * 4. Release chunks from memory after send
9 *
10 * Receiver side:
11 * 1. Receive chunk hashes, respond with which are missing
12 * 2. Receive chunks, write to <file>.overlay-partial
13 * 3. Verify each chunk hash
14 * 4. Atomic rename on completion
15 *
16 * No blob storage. Chunks exist in memory during transfer only.
17 */
18
19import fs from 'bare-fs'
20import path from 'bare-path'
21import os from 'bare-os'
22import c from 'compact-encoding'
23import sodium from 'sodium-universal'
24import { chunk as chunkBuffer, chunkStream, hashChunk, createStreamingHasher, selectTier, getTierParams } from './chunker.js'
25
26export const PARTIAL_SUFFIX = '.overlay-partial'
27
28export const partialPathFor = (targetPath) => path.join(path.dirname(targetPath), path.basename(targetPath) + PARTIAL_SUFFIX)
29
30// Resume verification + whole-file hashing run async + yielding so a multi-GB pass
31// never blocks the event loop (IPC, pause/cancel, progress stay live).
32const VERIFY_YIELD_EVERY = 64
33const yieldToLoop = () => new Promise((resolve) => setTimeout(resolve, 0))
34
35// Receive journal: bitmap + whole-file hash snapshot, stored in app-private storage
36// (opts.journalDir) keyed by a digest of the destination path — never next to the
37// download — so resume is O(1) and the incremental verify continues with no re-read.
38const JOURNAL_SUFFIX = '.journal'
39const JOURNAL_MAGIC = 0x4d4a5633
40const JOURNAL_FLUSH_EVERY = 512
41const JOURNAL_FLUSH_INTERVAL_MS = 1500
42export const journalNameFor = (targetPath) => hashChunk(Buffer.from(targetPath)) + JOURNAL_SUFFIX
43
44// Block-read prepare — host-adaptive read block size.
45const READ_BLOCK_DEFAULT = 8 * 1024 * 1024
46const READ_BLOCK_LOW_RAM = 4 * 1024 * 1024
47const LOW_RAM_THRESHOLD = 6 * 1024 * 1024 * 1024
48
49// Cap on stashed just-received chunk buffers (per transfer, insert-refusal), and the
50// hash pump's yield budget — stash-fed iterations never await, so the pump yields by
51// bytes hashed, not chunk count.
52const MEM_STASH_BYTES_DEFAULT = 16 * 1024 * 1024
53const DRAIN_YIELD_BYTES = 2 * 1024 * 1024
54
55// Emit file blocks >= tier maxSize so chunkStream drains every block
56// instead of re-copying a growing pending buffer (64 KiB createReadStream reads force
57// ~56x memcpy at tier 3). Owns its fd and closes it on every terminal path: normal
58// end, consumer break/throw (for-await calls .return() → finally), read error.
59async function * readFileBlocks (filePath, size, blockSize) {
60 const fd = await fs.open(filePath, 'r')
61 try {
62 let pos = 0
63 while (pos < size) {
64 const want = Math.min(blockSize, size - pos)
65 const buf = Buffer.allocUnsafe(want)
66 let filled = 0
67 while (filled < want) {
68 const n = await fs.read(fd, buf, filled, want - filled, pos + filled)
69 if (n <= 0) break
70 filled += n
71 }
72 if (filled === 0) break
73 pos += filled
74 yield filled === buf.length ? buf : buf.subarray(0, filled)
75 }
76 } finally {
77 try { await fs.close(fd) } catch {}
78 }
79}
80
81function bitmapOf (received, total) {
82 const bm = Buffer.alloc((total + 7) >> 3)
83 for (const i of received) bm[i >> 3] |= 1 << (i & 7)
84 return bm
85}
86
87// Drop journals whose partial is gone, are stale, or are corrupt/foreign. Each
88// journal self-describes its partial path (read cheaply from the header). Standalone
89// (no TransferManager instance) so a boot-time sweep can reclaim orphans without
90// the overlay stack being live.
91export function cleanupOrphanedJournals (journalDir, maxAge = 7 * 86400000) {
92 const cleaned = []
93 if (!journalDir) return cleaned
94 let entries
95 try { entries = fs.readdirSync(journalDir) } catch { return cleaned }
96 const now = Date.now()
97 for (const entry of entries) {
98 if (!entry.endsWith(JOURNAL_SUFFIX)) continue
99 const full = path.join(journalDir, entry)
100 let remove = false
101 try {
102 const raw = fs.readFileSync(full)
103 const s = { start: 0, end: raw.length, buffer: raw }
104 if (c.uint32.decode(s) !== JOURNAL_MAGIC) remove = true
105 else {
106 c.uint32.decode(s)
107 const partialPath = c.string.decode(s)
108 if (!fs.existsSync(partialPath)) remove = true
109 else if (now - fs.statSync(full).mtimeMs > maxAge) remove = true
110 }
111 } catch { remove = true }
112 if (remove) { try { fs.unlinkSync(full); cleaned.push(full) } catch {} }
113 }
114 return cleaned
115}
116
117export class TransferManager {
118 constructor (fileIndex, opts = {}) {
119 this._fileIndex = fileIndex
120 this._maxConcurrency = opts.maxConcurrency || 4
121 this._active = new Map() // path → transfer state
122 this._memStashBytes = opts.memStashBytes ?? MEM_STASH_BYTES_DEFAULT
123 this._journalDir = opts.journalDir || null
124 if (this._journalDir) { try { fs.mkdirSync(this._journalDir, { recursive: true }) } catch {} }
125 }
126
127 _journalPathFor (targetPath) {
128 if (!this._journalDir) return null
129 return path.join(this._journalDir, journalNameFor(targetPath))
130 }
131
132 // ── Sender side ─────────────────────────────────────────────
133
134 /**
135 * Prepare a file for sending — read from disk, chunk, return hashes.
136 * Chunk map is persisted for large files (>1MB).
137 *
138 * @param {string} filePath - absolute path on disk
139 * @param {string} overlayPath - path in the overlay namespace
140 * @returns {{ tier: number, chunks: Array<{ hash, offset, length }>, size: number } | null}
141 */
142 async prepareFile (filePath, overlayPath, opts = {}) {
143 // Check file exists and get stats
144 let stat
145 try {
146 stat = fs.statSync(filePath)
147 } catch {
148 return null
149 }
150
151 if (!stat.isFile()) return null
152
153 const mtimeBefore = stat.mtimeMs
154
155 // Check for existing chunk map (skip re-chunking if valid)
156 const existing = await this._fileIndex.getChunkMap(overlayPath)
157 const fileMeta = await this._fileIndex.getFile(overlayPath)
158
159 if (existing && fileMeta && fileMeta.mtime === mtimeBefore) {
160 // Chunk map still valid — file hasn't changed.
161 // Backfill the content-addressed alias so the serve path (a
162 // different overlay key) reuses this map instead of re-reading the file —
163 // covers maps chunked before content-addressed caching existed.
164 if (fileMeta.contentHash && stat.size >= 1048576 && !await this._fileIndex.hasChunkMapByHash(fileMeta.contentHash)) {
165 await this._fileIndex.putChunkMapByHash(fileMeta.contentHash, existing)
166 }
167 return {
168 tier: selectTier(stat.size),
169 chunks: existing,
170 size: stat.size,
171 contentHash: fileMeta.contentHash // surfaced so the publish path gets the hash from one pass
172 }
173 }
174
175 // Read in blocks >= tier maxSize and chunk from views (copy:false).
176 // Cut points are content-stable across block sizes, so peers still match hashes.
177 // size is passed to the hasher so the streaming digest matches hashChunk(buffer).
178 const tier = selectTier(stat.size)
179 const blockSize = Math.max(
180 getTierParams(tier).maxSize,
181 os.totalmem() < LOW_RAM_THRESHOLD ? READ_BLOCK_LOW_RAM : READ_BLOCK_DEFAULT
182 )
183 const fileHasher = createStreamingHasher({ size: stat.size })
184 const chunks = []
185
186 for await (const c of chunkStream(readFileBlocks(filePath, stat.size, blockSize), { tier, copy: false })) {
187 // honor a publish-cancel signal (per-chunk granularity); on throw the
188 // for-await calls readFileBlocks.return() which closes the fd.
189 if (opts.signal?.aborted) { const e = new Error('publish aborted'); e.code = 'ECANCELLED'; throw e }
190 fileHasher.update(c.data)
191 chunks.push({ hash: c.hash, offset: c.offset, length: c.length })
192 opts.onProgress?.(c.length)
193 }
194
195 // mtime guard — abort if file changed during the streaming read. A
196 // vanished source (moved out of the folder mid-read) is just another "changed
197 // during read" case — the open fd kept the read alive after a same-volume
198 // rename, so it is this path-resolving stat that trips ENOENT. Guard it like
199 // the pre-read stat above and re-queue rather than throw.
200 let statAfter
201 try {
202 statAfter = fs.statSync(filePath)
203 } catch {
204 return null // source vanished during read, caller should re-queue
205 }
206 if (statAfter.mtimeMs !== mtimeBefore) {
207 return null // file changed during read, caller should re-queue
208 }
209
210 // Post-read guard: a mid-read shrink can feed the digest fewer bytes than the
211 // size baked into its leaf prefix — re-queue instead of persisting a bad hash.
212 if (fileHasher.bytes !== stat.size) return null
213
214 // Update file index
215 const contentHash = fileHasher.digest()
216
217 // Persist chunk map for large files — both path-keyed (legacy serve lookups)
218 // and content-addressed, so the serve path reuses it across keys.
219 // byHashOnly (the publish prepareForServe pass) keeps ONLY the
220 // content-addressed map — the serve path resolves by hash, so the path-keyed map
221 // + putFile under a throwaway overlay key would be dead, never-read state.
222 if (stat.size >= 1048576) {
223 if (!opts.byHashOnly) await this._fileIndex.putChunkMap(overlayPath, chunks)
224 if (!await this._fileIndex.hasChunkMapByHash(contentHash)) await this._fileIndex.putChunkMapByHash(contentHash, chunks)
225 }
226
227 if (!opts.byHashOnly) {
228 await this._fileIndex.putFile(overlayPath, {
229 contentHash,
230 size: stat.size,
231 mtime: mtimeBefore
232 })
233 }
234
235 return { tier, chunks, size: stat.size, contentHash }
236 }
237
238 /**
239 * Read a specific chunk from disk by its offset and length.
240 * Returns raw bytes — caller sends them to the peer.
241 *
242 * @param {string} filePath - absolute path on disk
243 * @param {number} offset - byte offset
244 * @param {number} length - bytes to read
245 * @returns {Buffer | null}
246 */
247 readChunk (filePath, offset, length) {
248 try {
249 const fd = fs.openSync(filePath, 'r')
250 const buf = Buffer.alloc(length)
251 const bytesRead = fs.readSync(fd, buf, 0, length, offset)
252 fs.closeSync(fd)
253 if (bytesRead !== length) return null
254 return buf
255 } catch {
256 return null
257 }
258 }
259
260 /**
261 * Determine which chunks a peer needs (chunks they don't already have).
262 *
263 * @param {Array<{ hash, offset, length }>} offered - chunks the sender has
264 * @param {Set<string>} peerHas - set of chunk hashes the peer already has
265 * @returns {number[]} indices of chunks the peer needs
266 */
267 computeNeeded (offered, peerHas) {
268 const needed = []
269 for (let i = 0; i < offered.length; i++) {
270 if (!peerHas.has(offered[i].hash)) {
271 needed.push(i)
272 }
273 }
274 return needed
275 }
276
277 // ── Receiver side ───────────────────────────────────────────
278
279 // Start receiving a file. A same-size partial is resumed: the journal restores the
280 // received-set + whole-file hash snapshot in O(1) (no re-read); failing that, an
281 // async, yielding re-verify rebuilds them without blocking the event loop.
282 // opts.isCancelled aborts a long re-verify; opts.onVerifyProgress(0..1) drives the UI.
283 async startReceive (targetPath, meta, opts = {}) {
284 const isCancelled = opts.isCancelled || (() => false)
285 const dir = path.dirname(targetPath)
286 const base = path.basename(targetPath)
287 // Visible partial (no leading dot) so an in-progress download shows in the
288 // destination folder; the distinct ".overlay-partial" suffix keeps host-app
289 // sweeps keyed on a plain ".partial" suffix from deleting a paused overlay
290 // partial.
291 const partialPath = path.join(dir, base + PARTIAL_SUFFIX)
292 const journalPath = this._journalPathFor(targetPath)
293
294 fs.mkdirSync(dir, { recursive: true })
295
296 let resumed = false
297 try { resumed = fs.statSync(partialPath).size === meta.size } catch {}
298
299 let received = new Set()
300 let hasher = meta.contentHash ? createStreamingHasher({ size: meta.size }) : null
301 let hashFrontier = 0
302
303 if (resumed) {
304 const j = journalPath ? this._loadJournal(journalPath, partialPath, meta) : null
305 if (j) {
306 received = j.received
307 hashFrontier = j.hashFrontier
308 if (meta.contentHash) hasher = createStreamingHasher({ size: meta.size, restore: { state: j.hasherState, bytes: j.hasherBytes } })
309 } else {
310 const rec = await this._recoverPartialAsync(partialPath, meta, isCancelled, opts.onVerifyProgress)
311 received = rec.received; hasher = rec.hasher; hashFrontier = rec.hashFrontier
312 }
313 } else {
314 if (journalPath) { try { fs.unlinkSync(journalPath) } catch {} }
315 }
316
317 if (isCancelled()) { const e = new Error('receive cancelled during setup'); e.code = 'ECANCELLED'; throw e }
318
319 // ONE persistent fd for the whole transfer — positioned writeSync/fs.read never
320 // move a shared cursor, so a single handle serves the chunk writes, the hash
321 // pump's gap read-back, and the journal fsync. Replaces an openSync+closeSync per
322 // chunk; 'w+' creates+truncates the fresh partial in the same open.
323 const fd = fs.openSync(partialPath, resumed ? 'r+' : 'w+')
324 if (!resumed) {
325 try { fs.ftruncateSync(fd, meta.size) } catch (err) { try { fs.closeSync(fd) } catch {}; throw err }
326 }
327
328 const total = meta.chunks.length
329 const state = {
330 partialPath,
331 targetPath,
332 journalPath,
333 chunks: meta.chunks,
334 received,
335 bitmap: bitmapOf(received, total),
336 total,
337 size: meta.size,
338 contentHash: meta.contentHash || null,
339 hasher,
340 hashFrontier,
341 advancePromise: null,
342 draining: false,
343 sinceFlush: 0,
344 lastFlush: 0,
345 flushing: false,
346 flushPending: false,
347 fd,
348 // Bounded stash of just-received verified chunk buffers so the hash pump feeds
349 // from memory instead of reading the bytes back off disk. Never load-bearing:
350 // bytes hit disk first; a miss falls back to read-back.
351 memChunks: new Map(),
352 memBytes: 0,
353 stats: { readbacks: 0, stashHits: 0 }
354 }
355
356 // A prior transfer for this path that ended via _fail (stall / all-peers-
357 // gone / disk error) leaves its state here with an open fd — close it before
358 // overwriting so a retry/resume of the same file can't orphan the fd.
359 const prev = this._active.get(targetPath)
360 if (prev) this._closeFd(prev)
361
362 this._active.set(targetPath, state)
363 // A fresh transfer has nothing worth journaling yet; a resume that recovered
364 // chunks persists them now so an immediate re-pause is O(1).
365 if (received.size > 0) this._flushJournal(state)
366 return state
367 }
368
369 // Rebuild the received-set from a same-size partial without blocking the loop:
370 // async reads, a yield every VERIFY_YIELD_EVERY chunks, feeding the contiguous
371 // prefix into the whole-file hasher inline (single pass, no second read).
372 async _recoverPartialAsync (partialPath, meta, isCancelled, onVerifyProgress) {
373 const received = new Set()
374 const hasher = meta.contentHash ? createStreamingHasher({ size: meta.size }) : null
375 let hashFrontier = 0
376 const total = meta.chunks.length
377 const maxLen = meta.chunks.reduce((m, c) => (c.length > m ? c.length : m), 0)
378 const buf = Buffer.allocUnsafe(maxLen)
379 let lastPct = -1
380 onVerifyProgress?.(0)
381 const fd = await fs.open(partialPath, 'r')
382 try {
383 for (let i = 0; i < total; i++) {
384 if (isCancelled()) { const e = new Error('receive recovery cancelled'); e.code = 'ECANCELLED'; throw e }
385 const ci = meta.chunks[i]
386 let n = 0
387 try { n = await fs.read(fd, buf, 0, ci.length, ci.offset) } catch {}
388 const view = buf.subarray(0, ci.length)
389 if (n === ci.length && hashChunk(view) === ci.hash) {
390 received.add(i)
391 if (hasher && i === hashFrontier) { hasher.update(view); hashFrontier++ }
392 }
393 if ((i & (VERIFY_YIELD_EVERY - 1)) === VERIFY_YIELD_EVERY - 1) {
394 const pct = Math.floor(((i + 1) / total) * 100)
395 if (pct !== lastPct) { lastPct = pct; onVerifyProgress?.(pct / 100) }
396 await yieldToLoop()
397 }
398 }
399 } finally {
400 try { await fs.close(fd) } catch {}
401 }
402 onVerifyProgress?.(1)
403 return { received, hasher, hashFrontier }
404 }
405
406 _encodeJournal (state) {
407 const total = state.total
408 const bm = state.bitmap
409 const ch = state.contentHash ? Buffer.from(state.contentHash, 'hex') : Buffer.alloc(32)
410 const snap = state.hasher ? state.hasher.snapshot() : { state: Buffer.alloc(0), bytes: 0 }
411 const s = { start: 0, end: 0, buffer: null }
412 c.uint32.preencode(s, JOURNAL_MAGIC)
413 c.uint32.preencode(s, sodium.crypto_generichash_STATEBYTES)
414 c.string.preencode(s, state.partialPath)
415 c.uint64.preencode(s, state.size)
416 c.uint32.preencode(s, total)
417 c.fixed32.preencode(s, ch)
418 c.uint32.preencode(s, state.hashFrontier)
419 c.uint64.preencode(s, snap.bytes)
420 c.buffer.preencode(s, snap.state)
421 c.buffer.preencode(s, bm)
422 s.buffer = Buffer.allocUnsafe(s.end)
423 c.uint32.encode(s, JOURNAL_MAGIC)
424 c.uint32.encode(s, sodium.crypto_generichash_STATEBYTES)
425 c.string.encode(s, state.partialPath)
426 c.uint64.encode(s, state.size)
427 c.uint32.encode(s, total)
428 c.fixed32.encode(s, ch)
429 c.uint32.encode(s, state.hashFrontier)
430 c.uint64.encode(s, snap.bytes)
431 c.buffer.encode(s, snap.state)
432 c.buffer.encode(s, bm)
433 return s.buffer
434 }
435
436 // Returns the resume state only when the journal is for this partial, binds to
437 // this content, and its snapshot is restorable on this libsodium build; else null
438 // so the caller falls back to the async re-verify.
439 _loadJournal (journalPath, partialPath, meta) {
440 let raw
441 try { raw = fs.readFileSync(journalPath) } catch { return null }
442 try {
443 const s = { start: 0, end: raw.length, buffer: raw }
444 if (c.uint32.decode(s) !== JOURNAL_MAGIC) return null
445 const stateBytes = c.uint32.decode(s)
446 const storedPartial = c.string.decode(s)
447 const size = c.uint64.decode(s)
448 const total = c.uint32.decode(s)
449 const ch = c.fixed32.decode(s).toString('hex')
450 const hashFrontier = c.uint32.decode(s)
451 const hasherBytes = c.uint64.decode(s)
452 const hasherState = c.buffer.decode(s)
453 const bm = c.buffer.decode(s)
454 if (storedPartial !== partialPath) return null
455 if (size !== meta.size || total !== meta.chunks.length) return null
456 if (meta.contentHash && ch !== meta.contentHash) return null
457 if (meta.contentHash) {
458 if (stateBytes !== sodium.crypto_generichash_STATEBYTES) return null
459 if (hasherState.length !== sodium.crypto_generichash_STATEBYTES) return null
460 }
461 const received = new Set()
462 for (let i = 0; i < total; i++) if (bm[i >> 3] & (1 << (i & 7))) received.add(i)
463 return { received, hashFrontier, hasherState, hasherBytes }
464 } catch { return null }
465 }
466
467 // Best-effort async crash-hedge flush: never blocks the receive loop, single-flight
468 // (a flush requested while one runs coalesces into one more pass). fsync the partial
469 // BEFORE the journal references it so a power-loss can never leave the journal
470 // marking a chunk received whose bytes aren't durable.
471 _flushJournal (state) {
472 if (!state.contentHash || !state.journalPath || state.fd == null) return
473 if (state.flushing) { state.flushPending = true; return }
474 if (Date.now() - state.lastFlush < JOURNAL_FLUSH_INTERVAL_MS) return
475 state.flushing = true
476 this._drainFlush(state)
477 }
478
479 async _drainFlush (state) {
480 do {
481 state.flushPending = false
482 state.lastFlush = Date.now()
483 if (state.fd == null) break
484 let buf
485 try { buf = this._encodeJournal(state) } catch { break }
486 const tmp = state.journalPath + '.tmp'
487 try {
488 try { await fs.fsync(state.fd) } catch {}
489 await fs.writeFile(tmp, buf)
490 if (state.fd == null) break // paused/cancelled mid-flush — don't re-create the journal
491 await fs.rename(tmp, state.journalPath)
492 } catch { try { await fs.unlink(tmp) } catch {} }
493 } while (state.flushPending && state.fd != null && state.contentHash)
494 try { await fs.unlink(state.journalPath + '.tmp') } catch {}
495 state.flushing = false
496 }
497
498 // Synchronous durable flush for pause/shutdown, where the app may quit before an
499 // async flush would land.
500 _flushJournalSync (state) {
501 if (!state.contentHash || !state.journalPath) return
502 let buf
503 try { buf = this._encodeJournal(state) } catch { return }
504 if (state.fd != null) { try { fs.fsyncSync(state.fd) } catch {} }
505 try {
506 const tmp = state.journalPath + '.tmp'
507 fs.writeFileSync(tmp, buf)
508 fs.renameSync(tmp, state.journalPath)
509 state.lastFlush = Date.now()
510 } catch { try { fs.unlinkSync(state.journalPath + '.tmp') } catch {} }
511 }
512
513 cleanJournals (maxAge) { return cleanupOrphanedJournals(this._journalDir, maxAge) }
514
515 async _hashWholeFileAsync (filePath, size) {
516 const h = createStreamingHasher({ size })
517 for await (const buf of readFileBlocks(filePath, size, 8 * 1024 * 1024)) {
518 h.update(buf)
519 await yieldToLoop()
520 }
521 return h.digest()
522 }
523
524 // Advance the whole-file hash over the contiguous run of received chunks
525 // as a single-flight BACKGROUND pump: chunk index order == file offset order, so
526 // once chunk `hashFrontier` is present we feed its bytes and move on. The drain is
527 // async + yielding so a large contiguous run that suddenly becomes available (a
528 // gap filling after a low-frontier resume) cannot block the event loop.
529 // Single-flight is `state.draining`, set/cleared SYNCHRONOUSLY inside _drainHash —
530 // a stash-fed drain completes without awaiting, so a promise-based gate would stay
531 // stale until its microtask clear and block same-macrotask restarts (protomux
532 // delivers frame bursts synchronously). advancePromise is only the awaitable
533 // handle for finalize/pause; its clear is guarded against nulling a newer drain.
534 _advanceHash (state) {
535 if (!state.hasher || state.draining) return
536 if (!(state.hashFrontier < state.chunks.length && state.received.has(state.hashFrontier))) return
537 const p = this._drainHash(state)
538 state.advancePromise = p
539 p.catch(() => {}).then(() => { if (state.advancePromise === p) state.advancePromise = null })
540 }
541
542 async _drainHash (state) {
543 state.draining = true
544 try {
545 let sinceYield = 0
546 while (state.fd != null && state.hashFrontier < state.chunks.length && state.received.has(state.hashFrontier)) {
547 const i = state.hashFrontier
548 const c = state.chunks[i]
549 let buf = state.memChunks.get(i)
550 if (buf !== undefined) {
551 state.memChunks.delete(i)
552 state.memBytes -= buf.length
553 state.stats.stashHits++
554 } else {
555 buf = Buffer.alloc(c.length)
556 try { await fs.read(state.fd, buf, 0, c.length, c.offset) } catch { return }
557 state.stats.readbacks++
558 }
559 state.hasher.update(buf)
560 state.hashFrontier++
561 // Yield by BYTE volume: the stash path has no awaits, so a chunk-count
562 // cadence would let a gap-fill hash the whole stash synchronously inside
563 // the network event that delivered the gap chunk.
564 sinceYield += buf.length
565 if (sinceYield >= DRAIN_YIELD_BYTES) { sinceYield = 0; await yieldToLoop() }
566 }
567 } finally {
568 state.draining = false
569 }
570 }
571
572 // Close the persistent fd — MUST run before finalize's rename (an open handle
573 // blocks the rename on Windows) — and drop the chunk stash.
574 _closeFd (state) {
575 if (state.fd != null) {
576 try { fs.closeSync(state.fd) } catch {}
577 state.fd = null
578 }
579 state.memChunks.clear()
580 state.memBytes = 0
581 }
582
583 /**
584 * Write a received chunk to the partial file. Verifies the hash.
585 *
586 * @param {string} targetPath - the final file path (used as key)
587 * @param {number} index - chunk index
588 * @param {Buffer} data - raw chunk bytes
589 * @returns {{ ok: boolean, error?: string }}
590 */
591 writeChunk (targetPath, index, data) {
592 const state = this._active.get(targetPath)
593 if (!state) return { ok: false, error: 'no active transfer' }
594 if (index < 0 || index >= state.chunks.length) return { ok: false, error: 'index out of range' }
595 // Terminal race (chunk landed after close) — codeless so the scheduler
596 // re-assigns instead of failing the fetch.
597 if (state.fd == null) return { ok: false, error: 'transfer closed' }
598
599 const expected = state.chunks[index]
600
601 // Verify hash
602 const actual = hashChunk(data)
603 if (actual !== expected.hash) {
604 return { ok: false, error: `hash mismatch at index ${index}: expected ${expected.hash.slice(0, 16)}... got ${actual.slice(0, 16)}...` }
605 }
606
607 // Verify length
608 if (data.length !== expected.length) {
609 return { ok: false, error: `length mismatch at index ${index}: expected ${expected.length} got ${data.length}` }
610 }
611
612 // Write to partial at the chunk's offset through the persistent fd. A local
613 // I/O failure here (ENOSPC / EACCES / EROFS) is fatal and non-retryable —
614 // return its code so the scheduler fails the fetch instead of looping over
615 // other peers. Loop on a SHORT count: libuv swallows an error after partial
616 // progress and returns short without throwing; retrying the remainder
617 // surfaces the real coded error instead of leaving a silent hole the
618 // stash-fed digest would never see.
619 try {
620 let written = 0
621 while (written < data.length) {
622 const n = fs.writeSync(state.fd, data, written, data.length - written, expected.offset + written)
623 if (!(n > 0)) { const e = new Error(`short write at index ${index}: ${written}/${data.length} bytes`); e.code = 'EIO'; throw e }
624 written += n
625 }
626 } catch (err) {
627 return { ok: false, error: err.message, code: err.code }
628 }
629
630 state.received.add(index)
631 state.bitmap[index >> 3] |= 1 << (index & 7)
632 // Stash only what the pump can consume cleanly: skip when already hashed or
633 // stashed, when the pump is mid-drain on exactly this index, or over cap.
634 // Copied, not aliased — wire buffers are views into shared network slabs (udx
635 // read slab, decrypted in place by secret-stream), so retaining `data` would
636 // pin whole slabs and bet the digest on their write-once behavior.
637 if (state.hasher && !state.memChunks.has(index) && state.memBytes + data.length <= this._memStashBytes &&
638 (index > state.hashFrontier || (index === state.hashFrontier && !state.draining))) {
639 state.memChunks.set(index, Buffer.from(data))
640 state.memBytes += data.length
641 }
642 this._advanceHash(state)
643 if (++state.sinceFlush >= JOURNAL_FLUSH_EVERY) { state.sinceFlush = 0; this._flushJournal(state) }
644
645 return { ok: true }
646 }
647
648 /**
649 * Check if a transfer is complete (all chunks received).
650 *
651 * @param {string} targetPath
652 * @returns {boolean}
653 */
654 isComplete (targetPath) {
655 const state = this._active.get(targetPath)
656 if (!state) return false
657 return state.received.size === state.total
658 }
659
660 /**
661 * Finalize a completed transfer — atomic rename from partial to target.
662 *
663 * @param {string} targetPath
664 * @returns {{ ok: boolean, error?: string }}
665 */
666 async finalize (targetPath) {
667 const state = this._active.get(targetPath)
668 if (!state) return { ok: false, error: 'no active transfer' }
669 if (state.received.size !== state.total) {
670 return { ok: false, error: `incomplete: ${state.received.size}/${state.total} chunks` }
671 }
672
673 // whole-file integrity — computed incrementally by the background hash
674 // pump during the transfer. Kick the pump and await it to quiescence, then read
675 // the digest (no full re-read). On a mismatch, drop the corrupt partial so a
676 // retry starts clean.
677 if (state.hasher) {
678 this._advanceHash(state)
679 while (state.advancePromise) { try { await state.advancePromise } catch {} }
680 }
681 if (state.contentHash) {
682 const actual = state.hashFrontier === state.chunks.length ? state.hasher.digest() : null
683 if (actual !== state.contentHash) {
684 this._closeFd(state)
685 try { fs.unlinkSync(state.partialPath) } catch {}
686 if (state.journalPath) { try { fs.unlinkSync(state.journalPath) } catch {} }
687 this._active.delete(targetPath)
688 return { ok: false, error: 'content-hash mismatch', code: 'EHASHMISMATCH' }
689 }
690 }
691 this._closeFd(state)
692
693 try {
694 fs.renameSync(state.partialPath, state.targetPath)
695 } catch (err) {
696 // Carry the code (routes through the coded local-I/O rethrow) and clear the
697 // state — the partial + journal stay for a clean resume-retry.
698 this._active.delete(targetPath)
699 return { ok: false, error: 'rename failed: ' + err.message, code: err.code }
700 }
701
702 if (state.journalPath) { try { fs.unlinkSync(state.journalPath) } catch {} }
703 this._active.delete(targetPath)
704 return { ok: true }
705 }
706
707 // Stop receiving but KEEP the partial on disk so a later startReceive
708 // verifies and resumes it. Distinct from cancel(), which unlinks the partial.
709 // Drains the in-flight hash advance to the full contiguous frontier first, so the
710 // journal captures it and resume stays O(1) (no re-read) and deterministic.
711 async pause (targetPath) {
712 const state = this._active.get(targetPath)
713 if (!state) return
714 if (state.hasher) {
715 this._advanceHash(state)
716 while (state.advancePromise) { try { await state.advancePromise } catch {} }
717 }
718 this._flushJournalSync(state)
719 this._closeFd(state)
720 this._active.delete(targetPath)
721 }
722
723 /**
724 * Cancel an active transfer and clean up the partial file.
725 *
726 * @param {string} targetPath
727 */
728 cancel (targetPath) {
729 const state = this._active.get(targetPath)
730 if (!state) return
731
732 this._closeFd(state)
733 try {
734 fs.unlinkSync(state.partialPath)
735 } catch {
736 // partial file may not exist
737 }
738 if (state.journalPath) { try { fs.unlinkSync(state.journalPath) } catch {} }
739
740 this._active.delete(targetPath)
741 }
742
743 /**
744 * Get transfer progress for an active transfer.
745 *
746 * @param {string} targetPath
747 * @returns {{ received: number, total: number, percentage: number } | null}
748 */
749 getProgress (targetPath) {
750 const state = this._active.get(targetPath)
751 if (!state) return null
752
753 return {
754 received: state.received.size,
755 total: state.total,
756 percentage: state.total > 0 ? Math.round(state.received.size / state.total * 100) : 0
757 }
758 }
759
760 /**
761 * List all active transfers.
762 *
763 * @returns {Array<{ path: string, received: number, total: number, percentage: number }>}
764 */
765 listActive () {
766 const result = []
767 for (const [targetPath, state] of this._active) {
768 result.push({
769 path: targetPath,
770 received: state.received.size,
771 total: state.total,
772 percentage: state.total > 0 ? Math.round(state.received.size / state.total * 100) : 0
773 })
774 }
775 return result
776 }
777
778 /**
779 * Clean up stale partial files in a directory (older than maxAge).
780 *
781 * @param {string} dir - directory to scan
782 * @param {number} [maxAge=86400000] - max age in ms (default 24h)
783 * @returns {string[]} paths of cleaned files
784 */
785 cleanPartials (dir, maxAge = 86400000) {
786 const cleaned = []
787 try {
788 const entries = fs.readdirSync(dir)
789 const now = Date.now()
790 for (const entry of entries) {
791 if (!entry.endsWith(PARTIAL_SUFFIX)) continue
792 const full = path.join(dir, entry)
793 try {
794 const stat = fs.statSync(full)
795 if (now - stat.mtimeMs > maxAge) {
796 fs.unlinkSync(full)
797 cleaned.push(full)
798 }
799 } catch {
800 // skip unreadable files
801 }
802 }
803 } catch {
804 // directory may not exist
805 }
806 return cleaned
807 }
808}
lib/v3/chunk-scheduler.js +0 -340
……@@ -1,340 +0,0 @@
1/**
2 * ChunkScheduler — multi-source (torrent-style) fetch of one content-addressed
3 * file across several peers, modelled on Hypercore's block replicator.
4 *
5 * Chunks are content-addressed, so any peer holding the file can serve any
6 * chunk. The scheduler:
7 * - requests the chunk list from each peer (identical, content-addressed),
8 * - tracks a global `inflight` map so each chunk is requested from exactly
9 * ONE peer at a time (no wasted bandwidth),
10 * - keeps each peer busy up to a per-peer inflight cap (parallel download),
11 * - reassigns a peer's inflight chunks to others on stall/disconnect,
12 * - hash-verifies every chunk (via TransferManager.writeChunk) and the whole
13 * file before finalizing.
14 *
15 * It drives the EXISTING v2 wire messages (chunk-need / chunk-data) — no new
16 * message type is needed for the full-copy-seeder case. The protocol delegates
17 * `_onChunkHashes` / `_onChunkData` to a scheduler ONLY when one is registered
18 * for the synthetic `content:<hash>` path, so the legacy single-peer flow
19 * is untouched.
20 */
21
22import { hashChunk } from './chunker.js'
23
24const DEFAULT_CAP = 8
25// Head-of-line fairness — cede the event loop every YIELD_EVERY accepted chunks
26// so a burst of buffered chunkData frames (protomux dispatches every frame from
27// one socket read synchronously) can't starve other listeners multiplexed on the
28// same connection (e.g. Hypercore replication). Kept local (not imported from
29// transfer.js) so this module stays Node-loadable for the unit suite.
30const yieldToLoop = () => new Promise((resolve) => setTimeout(resolve, 0))
31const YIELD_EVERY = 16
32// Idle (no-progress) timeout, reset on every accepted chunk and on the
33// initial chunk-list response. A steadily-progressing transfer never trips it —
34// only a genuine stall (no bytes for this long) fails. A fixed OVERALL timeout
35// here capped large transfers (e.g. a multi-GB file at any bandwidth always
36// exceeded 30s) and aborted them mid-stream.
37const DEFAULT_IDLE_TIMEOUT = 30000
38// Local write-error codes that may recover on retry (vs ENOSPC/EACCES/…
39// which are fatal): a transient one keeps the chunk retryable instead of failing
40// the whole multi-source fetch.
41const TRANSIENT_WRITE_CODES = new Set(['EBUSY', 'EAGAIN', 'EINTR', 'EMFILE', 'ENFILE'])
42// Min interval between have-progress reports to holders (their sender-side bar).
43// Throttles the transferProgress frames so a fast transfer can't spam holders.
44const DEFAULT_REPORT_INTERVAL = 1000
45
46export class ChunkScheduler {
47 /**
48 * @param {object} opts
49 * opts.path synthetic path key ('content:<hash>')
50 * opts.destPath where the assembled file is written
51 * opts.transfer TransferManager
52 * opts.sendNeed (peer, indices[]) => void — sends a chunk-need message
53 * opts.cap per-peer inflight cap (default 8)
54 * opts.timeout idle (no-progress) timeout ms (default 30000)
55 * opts.onProgress optional (receivedBytes, totalBytes) => void on each accepted chunk
56 */
57 constructor (opts) {
58 this.path = opts.path
59 this.destPath = opts.destPath
60 this._transfer = opts.transfer
61 this._sendNeed = opts.sendNeed
62 this._cap = opts.cap || DEFAULT_CAP
63 this._onProgress = opts.onProgress || null
64 this._onVerify = opts.onVerify || null
65 this._onBaseline = opts.onBaseline || null
66 this._reportInterval = opts.reportInterval ?? DEFAULT_REPORT_INTERVAL
67 this._lastReportAt = 0
68 // cumulative byte accounting so onProgress can drive a
69 // remaining-bytes/ETA bar, not just a chunk count.
70 this._receivedBytes = 0
71 this._totalBytes = 0
72 this._sinceYield = 0 // event-loop yield counter (see YIELD_EVERY)
73
74 this._peers = new Set()
75 this._started = false
76 this._settingUp = false
77 this._finalizing = false
78 this._done = false
79 this._chunks = null // [{ hash, offset, length }]
80 this._needed = new Set() // indices not yet written
81 this._inflight = new Map() // index → peer (one peer per chunk)
82 this._peerInflight = new Map() // peer → count
83 this._received = new Map() // peer → chunks accepted (multi-source proof)
84
85 this._resolve = null
86 this._reject = null
87 this._onEnd = opts.onEnd || null
88 this._contentHash = opts.contentHash || null // enables incremental whole-file verify
89 this._startedAt = Date.now()
90 // idle timeout — re-armed on each progress signal (see _armIdleTimer).
91 this._idleTimeout = opts.timeout || DEFAULT_IDLE_TIMEOUT
92 this._timer = null
93 this._armIdleTimer()
94 }
95
96 promise () {
97 return new Promise((resolve, reject) => { this._resolve = resolve; this._reject = reject })
98 }
99
100 // (Re)arm the no-progress watchdog. Called at construction and on every
101 // forward-progress signal so only a genuine stall — no accepted bytes for the
102 // idle window — aborts the fetch, regardless of total file size.
103 _armIdleTimer () {
104 if (this._done) return
105 clearTimeout(this._timer)
106 this._timer = setTimeout(
107 () => this._fail(new Error('multi-source fetch stalled (no progress for ' + this._idleTimeout + 'ms)')),
108 this._idleTimeout,
109 )
110 }
111
112 _reportEnd (ok, reason) {
113 if (!this._onEnd) return
114 try {
115 this._onEnd({
116 ok,
117 reason,
118 receivedBytes: this._receivedBytes,
119 totalBytes: this._totalBytes,
120 chunksRemaining: this._needed.size,
121 totalChunks: this._chunks ? this._chunks.length : 0,
122 peers: this._peers.size,
123 elapsedMs: Date.now() - this._startedAt,
124 })
125 } catch {}
126 }
127
128 _fail (err) {
129 if (this._done) return
130 this._done = true
131 clearTimeout(this._timer)
132 this._reportEnd(false, err.message)
133 // Release the receiver state at the failure boundary: journal the hash
134 // frontier and close the fd + chunk stash so a never-retried failure parks
135 // nothing (the partial stays on disk for a later resume). Best-effort — the
136 // external-stop path (cancel) does its own pause/cancel before rejecting.
137 if (this._transfer && typeof this._transfer.pause === 'function') {
138 try { Promise.resolve(this._transfer.pause(this.destPath)).catch(() => {}) } catch {}
139 }
140 if (this._reject) this._reject(err)
141 }
142
143 _finish () {
144 if (this._done) return
145 this._done = true
146 clearTimeout(this._timer)
147 this._reportEnd(true, 'complete')
148 if (this._resolve) this._resolve({ chunksPerPeer: this._received, peerCount: this._received.size, size: this._totalBytes })
149 }
150
151 // External stop (pause or discard). The partial is kept or unlinked by
152 // the caller (OverlayProtocol.cancelContent via TransferManager.pause/cancel)
153 // BEFORE this; here we only stop the loop and reject the fetch with ECANCELLED.
154 cancel () {
155 if (this._done) return
156 this._done = true
157 clearTimeout(this._timer)
158 this._reportEnd(false, 'cancelled')
159 const err = new Error('fetch cancelled')
160 err.code = 'ECANCELLED'
161 if (this._reject) this._reject(err)
162 }
163
164 get done () { return this._done }
165
166 // A chunkHashes page arrived while a large map is still
167 // streaming across multiple frames. Receiving it is forward progress, so
168 // re-arm the no-progress watchdog (the full list dispatches via onChunkHashes
169 // only once the final page lands).
170 notePageProgress () { if (!this._done && !this._settingUp) this._armIdleTimer() }
171
172 /** A peer responded with the chunk list. First response starts the transfer. */
173 async onChunkHashes (peer, chunks) {
174 if (this._done) return
175 this._peers.add(peer)
176 if (!this._peerInflight.has(peer)) this._peerInflight.set(peer, 0)
177
178 if (!this._started) {
179 this._started = true
180 // The async startReceive (journal-less resume re-verify) can run for a while
181 // and accepts no chunks; suppress the no-progress watchdog (including later
182 // peers' re-arms) until setup completes, or a concurrent seeder's chunk list
183 // would re-arm it and self-trip a healthy resume.
184 this._settingUp = true
185 clearTimeout(this._timer)
186 // Derive offsets from the ordered chunk lengths.
187 let offset = 0
188 this._chunks = chunks.map((c) => {
189 const entry = { hash: c.hash, offset, length: c.length }
190 offset += c.length
191 return entry
192 })
193 this._totalBytes = offset // file size = sum of chunk lengths
194 let state
195 try {
196 state = await this._transfer.startReceive(
197 this.destPath,
198 { size: offset, chunks: this._chunks, contentHash: this._contentHash },
199 { isCancelled: () => this._done, onVerifyProgress: this._onVerify }
200 )
201 } catch (err) {
202 this._settingUp = false
203 if (this._done) return
204 return this._fail(err)
205 }
206 this._settingUp = false
207 if (this._done) return
208 // resume: only fetch chunks the partial doesn't already hold, and
209 // seed the byte counter so progress/ETA continue from the resumed offset.
210 let have = 0
211 for (let i = 0; i < this._chunks.length; i++) {
212 if (state.received.has(i)) have += this._chunks[i].length
213 else this._needed.add(i)
214 }
215 this._receivedBytes = have
216 if (have > 0) {
217 // report our resume have-bytes so holders' sender-side bars mirror our
218 // true progress, not just the bytes they re-serve. Re-sent periodically as more
219 // chunks land (onChunkData) so late-joining and multi-source holders stay in sync.
220 this._reportHave(Date.now())
221 if (this._onProgress) { try { this._onProgress(this._receivedBytes, this._totalBytes) } catch {} }
222 }
223 this._armIdleTimer()
224 if (this._needed.size === 0) return this._finalize()
225 return this._assign()
226 }
227 if (!this._settingUp) this._armIdleTimer() // a later peer's list is forward progress
228 this._assign()
229 }
230
231 /** A chunk arrived from a peer. Verify + write, then schedule more. */
232 async onChunkData (peer, index, data) {
233 if (this._done) return
234 const res = this._transfer.writeChunk(this.destPath, index, data)
235 // Free the inflight slot regardless — a bad chunk should be re-fetched.
236 if (this._inflight.get(index) === peer) {
237 this._inflight.delete(index)
238 this._peerInflight.set(peer, Math.max(0, (this._peerInflight.get(peer) || 1) - 1))
239 }
240 if (!res.ok) {
241 // A coded failure is a local fs error. A TRANSIENT code (device busy,
242 // fd pressure, interrupted syscall) can succeed on retry — leave the chunk in
243 // `needed` and reassign. Any other coded error (ENOSPC / EACCES / ENOENT / …) is
244 // fatal: fail the whole fetch carrying the code. No code = a hash/length
245 // mismatch — also retried elsewhere.
246 if (res.code && !TRANSIENT_WRITE_CODES.has(res.code)) {
247 const err = new Error('write failed: ' + res.error)
248 err.code = res.code
249 return this._fail(err)
250 }
251 this._assign()
252 return
253 }
254 this._needed.delete(index)
255 this._received.set(peer, (this._received.get(peer) || 0) + 1)
256 this._receivedBytes += (data ? data.length : 0)
257 this._armIdleTimer() // accepted bytes — reset the no-progress watchdog
258 if (this._onProgress) { try { this._onProgress(this._receivedBytes, this._totalBytes) } catch {} }
259 this._maybeReportHave()
260 if (this._needed.size === 0 && this._inflight.size === 0) return this._finalize()
261 // Periodic event-loop yield (see YIELD_EVERY)
262 if (++this._sinceYield >= YIELD_EVERY) {
263 this._sinceYield = 0
264 await yieldToLoop()
265 if (this._done) return
266 }
267 this._assign()
268 }
269
270 // Report our cumulative have-bytes to holders (their sender-side bar). Fired
271 // once at resume, then throttled as chunks land, and once at finalize so the bar settles
272 // at 100% across every holder regardless of who served what.
273 _reportHave (now) {
274 if (!this._onBaseline || this._receivedBytes <= 0) return
275 this._lastReportAt = now
276 try { this._onBaseline(this._receivedBytes) } catch {}
277 }
278
279 _maybeReportHave () {
280 if (!this._onBaseline) return
281 const now = Date.now()
282 if (now - this._lastReportAt >= this._reportInterval) this._reportHave(now)
283 }
284
285 /** A peer went away — return its inflight chunks to the pool. */
286 removePeer (peer) {
287 if (!this._peers.has(peer)) return
288 this._peers.delete(peer)
289 this._peerInflight.delete(peer)
290 for (const [index, p] of this._inflight) {
291 if (p === peer) this._inflight.delete(index) // back to needed (still in _needed)
292 }
293 if (this._done) return
294 if (this._peers.size === 0 && this._needed.size > 0) {
295 return this._fail(new Error('all peers gone with ' + this._needed.size + ' chunk(s) outstanding'))
296 }
297 this._assign()
298 }
299
300 async _finalize () {
301 if (this._finalizing || this._done) return
302 this._finalizing = true
303 this._reportHave(Date.now()) // final have=total so holders' bars settle at 100%
304 clearTimeout(this._timer) // the digest drain can outlast the idle window
305 let fin
306 try { fin = await this._transfer.finalize(this.destPath) }
307 catch (err) { return this._fail(err) }
308 if (!fin.ok) {
309 const err = new Error('finalize failed: ' + fin.error)
310 if (fin.code) err.code = fin.code // preserve EHASHMISMATCH for the caller
311 return this._fail(err)
312 }
313 this._finish()
314 }
315
316 /** Assign needed-but-not-inflight chunks to peers with spare capacity. */
317 _assign () {
318 if (this._done || !this._chunks) return
319 // Per peer, batch the indices we hand it this round.
320 const batches = new Map()
321 for (const peer of this._peers) {
322 let slots = this._cap - (this._peerInflight.get(peer) || 0)
323 if (slots <= 0) continue
324 for (const index of this._needed) {
325 if (slots <= 0) break
326 if (this._inflight.has(index)) continue
327 this._inflight.set(index, peer)
328 this._peerInflight.set(peer, (this._peerInflight.get(peer) || 0) + 1)
329 if (!batches.has(peer)) batches.set(peer, [])
330 batches.get(peer).push(index)
331 slots--
332 }
333 }
334 for (const [peer, indices] of batches) {
335 if (indices.length) this._sendNeed(peer, indices)
336 }
337 }
338}
339
340export { hashChunk }
lib/v3/chunker.js +0 -333
……@@ -1,333 +0,0 @@
1/**
2 * FastCDC Content-Defined Chunker
3 *
4 * Gear rolling hash with normalized chunking (Xia et al. 2016).
5 * Produces content-stable chunk boundaries — editing the middle of a file
6 * only changes ~2 chunks regardless of file size.
7 *
8 * v2: Adaptive chunk sizing — tier selected by file size.
9 * All peers MUST use the same tier for a given file to produce matching boundaries.
10 */
11
12import crypto from 'hypercore-crypto'
13
14// ── Tier 0 (default, < 1 MB) ─────────────────────────────────
15export const MIN_SIZE = 4096
16export const AVG_SIZE = 16384
17export const MAX_SIZE = 65536
18
19// ── Adaptive chunk size tiers ─────────────────────────────────
20export const TIERS = [
21 { minSize: 4096, avgSize: 16384, maxSize: 65536 }, // Tier 0: < 1 MB
22 { minSize: 16384, avgSize: 65536, maxSize: 262144 }, // Tier 1: 1 MB - 100 MB
23 { minSize: 65536, avgSize: 262144, maxSize: 1048576 }, // Tier 2: 100 MB - 10 GB
24 { minSize: 262144, avgSize: 1048576, maxSize: 4194304 } // Tier 3: > 10 GB
25]
26
27const TIER_THRESHOLDS = [
28 1048576, // 1 MB
29 104857600, // 100 MB
30 10737418240 // 10 GB
31]
32
33/**
34 * Select the appropriate chunk size tier for a file
35 * @param {number} fileSize - total file size in bytes
36 * @returns {number} tier index (0-3)
37 */
38export function selectTier (fileSize) {
39 for (let i = 0; i < TIER_THRESHOLDS.length; i++) {
40 if (fileSize < TIER_THRESHOLDS[i]) return i
41 }
42 return TIERS.length - 1
43}
44
45/**
46 * Get chunk size parameters for a tier
47 * @param {number} tier - tier index (0-3)
48 * @returns {{ minSize: number, avgSize: number, maxSize: number }}
49 */
50export function getTierParams (tier) {
51 return TIERS[tier] || TIERS[0]
52}
53
54// Gear hash table — 256 random uint32 values
55// MUST be identical across all implementations for chunk boundary compatibility
56const GEAR = new Uint32Array([
57 0x5c95c078, 0x22408989, 0x2d48a214, 0x12842087, 0x530f8afb, 0x474536b9, 0x2963b4f1, 0x44cb738b,
58 0x4ea7403d, 0x4d606b6e, 0x074ec5d3, 0x3af39d18, 0x726c7b74, 0x60b3f158, 0x56a76369, 0x3e8ab850,
59 0x0c215cf6, 0x42ad3c2f, 0x65b26fa7, 0x69a3e894, 0x2a60445d, 0x21b0f756, 0x5caca9af, 0x373b4cf0,
60 0x1a4a56e1, 0x5eaf1e7b, 0x7b5d5c8a, 0x48f5abf4, 0x02a42b68, 0x1f3f1a4e, 0x67e16b96, 0x404c7833,
61 0x6e4e3a3c, 0x4f769e04, 0x0bfef25f, 0x3f08d1ef, 0x3bee8eb5, 0x4302687c, 0x3e9e2964, 0x0fef6b59,
62 0x70b95d2c, 0x0cffb210, 0x2e6c3f2a, 0x6d8f0f67, 0x77a49ba2, 0x35fa6b91, 0x43d2d154, 0x5d4e6c25,
63 0x0e7e5283, 0x5fac3d19, 0x20e34b36, 0x46b6de87, 0x48eed852, 0x63aae4d9, 0x2c65c18b, 0x5e5bc6ad,
64 0x7802c749, 0x44db9a82, 0x50e76e40, 0x532dfa2b, 0x56a17be8, 0x0a81e2e7, 0x5cf2fccc, 0x3ab39d12,
65 0x5c0c0b78, 0x21e8af93, 0x3dca216f, 0x15b4b92e, 0x55445f9c, 0x44f24b2e, 0x2db33f38, 0x444f6ec8,
66 0x4eeaede8, 0x48b27a8d, 0x00b7621d, 0x3c916d41, 0x72f65778, 0x60b1a155, 0x5a946e70, 0x3ea5d857,
67 0x0aca0e86, 0x41b3b22e, 0x6572e2a6, 0x6aa8e89a, 0x2d496f61, 0x23b6f159, 0x5eb7e1a6, 0x312e4fea,
68 0x1baf6dc4, 0x5eb0f27e, 0x7a505c92, 0x49e8abf2, 0x03b12c6e, 0x1e4a7a4d, 0x64e76a98, 0x414c7c39,
69 0x6f4a3f38, 0x4c719e04, 0x0efef963, 0x3d08c1e9, 0x3ce88fba, 0x44026c79, 0x3e1e2469, 0x10ef6858,
70 0x73b55e21, 0x0dcfb614, 0x2b6d3927, 0x6c8e1066, 0x74a09bb5, 0x36fa7690, 0x44d2c158, 0x5e4f6b26,
71 0x0e7e5788, 0x5dad3e1c, 0x22e14a30, 0x47b6dd88, 0x4befd953, 0x64aae5d8, 0x2b65b28a, 0x5f5ac6aa,
72 0x7903c648, 0x43db1a89, 0x50e76f45, 0x542dfb28, 0x55a07cef, 0x0b80e3ea, 0x5ef2f9cf, 0x38b31e15,
73 0x5d0d0a7d, 0x22e9ae91, 0x3ecb216a, 0x16b5b82f, 0x54445e99, 0x45f34c2d, 0x2eb23e3b, 0x454e6dc9,
74 0x4febedef, 0x49b37a88, 0x01b66320, 0x3d926e44, 0x71f75779, 0x61b0a054, 0x5b947f71, 0x3fa4d754,
75 0x0bcb0f83, 0x42b4b32f, 0x6473e3a5, 0x6ba9e999, 0x2e497064, 0x24b7f05a, 0x5fb6e0a7, 0x322f5fe9,
76 0x1cae7ec3, 0x5db0e37f, 0x7b514d91, 0x4ae9abf3, 0x02b22d6b, 0x1f4b7b50, 0x63e66b97, 0x414d7d34,
77 0x6e4f3a3d, 0x4d779f05, 0x0afe0960, 0x3e09d0ee, 0x3de98ebb, 0x43036c7a, 0x3f1f2560, 0x11f06959,
78 0x72b65e22, 0x0dceb711, 0x2a6e3826, 0x6d8f1165, 0x75a19ab4, 0x37fb7793, 0x45d3c057, 0x5f4e6a27,
79 0x0f7f5684, 0x5eae3f1d, 0x23e24b31, 0x48b7de89, 0x4ceed854, 0x63abe5db, 0x2c64b38b, 0x5e5bc7ab,
80 0x7a04c54a, 0x44db1b88, 0x51e86e41, 0x552efb2a, 0x54a17de0, 0x0a81e3e6, 0x5df3f8ce, 0x39b41f14,
81 0x5e0e0b79, 0x23e8af92, 0x3fcc216e, 0x17b4b92d, 0x55456e9d, 0x46f44d2f, 0x2fb33e39, 0x464f6ec7,
82 0x4febe8e9, 0x49b27b8c, 0x02b7631e, 0x3e916e42, 0x73f75677, 0x62b2a256, 0x5c957e72, 0x3ea6d856,
83 0x0cca0f85, 0x43b4b231, 0x6574e2a7, 0x6ca9e89b, 0x2f496e62, 0x25b7f058, 0x5eb7e0a5, 0x332e4eeb,
84 0x1daf6ec5, 0x5eb0e27d, 0x7c515d93, 0x4be8abf1, 0x04b32d6a, 0x204b7a4e, 0x64e76b99, 0x424c7d3a,
85 0x704a3e3b, 0x4e789f06, 0x0bfefa62, 0x3f09d1e8, 0x3ee98fbd, 0x45036d7b, 0x401f2461, 0x12f0685a,
86 0x74b65f23, 0x0ecfb713, 0x2b6d3928, 0x6e901267, 0x76a29bb6, 0x38fc7892, 0x46d3c159, 0x604e6b28,
87 0x107e5785, 0x5fae3e1e, 0x24e24b32, 0x49b8df8a, 0x4defd955, 0x64abe6da, 0x2d65b48c, 0x605bc8ac,
88 0x7b05c64b, 0x45dc1c87, 0x52e96f46, 0x562ffc29, 0x55a27ee1, 0x0b82e4e5, 0x5ef3f7cd, 0x3ab42017
89])
90
91// Precomputed masks per tier (controls cut-point frequency)
92// Below average: larger mask → fewer cuts → chunks grow toward average
93// Above average: smaller mask → more cuts → chunks stay near average
94const TIER_MASKS = TIERS.map(t => ({
95 large: (1 << Math.ceil(Math.log2(t.avgSize))) - 1,
96 small: (1 << Math.floor(Math.log2(t.avgSize))) - 1
97}))
98
99/**
100 * Chunk a buffer using FastCDC with adaptive tier selection
101 *
102 * @param {Buffer} data - Data to chunk
103 * @param {{ tier?: number }} [opts] - Options. tier: force a specific tier (0-3). If omitted, auto-selects based on data.length.
104 * @returns {Array<{ hash: string, offset: number, length: number, data: Buffer }>}
105 */
106export function chunk (data, opts) {
107 if (data.length === 0) return []
108
109 const tier = (opts && typeof opts.tier === 'number') ? opts.tier : selectTier(data.length)
110 const params = TIERS[tier] || TIERS[0]
111 const masks = TIER_MASKS[tier] || TIER_MASKS[0]
112
113 const chunks = []
114 let offset = 0
115
116 while (offset < data.length) {
117 const remaining = data.length - offset
118
119 // If remaining data is smaller than minimum, it's the final chunk
120 if (remaining <= params.minSize) {
121 const slice = data.subarray(offset, offset + remaining)
122 chunks.push({
123 hash: hashChunk(slice),
124 offset,
125 length: remaining,
126 data: slice
127 })
128 break
129 }
130
131 const maxLen = Math.min(remaining, params.maxSize)
132 let hash = 0
133 let i = params.minSize
134
135 // Phase 1: below average — use large mask (fewer cuts)
136 const avgBound = Math.min(params.avgSize, maxLen)
137 while (i < avgBound) {
138 hash = ((hash << 1) + GEAR[data[offset + i]]) >>> 0
139 if ((hash & masks.large) === 0) break
140 i++
141 }
142
143 // Phase 2: above average — use small mask (more cuts)
144 if (i >= avgBound) {
145 while (i < maxLen) {
146 hash = ((hash << 1) + GEAR[data[offset + i]]) >>> 0
147 if ((hash & masks.small) === 0) break
148 i++
149 }
150 }
151
152 const slice = data.subarray(offset, offset + i)
153 chunks.push({
154 hash: hashChunk(slice),
155 offset,
156 length: i,
157 data: slice
158 })
159 offset += i
160 }
161
162 return chunks
163}
164
165/**
166 * Hash a chunk using blake2b-256
167 * @param {Buffer} data
168 * @returns {string} hex hash
169 */
170export function hashChunk (data) {
171 return crypto.data(data).toString('hex')
172}
173
174// ── Streaming primitives ──────────────────────────────────────
175//
176// `chunk(buffer)` requires the entire file in memory and returns chunks
177// whose `.data` is a `Buffer.subarray` of the input — both keep the
178// original buffer pinned until every yielded chunk is GC'd. For
179// multi-GB files (or limited-RAM runtimes) use the streaming versions
180// below: `chunkStream` consumes a Readable and yields chunks one at a
181// time, releasing bytes after each yield.
182
183import sodium from 'sodium-universal'
184import c from 'compact-encoding'
185
186const LEAF_TYPE = Buffer.from([0])
187
188/**
189 * Streaming blake2b-256 hasher matching `hashChunk(buffer)`'s digest
190 * format (hypercore-crypto's `crypto.data` shape:
191 * blake2b(LEAF_TYPE || uint64-LE(size) || data)
192 * ). Lets you hash multi-GB files without buffering them.
193 *
194 * const h = createStreamingHasher({ size }) // size known up front
195 * for await (const buf of stream) h.update(buf)
196 * const oid = h.digest() // == hashChunk(file)
197 *
198 * `size` is required for hash compatibility with `hashChunk` — the
199 * leaf-prefix construction includes the byte length, which is fixed at
200 * stream open. If you don't have the size, omit `size` and you'll get
201 * a plain blake2b digest of the bytes (does NOT match hashChunk).
202 */
203export function createStreamingHasher (opts = {}) {
204 const state = Buffer.alloc(sodium.crypto_generichash_STATEBYTES)
205 let bytes = 0
206 if (opts && opts.restore) {
207 // The saved state already includes the init + leaf/size prefix + every byte
208 // consumed so far, so do not re-init.
209 opts.restore.state.copy(state)
210 bytes = opts.restore.bytes
211 } else {
212 sodium.crypto_generichash_init(state, null, 32)
213 const knownSize = (opts && typeof opts.size === 'number') ? opts.size : null
214 if (knownSize !== null) {
215 sodium.crypto_generichash_update(state, LEAF_TYPE)
216 sodium.crypto_generichash_update(state, c.encode(c.uint64, knownSize))
217 }
218 }
219 return {
220 update (buf) {
221 sodium.crypto_generichash_update(state, buf)
222 bytes += buf.length
223 },
224 digest () {
225 const out = Buffer.alloc(32)
226 sodium.crypto_generichash_final(state, out)
227 return out.toString('hex')
228 },
229 snapshot () { return { state: Buffer.from(state), bytes } },
230 get bytes () { return bytes }
231 }
232}
233
234// Find a FastCDC cut point inside `data` starting at offset 0. Returns
235// the chunk length. Mirrors the inner search loop of `chunk()` but
236// takes pre-resolved params + masks so it works in a streaming loop.
237function findCutPoint (data, params, masks) {
238 const remaining = data.length
239 if (remaining <= params.minSize) return remaining
240
241 const maxLen = Math.min(remaining, params.maxSize)
242 let hash = 0
243 let i = params.minSize
244
245 const avgBound = Math.min(params.avgSize, maxLen)
246 while (i < avgBound) {
247 hash = ((hash << 1) + GEAR[data[i]]) >>> 0
248 if ((hash & masks.large) === 0) return i
249 i++
250 }
251 if (i >= avgBound) {
252 while (i < maxLen) {
253 hash = ((hash << 1) + GEAR[data[i]]) >>> 0
254 if ((hash & masks.small) === 0) return i
255 i++
256 }
257 }
258 return maxLen
259}
260
261// Opt-in memcpy accounting for perf validation. Zero cost when off (one
262// boolean check at the increment sites). Module-level singletons — enabled
263// only by tests + bench/prepare.js, never in library code; NOT
264// concurrency-safe, so do not enable it in production.
265let STATS_ON = false
266export const chunkStats = { concatBytes: 0, copyBytes: 0, blocks: 0, chunks: 0 }
267export function resetChunkStats () { chunkStats.concatBytes = 0; chunkStats.copyBytes = 0; chunkStats.blocks = 0; chunkStats.chunks = 0 }
268export function setChunkStats (on) { STATS_ON = !!on; if (on) resetChunkStats() }
269
270/**
271 * Stream-friendly FastCDC chunker. Consumes an AsyncIterable<Buffer>
272 * (e.g. fs.createReadStream) and yields chunks one at a time. Memory
273 * footprint ≈ 2 × params.maxSize regardless of stream length —
274 * chunks are emitted as bytes arrive, the pending buffer is sliced
275 * past each cut point.
276 *
277 * Each yielded chunk has the same shape as `chunk()` entries:
278 * { hash, offset, length, data }
279 * With the default `copy: true`, `data` is a fresh `Buffer.from(slice)`
280 * detached from the input stream's pool, so callers can store/forward it.
281 * With `copy: false`, `data` is a view backed by the reader's memory —
282 * valid only until the next iteration; the caller MUST consume it
283 * synchronously and not retain it across an await.
284 *
285 * Cut-point determinism: same content + same tier → identical chunk
286 * boundaries as `chunk()`, regardless of input block size. Peers using
287 * either function on the same file produce matching chunk hashes.
288 *
289 * @param {AsyncIterable<Buffer>} readable
290 * @param {{ tier?: number, copy?: boolean }} [opts] tier index (0–3); copy
291 * defaults true. Callers with a known file size should pass `selectTier(size)`.
292 */
293export async function * chunkStream (readable, opts = {}) {
294 const tier = (opts && typeof opts.tier === 'number') ? opts.tier : 0
295 const copy = opts.copy !== false // false → yield views (see JSDoc above)
296 const params = TIERS[tier] || TIERS[0]
297 const masks = TIER_MASKS[tier] || TIER_MASKS[0]
298
299 let pending = Buffer.alloc(0)
300 let absoluteOffset = 0
301
302 // Single emit point for the stats accounting + copy:false decision.
303 const emit = (slice) => {
304 if (STATS_ON) { chunkStats.chunks++; if (copy) chunkStats.copyBytes += slice.length }
305 return { hash: hashChunk(slice), offset: absoluteOffset, length: slice.length, data: copy ? Buffer.from(slice) : slice }
306 }
307
308 for await (const inChunk of readable) {
309 const buf = Buffer.isBuffer(inChunk) ? inChunk : Buffer.from(inChunk)
310 if (STATS_ON) { chunkStats.blocks++; if (pending.length !== 0) chunkStats.concatBytes += pending.length + buf.length }
311 pending = pending.length === 0 ? buf : Buffer.concat([pending, buf])
312
313 // While we have at least maxSize ahead, we can definitively find
314 // the next cut point inside `pending` (the cut is bounded by maxSize).
315 while (pending.length >= params.maxSize) {
316 const cutLen = findCutPoint(pending, params, masks)
317 yield emit(pending.subarray(0, cutLen))
318 absoluteOffset += cutLen
319 pending = pending.subarray(cutLen)
320 }
321 }
322
323 // Stream ended. Process leftover. Same minSize-final-chunk rule as
324 // the buffer-mode `chunk()`.
325 while (pending.length > params.minSize) {
326 const cutLen = findCutPoint(pending, params, masks)
327 yield emit(pending.subarray(0, cutLen))
328 absoluteOffset += cutLen
329 pending = pending.subarray(cutLen)
330 }
331
332 if (pending.length > 0) yield emit(pending)
333}
lib/v3/file-index.js +0 -576
……@@ -1,576 +0,0 @@
1/**
2 * FileIndex — Metadata-only file index backed by Hyperbee
3 *
4 * Stores file metadata and chunk maps. Never stores file content.
5 * The filesystem is the source of truth for actual bytes.
6 *
7 * Hyperbee schema:
8 * file:<path> → { contentHash, size, mtime, hashing, executable, linkname }
9 * chunkmap:<path> → [{ hash, offset, length }] (small) | { __paged: N } (large; see below)
10 * chunkmap-oid:<contentHash> → [{ hash, offset, length }] (content-addressed; reused across overlay keys + restarts)
11 * sync:<peerKey>:<path> → { lastSeq, lastHash }
12 * config:sync → { folders: [...] }
13 *
14 * tree:<hex-hash> → { entries: [{kind, exec, name, childHash, size}], size }
15 * treepath:<path> → <hex-hash> (secondary: path → current tree hash)
16 *
17 * Chunk-map paging. A chunk map for a very large file (a 1.1 TB
18 * file at tier 3 ≈ 1M entries ≈ ~120 MB of JSON) does not fit in one Hyperbee
19 * value: Hypercore caps a block at 15 MiB (MAX_SUGGESTED_BLOCK_SIZE) and throws
20 * BAD_ARGUMENT above it. Maps with > CHUNKS_PER_PAGE entries are stored as a
21 * header value { __paged: N } at the base key plus N page values keyed
22 * `<base>\x00<i>` (the NUL byte never occurs in a path or hex hash, so a page key
23 * can't collide with a real one). Small maps stay inline as a plain array —
24 * byte-identical to before, and back-compatible with already-stored maps. The
25 * public API (put/get/del) is unchanged; paging is transparent to callers.
26 */
27
28import Hyperbee from 'hyperbee'
29import ReadyResource from 'ready-resource'
30
31// Entries per chunk-map page. ~116 B/entry worst-case JSON ⇒
32// ≤ ~3.8 MB/page, well under Hypercore's 15 MiB block limit.
33const CHUNKS_PER_PAGE = 32768
34
35// Overlay-path prefix for host-registered, content-addressed serve entries.
36// Hosts that make a file servable purely by content hash should register it at
37// `${CONTENT_REGISTER_PREFIX}${contentHash}` so compact()/evictContent() can
38// recognize (and reclaim) the entry once the hash is no longer served.
39export const CONTENT_REGISTER_PREFIX = '/mir/'
40
41// The content hash an owner-side entry is addressed by (page suffix folded in), or
42// null for entries not keyed by a single content hash (real-path file:/chunkmap:,
43// tree:, sync:, config:). compact() drops the content-addressed entries whose hash is
44// no longer served, and keeps everything else.
45function contentHashOfKey (key) {
46 if (key.startsWith('chunkmap-oid:')) return key.slice('chunkmap-oid:'.length).split('\x00')[0]
47 if (key.startsWith('chunkmap:content:')) return key.slice('chunkmap:content:'.length).split('\x00')[0]
48 if (key.startsWith(`chunkmap:${CONTENT_REGISTER_PREFIX}`)) return key.slice(`chunkmap:${CONTENT_REGISTER_PREFIX}`.length).split('\x00')[0]
49 if (key.startsWith(`file:${CONTENT_REGISTER_PREFIX}`)) return key.slice(`file:${CONTENT_REGISTER_PREFIX}`.length)
50 return null
51}
52
53// v1 keeps the original 'file-index' name so existing stores aren't orphaned on
54// upgrade; compaction moves to file-index-v2, v3, … and purges the predecessor.
55export function indexCoreName (version) {
56 return version === 1 ? 'file-index' : `file-index-v${version}`
57}
58
59export class FileIndex extends ReadyResource {
60 constructor (store, opts = {}) {
61 super()
62 this._store = store
63 this._bee = null
64 this._meta = null
65 this._version = 1
66 this._opts = opts
67 }
68
69 // Local index cores are encrypted at rest when the caller passes an encryptionKey
70 // (opts.encryptionKey); absent, they stay plaintext.
71 _coreOpts (name) {
72 const o = { name, valueEncoding: 'binary' }
73 if (this._opts.encryptionKey) o.encryptionKey = this._opts.encryptionKey
74 return o
75 }
76
77 async _open () {
78 this._meta = new Hyperbee(this._store.get(this._coreOpts('index-meta')), {
79 keyEncoding: 'utf-8',
80 valueEncoding: 'json'
81 })
82 await this._meta.ready()
83 this._version = (await this._meta.get('version'))?.value ?? 1
84 // A prior purge can leave a version's core with a dangling name alias — its data and
85 // by-discovery-key alias were deleted but the by-name alias was not — which throws
86 // STORAGE_EMPTY on open. The index is rebuildable cache, so advance to a fresh core
87 // name and record it rather than failing open.
88 for (let attempts = 0; ; attempts++) {
89 const bee = new Hyperbee(this._store.get(this._coreOpts(indexCoreName(this._version))), {
90 keyEncoding: 'utf-8',
91 valueEncoding: 'json'
92 })
93 try {
94 await bee.ready()
95 this._bee = bee
96 return
97 } catch (err) {
98 if (err.code !== 'STORAGE_EMPTY' || attempts >= 64) throw err
99 await bee.close().catch(() => {})
100 this._version += 1
101 await this._meta.put('version', this._version)
102 }
103 }
104 }
105
106 async _close () {
107 if (this._bee) await this._bee.close()
108 if (this._meta) await this._meta.close()
109 }
110
111 get bee () { return this._bee }
112
113 get version () { return this._version }
114
115 get cores () { return [this._bee?.core, this._meta?.core].filter(Boolean) }
116
117 // Reclaim the append-only index: stream the still-served entries into a fresh
118 // versioned core, flip the version pointer, and return the old core so the caller
119 // can clear+purge it (the only way to return an append-only bee's disk to the OS).
120 // Content-addressed entries (maps + content-register entries) whose hash isServed() is false
121 // are dropped; real-path, tree, sync and config entries are always kept.
122 async compact ({ isServed }) {
123 // Skip the rewrite entirely when nothing is droppable — otherwise a compaction of
124 // an already-clean index just churns (a fresh version core + a version-marker
125 // append), which grows the index without reclaiming anything.
126 let droppable = false
127 for await (const { key } of this._bee.createReadStream()) {
128 const hash = contentHashOfKey(key)
129 if (hash !== null && !isServed(hash)) { droppable = true; break }
130 }
131 if (!droppable) return null
132
133 const next = this._version + 1
134 const dst = new Hyperbee(this._store.get(this._coreOpts(indexCoreName(next))), {
135 keyEncoding: 'utf-8',
136 valueEncoding: 'json'
137 })
138 await dst.ready()
139 let batch = dst.batch()
140 let pending = 0
141 for await (const { key, value } of this._bee.createReadStream()) {
142 const hash = contentHashOfKey(key)
143 if (hash !== null && !isServed(hash)) continue
144 await batch.put(key, value)
145 if (++pending >= 500) { await batch.flush(); batch = dst.batch(); pending = 0 }
146 }
147 await batch.flush()
148
149 const oldCore = this._bee.core
150 await this._meta.put('version', next)
151 this._bee = dst
152 this._version = next
153 return oldCore // left open; caller clears + purges it
154 }
155
156 // ── File entries ────────────────────────────────────────────
157
158 /**
159 * Store or update a file entry (metadata only, no content)
160 * @param {string} path - file path
161 * @param {{ contentHash: string|null, size: number, mtime: number, hashing?: boolean, executable?: boolean, linkname?: string|null }} meta
162 */
163 async putFile (path, meta) {
164 await this._bee.put(`file:${path}`, {
165 contentHash: meta.contentHash || null,
166 size: meta.size,
167 mtime: meta.mtime,
168 hashing: meta.hashing || false,
169 executable: meta.executable || false,
170 linkname: meta.linkname || null
171 })
172 }
173
174 /**
175 * Get file metadata
176 * @param {string} path
177 * @returns {{ contentHash: string|null, size: number, mtime: number, hashing: boolean, executable: boolean, linkname: string|null } | null}
178 */
179 async getFile (path) {
180 const entry = await this._bee.get(`file:${path}`)
181 return entry ? entry.value : null
182 }
183
184 /**
185 * Check if file exists in the index
186 * @param {string} path
187 * @returns {boolean}
188 */
189 async hasFile (path) {
190 const entry = await this._bee.get(`file:${path}`)
191 return entry !== null
192 }
193
194 /**
195 * Remove a file entry and its chunk map
196 * @param {string} path
197 */
198 async delFile (path) {
199 await this._bee.del(`file:${path}`)
200 await this._delPagedValue(`chunkmap:${path}`)
201 }
202
203 /**
204 * List files under a directory prefix
205 * @param {string} [dir=''] - directory prefix
206 * @returns {Array<{ path: string, contentHash: string|null, size: number, mtime: number }>}
207 */
208 async listFiles (dir = '') {
209 const prefix = dir ? `file:${dir}` : 'file:'
210 const files = []
211
212 for await (const entry of this._bee.createReadStream({ gt: prefix, lt: prefix + '\xff' })) {
213 files.push({
214 path: entry.key.slice('file:'.length),
215 ...entry.value
216 })
217 }
218
219 return files
220 }
221
222 // ── Index meta (from hyper-overlay 0.2.14; carried in v3) ───
223 // Small key/value bag on the main bee under `meta:` — used by the fast
224 // manifest-diff boot path (e.g. hyper-deploy). compact() keeps these
225 // (contentHashOfKey returns null for `meta:` → not droppable).
226
227 async putIndexMeta (key, value) {
228 await this._bee.put(`meta:${key}`, value)
229 }
230
231 async getIndexMeta (key) {
232 const entry = await this._bee.get(`meta:${key}`)
233 return entry ? entry.value : null
234 }
235
236 // ── Chunk maps ──────────────────────────────────────────────
237
238 /**
239 * Store a chunk map for a file (hash + offset + length per chunk, no data).
240 * Large maps are paged transparently (see the header note) so they never exceed the
241 * Hypercore block limit.
242 * @param {string} path - file path
243 * @param {Array<{ hash: string, offset: number, length: number }>} chunks
244 */
245 async putChunkMap (path, chunks) {
246 await this._putPagedValue(`chunkmap:${path}`, chunks)
247 }
248
249 /**
250 * Get the chunk map for a file
251 * @param {string} path
252 * @returns {Array<{ hash: string, offset: number, length: number }> | null}
253 */
254 async getChunkMap (path) {
255 return this._getPagedValue(`chunkmap:${path}`)
256 }
257
258 /**
259 * Check if a chunk map exists for a file
260 * @param {string} path
261 * @returns {boolean}
262 */
263 async hasChunkMap (path) {
264 const entry = await this._bee.get(`chunkmap:${path}`)
265 return entry !== null
266 }
267
268 /**
269 * Remove a chunk map
270 * @param {string} path
271 */
272 async delChunkMap (path) {
273 await this._delPagedValue(`chunkmap:${path}`)
274 }
275
276 // Paged-value storage. Small maps stay inline as a plain array
277 // (unchanged on disk, back-compatible); maps over CHUNKS_PER_PAGE entries are
278 // split into a { __paged: N } header + N `<base>\x00<i>` page values, written
279 // in one Hyperbee batch so the whole map commits atomically. Stale pages from
280 // a previously-larger map at the same key are deleted in the same batch.
281 async _putPagedValue (baseKey, chunks) {
282 const prevPages = await this._pagedCount(baseKey)
283
284 if (chunks.length <= CHUNKS_PER_PAGE) {
285 if (prevPages === 0) {
286 await this._bee.put(baseKey, chunks)
287 return
288 }
289 await this._batched(async (b) => {
290 await b.put(baseKey, chunks)
291 for (let i = 0; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`)
292 })
293 return
294 }
295
296 const pageCount = Math.ceil(chunks.length / CHUNKS_PER_PAGE)
297 await this._batched(async (b) => {
298 await b.put(baseKey, { __paged: pageCount })
299 for (let i = 0; i < pageCount; i++) {
300 await b.put(`${baseKey}\x00${i}`, chunks.slice(i * CHUNKS_PER_PAGE, (i + 1) * CHUNKS_PER_PAGE))
301 }
302 for (let i = pageCount; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`)
303 })
304 }
305
306 async _getPagedValue (baseKey) {
307 const entry = await this._bee.get(baseKey)
308 if (!entry) return null
309 if (Array.isArray(entry.value)) return entry.value // inline / legacy
310 const pageCount = entry.value.__paged
311 if (!pageCount) return null
312 const chunks = []
313 for (let i = 0; i < pageCount; i++) {
314 const page = await this._bee.get(`${baseKey}\x00${i}`)
315 // The header + all pages commit in one atomic batch, so a missing or non-array
316 // page means the value is corrupt/incomplete. Return null — a clean "miss" so the
317 // caller re-chunks from the source file — never a silently truncated chunk map.
318 if (!page || !Array.isArray(page.value)) return null
319 for (const c of page.value) chunks.push(c)
320 }
321 return chunks
322 }
323
324 async _delPagedValue (baseKey) {
325 const prevPages = await this._pagedCount(baseKey)
326 if (prevPages === 0) {
327 await this._bee.del(baseKey)
328 return
329 }
330 await this._batched(async (b) => {
331 await b.del(baseKey)
332 for (let i = 0; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`)
333 })
334 }
335
336 // Page count of an existing paged value, or 0 if the key is absent or inline.
337 async _pagedCount (baseKey) {
338 const entry = await this._bee.get(baseKey)
339 if (!entry || Array.isArray(entry.value)) return 0
340 return entry.value.__paged || 0
341 }
342
343 // Run ops in a Hyperbee batch, closing it on error (flush releases the lock
344 // itself, even on append failure, via _appendBatch's finally).
345 async _batched (fn) {
346 const b = this._bee.batch()
347 try {
348 await fn(b)
349 await b.flush()
350 } catch (err) {
351 try { await b.close() } catch {}
352 throw err
353 }
354 }
355
356 // Content-addressed chunk map. FastCDC is deterministic on bytes, so a
357 // given content hash always chunks identically regardless of which overlay path
358 // it was registered under. Keying by content hash lets the publish-time chunking
359 // be reused at serve time (a synthetic 'content:<hash>' key) and across restarts,
360 // instead of re-reading the whole file to re-chunk it before the first byte ships.
361 async putChunkMapByHash (contentHash, chunks) {
362 await this._putPagedValue(`chunkmap-oid:${contentHash}`, chunks)
363 }
364
365 async getChunkMapByHash (contentHash) {
366 return this._getPagedValue(`chunkmap-oid:${contentHash}`)
367 }
368
369 async hasChunkMapByHash (contentHash) {
370 return (await this._bee.get(`chunkmap-oid:${contentHash}`)) !== null
371 }
372
373 async delChunkMapByHash (contentHash) {
374 await this._delPagedValue(`chunkmap-oid:${contentHash}`)
375 }
376
377 // Evict every durable entry a served content hash leaves behind: the content-
378 // addressed map plus the deterministic owner-side synthetic paths (the serve
379 // copy at content:<hash> and the register entry under CONTENT_REGISTER_PREFIX).
380 // Peer-path-keyed receiver entries are unknowable from the hash — the host's
381 // own index sweep handles those.
382 async evictContent (contentHash) {
383 await this.delChunkMapByHash(contentHash)
384 await this.delChunkMap(`content:${contentHash}`)
385 await this.delFile(`${CONTENT_REGISTER_PREFIX}${contentHash}`)
386 }
387
388 // ── Sync state ──────────────────────────────────────────────
389
390 /**
391 * Store sync state for a peer + file
392 * @param {string} peerKey - hex peer key
393 * @param {string} path - file path
394 * @param {{ lastSeq: number, lastHash: string }} state
395 */
396 async putSyncState (peerKey, path, state) {
397 await this._bee.put(`sync:${peerKey}:${path}`, {
398 lastSeq: state.lastSeq,
399 lastHash: state.lastHash
400 })
401 }
402
403 /**
404 * Get sync state for a peer + file
405 * @param {string} peerKey
406 * @param {string} path
407 * @returns {{ lastSeq: number, lastHash: string } | null}
408 */
409 async getSyncState (peerKey, path) {
410 const entry = await this._bee.get(`sync:${peerKey}:${path}`)
411 return entry ? entry.value : null
412 }
413
414 /**
415 * Remove sync state for a peer + file
416 * @param {string} peerKey
417 * @param {string} path
418 */
419 async delSyncState (peerKey, path) {
420 await this._bee.del(`sync:${peerKey}:${path}`)
421 }
422
423 /**
424 * List all sync states for a peer
425 * @param {string} peerKey
426 * @returns {Array<{ path: string, lastSeq: number, lastHash: string }>}
427 */
428 async listSyncStates (peerKey) {
429 const prefix = `sync:${peerKey}:`
430 const states = []
431
432 for await (const entry of this._bee.createReadStream({ gt: prefix, lt: prefix + '\xff' })) {
433 states.push({
434 path: entry.key.slice(prefix.length),
435 ...entry.value
436 })
437 }
438
439 return states
440 }
441
442 // ── Sync config ─────────────────────────────────────────────
443
444 /**
445 * Store sync configuration
446 * @param {{ folders: Array<{ path: string, strategy: string, quota?: string|null, peers?: string[] }> }} config
447 */
448 async putSyncConfig (config) {
449 await this._bee.put('config:sync', config)
450 }
451
452 /**
453 * Get sync configuration
454 * @returns {{ folders: Array } | null}
455 */
456 async getSyncConfig () {
457 const entry = await this._bee.get('config:sync')
458 return entry ? entry.value : null
459 }
460
461 // ── Trees ───────────────────────────────────────────────────
462
463 /**
464 * Store a tree by its content-addressed hash.
465 * Same hash put twice is a no-op (content-addressed — bytes can't differ).
466 *
467 * @param {string} hash - hex-encoded blake2b-256 tree hash
468 * @param {{ entries: Array, size: number }} tree
469 */
470 async putTree (hash, tree) {
471 await this._bee.put(`tree:${hash}`, {
472 entries: tree.entries,
473 size: tree.size
474 })
475 }
476
477 /**
478 * Get a tree by its hash.
479 * @param {string} hash
480 * @returns {{ entries: Array, size: number } | null}
481 */
482 async getTree (hash) {
483 const entry = await this._bee.get(`tree:${hash}`)
484 return entry ? entry.value : null
485 }
486
487 /**
488 * Check if a tree is present.
489 * @param {string} hash
490 * @returns {boolean}
491 */
492 async hasTree (hash) {
493 const entry = await this._bee.get(`tree:${hash}`)
494 return entry !== null
495 }
496
497 /**
498 * Remove a tree. Only safe when no treepath entries reference this hash.
499 * @param {string} hash
500 */
501 async delTree (hash) {
502 await this._bee.del(`tree:${hash}`)
503 }
504
505 /**
506 * Record the current tree hash for a path (secondary index).
507 * @param {string} path
508 * @param {string} hash
509 */
510 async putTreePath (path, hash) {
511 await this._bee.put(`treepath:${path}`, hash)
512 }
513
514 /**
515 * Look up the current tree hash for a path.
516 * @param {string} path
517 * @returns {string|null}
518 */
519 async getTreePath (path) {
520 const entry = await this._bee.get(`treepath:${path}`)
521 return entry ? entry.value : null
522 }
523
524 /**
525 * Remove the treepath entry. Does NOT delete the underlying tree — other
526 * paths may still reference the same hash.
527 * @param {string} path
528 */
529 async delTreePath (path) {
530 await this._bee.del(`treepath:${path}`)
531 }
532
533 /**
534 * List all stored tree hashes.
535 * @param {object} [opts]
536 * @param {number} [opts.limit] - optional cap
537 * @returns {Array<{ hash: string, entryCount: number, size: number }>}
538 */
539 async listTrees (opts = {}) {
540 const trees = []
541 for await (const entry of this._bee.createReadStream({ gt: 'tree:', lt: 'tree:\xff' })) {
542 trees.push({
543 hash: entry.key.slice('tree:'.length),
544 entryCount: entry.value.entries.length,
545 size: entry.value.size
546 })
547 if (opts.limit && trees.length >= opts.limit) break
548 }
549 return trees
550 }
551
552 // ── Stats ───────────────────────────────────────────────────
553
554 /**
555 * Get index statistics
556 * @returns {{ fileCount: number, chunkMapCount: number, syncStateCount: number, treeCount: number, treePathCount: number }}
557 */
558 async stats () {
559 let fileCount = 0
560 let chunkMapCount = 0
561 let syncStateCount = 0
562 let treeCount = 0
563 let treePathCount = 0
564
565 for await (const entry of this._bee.createReadStream()) {
566 if (entry.key.startsWith('file:')) fileCount++
567 // Count one per file — skip the `\x00`-suffixed page values.
568 else if (entry.key.startsWith('chunkmap:') && !entry.key.includes('\x00')) chunkMapCount++
569 else if (entry.key.startsWith('sync:')) syncStateCount++
570 else if (entry.key.startsWith('tree:')) treeCount++
571 else if (entry.key.startsWith('treepath:')) treePathCount++
572 }
573
574 return { fileCount, chunkMapCount, syncStateCount, treeCount, treePathCount }
575 }
576}
lib/v3/index.js +0 -46
……@@ -1,46 +0,0 @@
1/**
2 * hyper-overlay — content-addressed P2P file distribution over Protomux.
3 *
4 * The canonical bytes are REAL FILES ON DISK; the overlay stores only
5 * metadata + chunk maps (Hyperbee), so there is no blob-storage balloon.
6 * Files are chunked with FastCDC, fetched multi-source (torrent-style),
7 * hash-verified incrementally, and written straight to the requester's disk.
8 *
9 * Main entry point: `HyperOverlay` (also the default export). The submodules
10 * are available as subpath exports (`hyper-overlay/chunker`, …) for consumers
11 * that only need a slice of the stack.
12 *
13 * Runs on Bare. The pure modules — chunker, messages, chunk-scheduler,
14 * file-index, sync-engine — also run on Node.
15 */
16
17export { HyperOverlay, HyperOverlay as default } from './lib/overlay.js'
18export { OverlayProtocol } from './lib/protocol.js'
19export {
20 TransferManager,
21 PARTIAL_SUFFIX,
22 partialPathFor,
23 journalNameFor,
24 cleanupOrphanedJournals
25} from './lib/transfer.js'
26export { ChunkScheduler } from './lib/chunk-scheduler.js'
27export { FileIndex, CONTENT_REGISTER_PREFIX } from './lib/file-index.js'
28export { SyncEngine, STRATEGIES } from './lib/sync-engine.js'
29export {
30 chunk,
31 chunkStream,
32 hashChunk,
33 createStreamingHasher,
34 selectTier,
35 getTierParams,
36 TIERS,
37 MIN_SIZE,
38 AVG_SIZE,
39 MAX_SIZE
40} from './lib/chunker.js'
41export * as messages from './lib/messages.js'
42
43// Transitional aliases for consumers migrating from the pre-3.0 v2 naming
44// (`overlay-v2.js` / `protocol-v2.js`). Same classes; prefer the new names.
45export { HyperOverlay as HyperOverlayV2 } from './lib/overlay.js'
46export { OverlayProtocol as OverlayProtocolV2 } from './lib/protocol.js'
lib/v3/messages.js +0 -444
……@@ -1,444 +0,0 @@
1/**
2 * Wire Protocol Messages for hyper-overlay/v2
3 *
4 * 13 message types for sync + transfer + trees (0.5a extension):
5 * 0-2: Sync messages (state exchange, file offers/requests)
6 * 3-8: Transfer messages (chunk hashes, data, completion)
7 * 9-11: Tree messages (tree-request, tree-response, content-request)
8 * 12: transfer-control (downloader→holder pause/stop notice)
9 *
10 * Messages 9-12 are backward-compatible additions — peers without
11 * this code simply don't register the corresponding protomux slots, and
12 * senders see no receiver for those messages. File-level sync (0-8)
13 * keeps working unchanged.
14 *
15 * All use compact-encoding codecs with preencode/encode/decode.
16 */
17
18import c from 'compact-encoding'
19
20// ── Helpers ───────────────────────────────────────────────────
21
22function toBuffer32 (hashOrBuf) {
23 if (Buffer.isBuffer(hashOrBuf)) return hashOrBuf
24 return Buffer.from(hashOrBuf, 'hex')
25}
26
27// Encode/decode an array of { hash (32-byte hex), length (uint) }
28const chunkInfoArray = {
29 preencode (state, arr) {
30 c.uint32.preencode(state, arr.length)
31 for (const item of arr) {
32 state.end += 32
33 c.uint.preencode(state, item.length)
34 }
35 },
36 encode (state, arr) {
37 c.uint32.encode(state, arr.length)
38 for (const item of arr) {
39 toBuffer32(item.hash).copy(state.buffer, state.start, 0, 32)
40 state.start += 32
41 c.uint.encode(state, item.length)
42 }
43 },
44 decode (state) {
45 const len = c.uint32.decode(state)
46 const arr = []
47 for (let i = 0; i < len; i++) {
48 const hash = state.buffer.subarray(state.start, state.start + 32).toString('hex')
49 state.start += 32
50 const length = c.uint.decode(state)
51 arr.push({ hash, length })
52 }
53 return arr
54 }
55}
56
57// Encode/decode an array of uints
58const uintArray = {
59 preencode (state, arr) {
60 c.uint32.preencode(state, arr.length)
61 for (const v of arr) c.uint.preencode(state, v)
62 },
63 encode (state, arr) {
64 c.uint32.encode(state, arr.length)
65 for (const v of arr) c.uint.encode(state, v)
66 },
67 decode (state) {
68 const len = c.uint32.decode(state)
69 const arr = []
70 for (let i = 0; i < len; i++) arr.push(c.uint.decode(state))
71 return arr
72 }
73}
74
75// ── Handshake ─────────────────────────────────────────────────
76
77export const handshake = {
78 preencode (state, m) {
79 c.uint8.preencode(state, m.version)
80 c.uint8.preencode(state, m.capabilities)
81 },
82 encode (state, m) {
83 c.uint8.encode(state, m.version)
84 c.uint8.encode(state, m.capabilities)
85 },
86 decode (state) {
87 return {
88 version: c.uint8.decode(state),
89 capabilities: c.uint8.decode(state)
90 }
91 }
92}
93
94// ── Sync Messages (0-2) ──────────────────────────────────────
95
96// Message 0: sync-state — exchange last-synced sequence numbers
97export const syncState = {
98 preencode (state, m) {
99 c.buffer.preencode(state, toBuffer32(m.feedKey))
100 c.uint.preencode(state, m.localSeq)
101 c.uint.preencode(state, m.remoteSeq)
102 },
103 encode (state, m) {
104 c.buffer.encode(state, toBuffer32(m.feedKey))
105 c.uint.encode(state, m.localSeq)
106 c.uint.encode(state, m.remoteSeq)
107 },
108 decode (state) {
109 return {
110 feedKey: c.buffer.decode(state).toString('hex'),
111 localSeq: c.uint.decode(state),
112 remoteSeq: c.uint.decode(state)
113 }
114 }
115}
116
117// Message 1: file-offer — announce a file change
118export const fileOffer = {
119 preencode (state, m) {
120 c.string.preencode(state, m.path)
121 c.buffer.preencode(state, toBuffer32(m.contentHash))
122 c.uint.preencode(state, m.size)
123 c.uint.preencode(state, m.mtime)
124 c.uint8.preencode(state, m.op)
125 },
126 encode (state, m) {
127 c.string.encode(state, m.path)
128 c.buffer.encode(state, toBuffer32(m.contentHash))
129 c.uint.encode(state, m.size)
130 c.uint.encode(state, m.mtime)
131 c.uint8.encode(state, m.op)
132 },
133 decode (state) {
134 return {
135 path: c.string.decode(state),
136 contentHash: c.buffer.decode(state).toString('hex'),
137 size: c.uint.decode(state),
138 mtime: c.uint.decode(state),
139 op: c.uint8.decode(state)
140 }
141 }
142}
143
144// Message 2: file-request — request a file transfer
145export const fileRequest = {
146 preencode (state, m) {
147 c.string.preencode(state, m.path)
148 c.buffer.preencode(state, toBuffer32(m.contentHash))
149 c.buffer.preencode(state, m.chunksHave || Buffer.alloc(0))
150 },
151 encode (state, m) {
152 c.string.encode(state, m.path)
153 c.buffer.encode(state, toBuffer32(m.contentHash))
154 c.buffer.encode(state, m.chunksHave || Buffer.alloc(0))
155 },
156 decode (state) {
157 return {
158 path: c.string.decode(state),
159 contentHash: c.buffer.decode(state).toString('hex'),
160 chunksHave: c.buffer.decode(state)
161 }
162 }
163}
164
165// ── Transfer Messages (3-8) ──────────────────────────────────
166
167// Message 3: chunk-hashes — chunk hash list for a requested file
168//
169// `more` (paging flag) is appended last. A very large file's
170// chunk list serializes past the 16 MiB-1 Noise frame limit
171// (@hyperswarm/secret-stream MAX_ATOMIC_WRITE), so the protocol splits it into
172// several chunkHashes frames: every page but the last sets more:1, the final
173// page sets more:0. The receiver concatenates pages (in arrival order, keyed by
174// path) before dispatching. Appended last so a pre-paging peer that omits it
175// decodes more:0 — i.e. as a single, complete page (back-compatible).
176export const chunkHashes = {
177 preencode (state, m) {
178 c.string.preencode(state, m.path)
179 c.uint8.preencode(state, m.tier)
180 chunkInfoArray.preencode(state, m.chunks)
181 c.uint8.preencode(state, m.more || 0)
182 },
183 encode (state, m) {
184 c.string.encode(state, m.path)
185 c.uint8.encode(state, m.tier)
186 chunkInfoArray.encode(state, m.chunks)
187 c.uint8.encode(state, m.more || 0)
188 },
189 decode (state) {
190 const path = c.string.decode(state)
191 const tier = c.uint8.decode(state)
192 const chunks = chunkInfoArray.decode(state)
193 // A frame from a pre-paging peer has no trailing byte → more:0.
194 const more = state.start < state.end ? c.uint8.decode(state) : 0
195 return { path, tier, chunks, more }
196 }
197}
198
199// Message 4: chunk-need — tell sender which chunks to send
200export const chunkNeed = {
201 preencode (state, m) {
202 c.string.preencode(state, m.path)
203 uintArray.preencode(state, m.indices)
204 },
205 encode (state, m) {
206 c.string.encode(state, m.path)
207 uintArray.encode(state, m.indices)
208 },
209 decode (state) {
210 return {
211 path: c.string.decode(state),
212 indices: uintArray.decode(state)
213 }
214 }
215}
216
217// Message 5: chunk-data — send chunk bytes
218export const chunkData = {
219 preencode (state, m) {
220 c.string.preencode(state, m.path)
221 c.uint.preencode(state, m.index)
222 c.buffer.preencode(state, m.data)
223 },
224 encode (state, m) {
225 c.string.encode(state, m.path)
226 c.uint.encode(state, m.index)
227 c.buffer.encode(state, m.data)
228 },
229 decode (state) {
230 return {
231 path: c.string.decode(state),
232 index: c.uint.decode(state),
233 data: c.buffer.decode(state)
234 }
235 }
236}
237
238// Message 6: chunk-cancel — cancel a pending transfer
239export const chunkCancel = {
240 preencode (state, m) { c.string.preencode(state, m.path) },
241 encode (state, m) { c.string.encode(state, m.path) },
242 decode (state) { return { path: c.string.decode(state) } }
243}
244
245// Message 7: transfer-complete — confirm file fully received
246export const transferComplete = {
247 preencode (state, m) {
248 c.string.preencode(state, m.path)
249 c.buffer.preencode(state, toBuffer32(m.contentHash))
250 },
251 encode (state, m) {
252 c.string.encode(state, m.path)
253 c.buffer.encode(state, toBuffer32(m.contentHash))
254 },
255 decode (state) {
256 return {
257 path: c.string.decode(state),
258 contentHash: c.buffer.decode(state).toString('hex')
259 }
260 }
261}
262
263// Message 8: conflict — notify peer of a detected conflict
264export const conflict = {
265 preencode (state, m) {
266 c.string.preencode(state, m.path)
267 c.buffer.preencode(state, toBuffer32(m.myHash))
268 c.buffer.preencode(state, toBuffer32(m.theirHash))
269 c.buffer.preencode(state, toBuffer32(m.ancestorHash || Buffer.alloc(32)))
270 },
271 encode (state, m) {
272 c.string.encode(state, m.path)
273 c.buffer.encode(state, toBuffer32(m.myHash))
274 c.buffer.encode(state, toBuffer32(m.theirHash))
275 c.buffer.encode(state, toBuffer32(m.ancestorHash || Buffer.alloc(32)))
276 },
277 decode (state) {
278 return {
279 path: c.string.decode(state),
280 myHash: c.buffer.decode(state).toString('hex'),
281 theirHash: c.buffer.decode(state).toString('hex'),
282 ancestorHash: c.buffer.decode(state).toString('hex')
283 }
284 }
285}
286
287// ── Tree Messages (9-11) — 0.5a extension ────────────────────
288
289// Encode/decode an array of tree entries: { kind, exec, name, childHash, size }
290const treeEntryArray = {
291 preencode (state, arr) {
292 c.uint32.preencode(state, arr.length)
293 for (const e of arr) {
294 c.uint8.preencode(state, e.kind)
295 c.uint8.preencode(state, e.exec || 0)
296 c.string.preencode(state, e.name)
297 state.end += 32 // childHash fixed32
298 c.uint.preencode(state, e.size || 0)
299 }
300 },
301 encode (state, arr) {
302 c.uint32.encode(state, arr.length)
303 for (const e of arr) {
304 c.uint8.encode(state, e.kind)
305 c.uint8.encode(state, e.exec || 0)
306 c.string.encode(state, e.name)
307 toBuffer32(e.childHash).copy(state.buffer, state.start, 0, 32)
308 state.start += 32
309 c.uint.encode(state, e.size || 0)
310 }
311 },
312 decode (state) {
313 const len = c.uint32.decode(state)
314 const arr = []
315 for (let i = 0; i < len; i++) {
316 const kind = c.uint8.decode(state)
317 const exec = c.uint8.decode(state)
318 const name = c.string.decode(state)
319 const childHash = state.buffer.subarray(state.start, state.start + 32).toString('hex')
320 state.start += 32
321 const size = c.uint.decode(state)
322 arr.push({ kind, exec, name, childHash, size })
323 }
324 return arr
325 }
326}
327
328// Message 9: tree-request — ask peer for a tree by its content hash
329//
330// `nonce` (per-request id) is appended last; the holder echoes it in every
331// treeResponse page so the requester can drop pages from a superseded attempt.
332// A pre-nonce peer omits it and decodes 0, which the requester treats as
333// "unverifiable".
334export const treeRequest = {
335 preencode (state, m) {
336 c.buffer.preencode(state, toBuffer32(m.hash))
337 c.uint.preencode(state, m.nonce || 0)
338 },
339 encode (state, m) {
340 c.buffer.encode(state, toBuffer32(m.hash))
341 c.uint.encode(state, m.nonce || 0)
342 },
343 decode (state) {
344 const hash = c.buffer.decode(state).toString('hex')
345 const nonce = state.start < state.end ? c.uint.decode(state) : 0
346 return { hash, nonce }
347 }
348}
349
350// Message 10: tree-response — return a tree's entries to the requester
351//
352// `more` (paging flag) mirrors chunkHashes: a tree too large for one 16 MiB-1
353// Noise frame is split into several frames, every page but the last more:1.
354// `nonce` echoes the requester's per-request id so pages from a superseded
355// attempt are dropped. Both are appended last; a pre-paging peer omits them and
356// decodes 0.
357export const treeResponse = {
358 preencode (state, m) {
359 c.buffer.preencode(state, toBuffer32(m.hash))
360 treeEntryArray.preencode(state, m.entries)
361 c.uint8.preencode(state, m.more || 0)
362 c.uint.preencode(state, m.nonce || 0)
363 },
364 encode (state, m) {
365 c.buffer.encode(state, toBuffer32(m.hash))
366 treeEntryArray.encode(state, m.entries)
367 c.uint8.encode(state, m.more || 0)
368 c.uint.encode(state, m.nonce || 0)
369 },
370 decode (state) {
371 const hash = c.buffer.decode(state).toString('hex')
372 const entries = treeEntryArray.decode(state)
373 const more = state.start < state.end ? c.uint8.decode(state) : 0
374 const nonce = state.start < state.end ? c.uint.decode(state) : 0
375 return { hash, entries, more, nonce }
376 }
377}
378
379// Message 11: content-request — fetch a file by its content hash (not path)
380// Sender locates any local file matching the hash and serves its chunks.
381// `from` carries the requester's profile-key (hex) so the
382// holder's serveAuthorizer can authenticate the asker. Appended last → an
383// overlay peer that predates this field decodes '' (empty), never crashes.
384export const contentRequest = {
385 preencode (state, m) {
386 c.buffer.preencode(state, toBuffer32(m.contentHash))
387 c.buffer.preencode(state, m.chunksHave || Buffer.alloc(0))
388 c.string.preencode(state, m.from || '')
389 },
390 encode (state, m) {
391 c.buffer.encode(state, toBuffer32(m.contentHash))
392 c.buffer.encode(state, m.chunksHave || Buffer.alloc(0))
393 c.string.encode(state, m.from || '')
394 },
395 decode (state) {
396 return {
397 contentHash: c.buffer.decode(state).toString('hex'),
398 chunksHave: c.buffer.decode(state),
399 from: c.string.decode(state)
400 }
401 }
402}
403
404// Message 12: transfer-control — downloader→holder pause/stop notice for a
405// content-addressed fetch. state: 0 = paused, 1 = stopped. Appended last so a
406// holder that predates this never registers slot 12 and silently ignores it.
407export const transferControl = {
408 preencode (state, m) {
409 c.buffer.preencode(state, toBuffer32(m.contentHash))
410 c.uint8.preencode(state, m.state)
411 },
412 encode (state, m) {
413 c.buffer.encode(state, toBuffer32(m.contentHash))
414 c.uint8.encode(state, m.state)
415 },
416 decode (state) {
417 return {
418 contentHash: c.buffer.decode(state).toString('hex'),
419 state: c.uint8.decode(state)
420 }
421 }
422}
423
424// Message 13: transfer-progress — downloader→holder one-shot have-baseline for a
425// resumed content-addressed fetch. `have` = bytes the downloader already holds on
426// disk at resume, so the holder's "who is downloading" bar reflects the downloader's
427// TRUE completion, not only the bytes it re-serves. Appended last so a holder that
428// predates this never registers slot 13 and silently ignores it.
429export const transferProgress = {
430 preencode (state, m) {
431 c.buffer.preencode(state, toBuffer32(m.contentHash))
432 c.uint.preencode(state, m.have || 0)
433 },
434 encode (state, m) {
435 c.buffer.encode(state, toBuffer32(m.contentHash))
436 c.uint.encode(state, m.have || 0)
437 },
438 decode (state) {
439 return {
440 contentHash: c.buffer.decode(state).toString('hex'),
441 have: c.uint.decode(state)
442 }
443 }
444}
lib/v3/overlay.js +0 -417
……@@ -1,417 +0,0 @@
1/**
2 * HyperOverlay — content-addressed file serving over the V2 protocol.
3 *
4 * A thin facade over the V2 stack (FileIndex + SyncEngine + TransferManager +
5 * OverlayProtocol) for the "serve a file to whoever asks, fetch a file from
6 * whoever has it" use case — distinct from V1's local chunk-registry blobs.
7 *
8 * Model (the deliberate post-Hyperdrive design): the canonical bytes are a
9 * REAL FILE ON DISK. The index holds only metadata + chunk maps (no blob
10 * bytes), so there is no hypercore storage balloon. A peer fetches a file by
11 * its content hash; chunks are read from the source file on demand, streamed,
12 * hash-verified, and written to the requester's disk.
13 *
14 * `autoSync` is OFF: this is on-demand content fetch (requestContent →
15 * _onContentRequest), NOT Syncthing-style folder push. Registering a file
16 * makes it servable; nothing is offered to peers unsolicited.
17 *
18 * Requester flow: pre-register a synthetic `content:<hash>` → destPath
19 * mapping, requestContent,
20 * await the hash-verified transferComplete, walk peers on timeout/mismatch.
21 */
22
23import ReadyResource from 'ready-resource'
24import fs from 'bare-fs'
25import path from 'bare-path'
26import os from 'bare-os'
27import crypto from 'hypercore-crypto'
28
29import { FileIndex } from './file-index.js'
30import { SyncEngine } from './sync-engine.js'
31import { TransferManager } from './transfer.js'
32import { OverlayProtocol } from './protocol.js'
33import { createStreamingHasher } from './chunker.js'
34
35const DEFAULT_PER_PEER_TIMEOUT = 30000
36const DEFAULT_IDLE_TIMEOUT = 10000
37
38// Stream-verify a file against its overlay content hash WITHOUT buffering
39// it — a readFileSync of a multi-GB blob OOMs the process. Matches
40// crypto.data(buffer): blake2b(LEAF_TYPE || uint64-LE(size) || bytes), which is
41// exactly createStreamingHasher({ size }). Returns the size so callers can re-seed
42// without a second stat.
43async function verifyOnDisk (filePath, contentHash) {
44 let size
45 try { size = fs.statSync(filePath).size } catch { return { ok: false, size: 0 } }
46 const h = createStreamingHasher({ size })
47 try {
48 await new Promise((resolve, reject) => {
49 const rs = fs.createReadStream(filePath)
50 rs.on('data', (c) => h.update(c))
51 rs.on('end', resolve)
52 rs.on('error', reject)
53 })
54 } catch { return { ok: false, size } }
55 return { ok: h.digest() === contentHash, size }
56}
57
58export class HyperOverlay extends ReadyResource {
59 constructor (corestore, opts = {}) {
60 super()
61 this._corestore = corestore.namespace(opts.namespace || 'overlay-v2')
62 this._destDir = opts.destDir || path.join(os.tmpdir(), 'hyper-overlay-v2-fetch')
63 this._journalDir = opts.journalDir || null
64 this._syncBaseDirs = opts.syncBaseDirs || []
65 this._perPeerTimeout = opts.perPeerTimeout || DEFAULT_PER_PEER_TIMEOUT
66 this._idleTimeout = opts.idleTimeout || DEFAULT_IDLE_TIMEOUT
67
68 // Serve-authorization + identity opts threaded to OverlayProtocol.
69 // serveAuthorizer gates every inbound content-request; localProfileKey stamps
70 // outbound content-requests so the remote's gate can authenticate the asker.
71 this._serveAuthorizer = opts.serveAuthorizer || null
72 this._localProfileKey = opts.localProfileKey || null
73
74 // At-rest key for the local index cores (file-index/index-meta/sync-feed).
75 this._indexEncryptionKey = opts.indexEncryptionKey || null
76
77 // serve-side download telemetry (sender-side download indicator).
78 this._onServeStart = opts.onServeStart || null
79 this._onChunkServe = opts.onChunkServe || null
80 this._onServeEnd = opts.onServeEnd || null
81 this._onServeControl = opts.onServeControl || null
82 this._onServeProgress = opts.onServeProgress || null
83
84 // Shared maps the protocol reads: overlayPath → diskPath (for serving an
85 // offered/known path), contentHash → diskPath (fast path for content
86 // requests). Both populated by registerFile.
87 this._filePaths = new Map()
88 this._contentHashPaths = new Map()
89
90 // contentHash → pending fetch { destPath, finish, timers, bytesReceived }
91 this._pendingContent = new Map()
92
93 // Built lazily on first use (Hyperdrive-style: a drive that never touches
94 // its blobs core never opens one). A daemon that never registers, serves,
95 // or connects a peer for attachments pays nothing — no feed, no index.
96 this._index = null
97 this._sync = null
98 this._transfer = null
99 this._protocol = null
100 this._stackPromise = null
101 }
102
103 // The protocol must exist synchronously when attachProtocol is called —
104 // protomux will NOT pair a channel created after the remote opened theirs
105 // (verified). So the stack is built at ready time. _ensure stays as the
106 // idempotent builder (used by register/fetch, and as a safety net).
107 async _open () { await this._ensure() }
108
109 async _close () {
110 // Only tear down what was actually built.
111 if (this._protocol) this._protocol.destroy()
112 if (this._sync) await this._sync.close()
113 if (this._index) await this._index.close()
114 }
115
116 // Lazily build the V2 stack. Idempotent + concurrency-safe (a single
117 // in-flight promise is shared). No-op cost once built.
118 _ensure () {
119 if (this._protocol) return Promise.resolve()
120 if (this._stackPromise) return this._stackPromise
121 this._stackPromise = (async () => {
122 const index = new FileIndex(this._corestore, { encryptionKey: this._indexEncryptionKey })
123 await index.ready()
124 const sync = new SyncEngine(index, this._corestore, { encryptionKey: this._indexEncryptionKey })
125 await sync.ready()
126 const transfer = new TransferManager(index, { journalDir: this._journalDir })
127 const protocol = new OverlayProtocol(sync, transfer, {
128 filePaths: this._filePaths,
129 contentHashPaths: this._contentHashPaths,
130 syncBaseDirs: this._syncBaseDirs,
131 autoSync: false,
132 serveAuthorizer: this._serveAuthorizer, // serve gate
133 localProfileKey: this._localProfileKey, // outbound identity
134 onSynced: (info) => this._onSynced(info),
135 onChunkProgress: (info) => this._onChunkProgress(info),
136 onServeStart: this._onServeStart,
137 onChunkServe: this._onChunkServe,
138 onServeEnd: this._onServeEnd,
139 onServeControl: this._onServeControl,
140 onServeProgress: this._onServeProgress
141 })
142 try { fs.mkdirSync(this._destDir, { recursive: true }) } catch {}
143 if (this._journalDir) { try { fs.mkdirSync(this._journalDir, { recursive: true }) } catch {} }
144 this._index = index
145 this._sync = sync
146 this._transfer = transfer
147 this._protocol = protocol
148 })()
149 return this._stackPromise
150 }
151
152 // The host's swarm transport must call this synchronously per connection. The
153 // channel MUST be created synchronously — protomux will not pair a channel
154 // opened after the remote's (verified). The stack is built at ready, so
155 // this._protocol is always present by the time connections arrive.
156 attachProtocol (mux) {
157 if (this.closing || this.closed || !this._protocol) return
158 return this._protocol.attach(mux)
159 }
160
161 get peerCount () { return this._protocol ? this._protocol.peerCount : 0 }
162
163 // The overlay's local-only cores (file-index + version marker + sync feed), so the
164 // leftover scan can treat them as wanted and the storage breakdown can size them.
165 localCores () {
166 const cores = this._index ? this._index.cores : []
167 if (this._sync?.feed) cores.push(this._sync.feed)
168 return cores
169 }
170
171 cleanJournals (maxAge) { try { return this._transfer?.cleanJournals(maxAge) || [] } catch { return [] } }
172
173 /**
174 * Make an on-disk file servable by its content hash.
175 * @param {string} overlayPath logical path, e.g. '/attachments/<oid>'
176 * @param {string} diskPath absolute path to the canonical bytes on disk
177 * @param {object} [meta]
178 * meta.contentHash preferred — the oid the caller already computed
179 * meta.size file size (avoids an extra stat if known)
180 * @returns {Promise<{ contentHash, size } | null>} null if the source vanished
181 */
182 async registerFile (overlayPath, diskPath, meta = {}) {
183 await this._ensure()
184 // Source can vanish between prepare and register (a file moved out of a
185 // shared folder mid-scan). Nothing to serve — bail with null instead of throwing
186 // so a routine move doesn't crash the publish path.
187 let stat
188 try {
189 stat = fs.statSync(diskPath)
190 } catch {
191 return null
192 }
193 const size = typeof meta.size === 'number' ? meta.size : stat.size
194 const contentHash = meta.contentHash || await hashFileOnDisk(diskPath, size)
195
196 await this._index.putFile(overlayPath, { contentHash, size, mtime: stat.mtimeMs })
197 this._filePaths.set(overlayPath, diskPath)
198 this._contentHashPaths.set(contentHash, diskPath)
199 // Build (and cache) the chunk map so a content request can stream
200 // immediately without re-chunking under the request. meta.prepare:false
201 // skips this eager full second read of the file — the file is already servable
202 // via the maps above and the serve path (_onContentRequest) chunks lazily on
203 // first request; publishers that already built the chunk map in one pass
204 // (prepareForServe) pass false so registration doesn't re-read the file.
205 if (meta.prepare !== false) await this._transfer.prepareFile(diskPath, overlayPath)
206 // Record in the local change feed (used by the protocol's markSynced
207 // bookkeeping; with autoSync off it is not broadcast).
208 await this._sync.logPut(overlayPath, { contentHash, size, mtime: stat.mtimeMs })
209 return { contentHash, size }
210 }
211
212 // One-pass hash + chunk-map build for the publish path: streams the file
213 // ONCE, persists the content-addressed chunk map (durable in FileIndex), and
214 // returns the content hash. Replaces a hash-only pass plus a lazy fetch-time
215 // re-chunk with a single read, so the first fetcher never waits on chunk indexing.
216 async prepareForServe (diskPath, opts = {}) {
217 await this._ensure()
218 // The overlay path below is a throwaway key — byHashOnly persists ONLY the
219 // content-addressed chunk map, nothing under this path.
220 const prepared = await this._transfer.prepareFile(diskPath, '/mir-prep' + diskPath, { onProgress: opts.onProgress, byHashOnly: true, signal: opts.signal })
221 return prepared ? { contentHash: prepared.contentHash, size: prepared.size } : null
222 }
223
224 async evictContent (contentHash) {
225 await this._ensure()
226 return this._index.evictContent(contentHash)
227 }
228
229 async compactIndex (opts) {
230 await this._ensure()
231 return this._index.compact(opts)
232 }
233
234 /**
235 * Fetch a file by its content hash. Serves from the local disk copy if we
236 * have it; otherwise pulls from a connected peer that does (hash-verified).
237 * @param {string} contentHash
238 * @param {object} [opts] opts.timeout (idle), opts.destPath
239 * @returns {Promise<{ destPath: string, local: boolean, size: number } | null>}
240 */
241 async fetchFile (contentHash, opts = {}) {
242 await this._ensure()
243
244 const localDisk = this._contentHashPaths.get(contentHash)
245 if (localDisk) {
246 const v = await verifyOnDisk(localDisk, contentHash)
247 if (v.ok) return { destPath: localDisk, local: true, size: v.size }
248 }
249
250 // Wait briefly for at least one peer — a connection may be mid-handshake
251 // (lazy attach) or just forming when the fetch is issued. Without this a
252 // fetch issued the instant a peer connects would spuriously 404.
253 const peerWaitMs = opts.peerWaitMs || 3000
254 for (let waited = 0; this._protocol._peers.size === 0 && waited < peerWaitMs; waited += 100) {
255 await new Promise(r => setTimeout(r, 100))
256 }
257 if (this._protocol._peers.size === 0) {
258 // No fetchContent will run, so drop any cancel recorded during the wait — a
259 // stale marker would otherwise cancel the next fetch of the same content.
260 this._protocol.clearCancelPending(contentHash)
261 return null
262 }
263
264 // Multi-source: fetch chunks in parallel from ALL connected peers that
265 // have the file (torrent-style). The scheduler dedups + fails over.
266 const peers = [...this._protocol._peers.values()]
267 // opts.mirrorSpool: store the verified blob CONTENT-ADDRESSED at
268 // <mirrorSpool>/<contentHash> rather than a throwaway destDir name. This is
269 // the mirror path — a node with no working tree of its own fetches files
270 // into a content-addressed spool so it durably re-serves them (via the
271 // <syncBaseDir>/<contentHash> resolution in _onContentRequest).
272 let destPath
273 if (opts.destPath) {
274 destPath = opts.destPath
275 } else if (opts.mirrorSpool) {
276 try { fs.mkdirSync(opts.mirrorSpool, { recursive: true }) } catch {}
277 destPath = path.join(opts.mirrorSpool, contentHash)
278 } else {
279 destPath = path.join(
280 this._destDir,
281 'blob-' + contentHash + '-' + Date.now() + '-' + Math.random().toString(36).slice(2, 8)
282 )
283 }
284 // Whole-file integrity is verified INCREMENTALLY during the transfer (the
285 // scheduler passes contentHash to TransferManager, which hashes the file in
286 // offset order as chunks land and rejects in finalize on a mismatch). So the
287 // assembled file is already verified here — no trailing whole-file re-read.
288 let result
289 try {
290 result = await this._protocol.fetchContent(contentHash, peers, {
291 destPath,
292 timeout: opts.timeout,
293 onProgress: opts.onProgress || (() => {}), // forward bytes/total
294 onVerify: opts.onVerify, // resume re-verify fraction
295 onEnd: opts.onEnd // terminal diagnostic (reason + bytes/chunks)
296 })
297 } catch (err) {
298 // An integrity failure or an explicit cancel/pause is distinct from a
299 // no-holder / stall — surface it so the caller doesn't treat it as a failure.
300 if (err?.code === 'EHASHMISMATCH' || err?.code === 'ECANCELLED') throw err
301 // A local I/O error (full disk / read-only / permission / vanished
302 // mount) is not a no-holder — surface it so the consumer can pause rather than
303 // retry forever.
304 if (err?.code === 'ENOSPC' || err?.code === 'EACCES' || err?.code === 'EROFS' || err?.code === 'EPERM' || err?.code === 'ENOENT') throw err
305 return null
306 }
307 const size = result?.size ?? fs.statSync(destPath).size
308 // Seeding multiplication: register the verified blob locally so this node
309 // now SERVES these chunks to other peers too. A file held by N peers then
310 // gets faster as more peers fetch it (torrent-style). Best-effort — the
311 // fetch still succeeds even if re-registration fails. Skipped if a peer
312 // raced us to register the same hash (registerFile is idempotent-safe).
313 // reSeed:false — opt-out for consumers that should hold nothing.
314 if (opts.reSeed !== false && !this._contentHashPaths.has(contentHash)) {
315 try {
316 await this.registerFile('/attachments/' + contentHash, destPath, { contentHash, size })
317 } catch {}
318 }
319 return { destPath, local: false, size }
320 }
321
322 // Stop an in-flight fetchFile for this content hash. opts.discardPartial
323 // true unlinks the partial (cancel), false keeps it for resume (pause). The
324 // pending fetchFile rejects with ECANCELLED, which the caller treats as not-a-failure.
325 cancelFetch (contentHash, opts = {}) {
326 return this._protocol ? this._protocol.cancelContent(contentHash, opts) : false
327 }
328
329 // Tell holders we stopped pulling this hash when there's no in-flight fetch
330 // to tear down (e.g. discarding an already-paused transfer), so their indicator clears.
331 notifyTransferStopped (contentHash) {
332 return this._protocol ? this._protocol.sendStopControl(contentHash) : false
333 }
334
335 _destPathFor (contentHash) {
336 return path.join(this._destDir, 'blob-' + contentHash)
337 }
338
339 _fetchFromPeer (peer, contentHash, opts = {}) {
340 const syntheticPath = 'content:' + contentHash
341 const destPath = path.join(
342 this._destDir,
343 'blob-' + contentHash + '-' + Date.now() + '-' + Math.random().toString(36).slice(2, 8)
344 )
345 // The protocol's chunk handlers write to whatever _filePaths maps the
346 // synthetic path to. Per-attempt destPath so concurrent peer tries don't
347 // clobber one another.
348 this._filePaths.set(syntheticPath, destPath)
349
350 return new Promise((resolve, reject) => {
351 let settled = false
352 const finish = (err, result) => {
353 if (settled) return
354 settled = true
355 clearTimeout(timer)
356 clearTimeout(pending.idleTimer)
357 this._pendingContent.delete(contentHash)
358 if (err) reject(err); else resolve(result)
359 }
360 const timer = setTimeout(() => {
361 finish(new Error('content request timed out: ' + contentHash.slice(0, 12)))
362 }, opts.timeout || this._perPeerTimeout)
363 const idleFire = () => finish(new Error('peer went silent mid-stream: ' + contentHash.slice(0, 12)))
364 const pending = {
365 syntheticPath, destPath, finish, peer, contentHash,
366 bytesReceived: 0,
367 idleTimer: setTimeout(idleFire, this._idleTimeout),
368 idleFire
369 }
370 this._pendingContent.set(contentHash, pending)
371 this._protocol.requestContent(peer, contentHash, null)
372 })
373 }
374
375 _onChunkProgress (info) {
376 if (!info || !info.path || !info.path.startsWith('content:')) return
377 const contentHash = info.path.slice('content:'.length)
378 const pending = this._pendingContent.get(contentHash)
379 if (!pending) return
380 pending.bytesReceived += (info.bytes || 0)
381 clearTimeout(pending.idleTimer)
382 pending.idleTimer = setTimeout(pending.idleFire, this._idleTimeout)
383 }
384
385 // The V2 chunk flow does NOT verify the reassembled file against the
386 // advertised hash (that's the caller's job in this protocol). We do it here
387 // so fetchFile never returns un-verified bytes.
388 _onSynced (info) {
389 if (!info || info.direction !== 'pull') return
390 if (!info.path || !info.path.startsWith('content:')) return
391 const contentHash = info.path.slice('content:'.length)
392 const pending = this._pendingContent.get(contentHash)
393 if (!pending) return
394 try {
395 const data = fs.readFileSync(pending.destPath)
396 const actual = crypto.data(data).toString('hex')
397 if (actual !== contentHash) {
398 pending.finish(Object.assign(
399 new Error('content-hash mismatch: expected ' + contentHash.slice(0, 12) + '…, got ' + actual.slice(0, 12) + '…'),
400 { code: 'EHASHMISMATCH', expected: contentHash, actual }
401 ))
402 return
403 }
404 pending.finish(null, data)
405 } catch (err) {
406 pending.finish(err)
407 }
408 }
409}
410
411// Stream a file through the chunker's hasher to get its oid without buffering
412// the whole file in memory (matches /api/upload's createStreamingHasher oid).
413async function hashFileOnDisk (diskPath, size) {
414 const hasher = createStreamingHasher({ size })
415 for await (const buf of fs.createReadStream(diskPath)) hasher.update(buf)
416 return hasher.digest()
417}
lib/v3/protocol.js +0 -974
……@@ -1,974 +0,0 @@
1/**
2 * OverlayProtocol — Protomux channel for sync + transfer
3 *
4 * Sync approach: each side sends its local feed entries directly via
5 * file-offer messages. No Hypercore feed replication needed — the feed
6 * entries are small JSON objects sent over our own Protomux channel.
7 *
8 * Flow:
9 * 1. Channel opens → both send sync-state (feed length)
10 * 2. Side with more entries sends file-offers for each change
11 * 3. Receiver auto-requests files it doesn't have
12 * 4. Sender chunks on demand → streams chunks → receiver writes to disk
13 */
14
15import path from 'bare-path'
16import * as messages from './messages.js'
17import { hashChunk, selectTier } from './chunker.js'
18import { ChunkScheduler } from './chunk-scheduler.js'
19import fs from 'bare-fs'
20
21const PROTOCOL = 'hyper-overlay/v2'
22const VERSION = 2
23const CAP_LOCAL_FILES = 0x01
24const CAP_ADAPTIVE_CHUNKS = 0x02
25
26// A file's chunk list, shipped in one frame, can exceed the
27// 16 MiB-1 Noise transport limit (@hyperswarm/secret-stream MAX_ATOMIC_WRITE):
28// a 1.25 TB file at tier 3 is ~1.2M entries ≈ ~44 MB, which the secret-stream
29// rejects ("Message is too large for an atomic write"), dropping the connection
30// before a byte ships. So chunkHashes is paged into frames of ≤ this many
31// entries. Each wire entry is 32 B (hash) + ≤5 B (varint length) ≈ 37 B, so
32// 100k entries ≈ 3.7 MB/frame — well under the limit even after Protomux's
33// ≤8 MiB send-batching.
34const MAX_CHUNKS_PER_MSG = 100000
35
36// Tree entries are variable length (name is a var-string), unlike the fixed-width
37// chunkHashes entries, so treeResponse pages by an encoded-byte budget rather than
38// a fixed count. 4 MiB keeps each frame well under the atomic-write limit even
39// after Protomux's ≤8 MiB send-batching.
40const MAX_TREE_BYTES_PER_MSG = 4 * 1024 * 1024
41
42// Defensive ceiling on the bytes a requester will accumulate across a paged
43// treeResponse. A single-directory tree is orders of magnitude smaller; this only
44// bounds memory so a peer streaming pages endlessly rejects the request instead
45// of OOMing.
46const MAX_TREE_RESPONSE_BYTES = 256 * 1024 * 1024
47
48// transfer-control (message 12) states, downloader→holder.
49const CONTROL_PAUSED = 0
50const CONTROL_STOPPED = 1
51
52// Abandon a serve loop whose stream neither drains nor closes within
53// this window (a wedged-but-TCP-alive peer); the receiver re-requests, so it self-heals.
54const DRAIN_TIMEOUT_MS = 60000
55
56export class OverlayProtocol {
57 constructor (syncEngine, transferManager, opts = {}) {
58 this._syncEngine = syncEngine
59 this._transferManager = transferManager
60 this._corestore = opts.corestore || null
61 this._overlayKey = opts.key || null
62 this._peers = new Map()
63 this._filePaths = opts.filePaths || new Map()
64 this._syncBaseDirs = opts.syncBaseDirs || []
65 this._defaultStrategy = opts.defaultStrategy || 'prompt'
66 this._autoSync = opts.autoSync !== false
67
68 this._conflictCb = opts.onConflict || null
69 this._syncedCb = opts.onSynced || null
70 this._fileOfferCb = opts.onFileOffer || null
71 // Called for every chunk that arrives on this peer. Lets callers
72 // (e.g. NetworkFetcher) implement silent-peer watchdogs by resetting
73 // an idle timer on every chunk. Receives { path, index, bytes, peer }.
74 this._chunkProgressCb = opts.onChunkProgress || null
75 // serve-side telemetry, symmetric to onChunkProgress — fired when WE
76 // serve a peer (the sender-side download indicator). onServeStart({ path, peer,
77 // from, total }) once a peer is authorized to fetch; onChunkServe({ path, peer,
78 // from, bytes }) per served chunk; onServeEnd({ path, peer, from }) on close.
79 this._serveStartCb = opts.onServeStart || null
80 this._chunkServeCb = opts.onChunkServe || null
81 this._serveEndCb = opts.onServeEnd || null
82 // A downloader paused/stopped a hash WE serve. { path, peer, from, state }.
83 this._serveControlCb = opts.onServeControl || null
84 // A downloader reported its on-disk have-bytes for a hash WE serve (resume
85 // baseline). { path, peer, from, have }.
86 this._serveProgressCb = opts.onServeProgress || null
87
88 // 0.5a: content-hash → local disk path, for responding to content-requests
89 this._contentHashPaths = opts.contentHashPaths || new Map()
90 // 0.5a: tree-request timeout (ms)
91 this._treeRequestTimeout = opts.treeRequestTimeout || 30000
92 // Cap on bytes accumulated across a paged treeResponse.
93 this._maxTreeResponseBytes = opts.maxTreeResponseBytes || MAX_TREE_RESPONSE_BYTES
94 // Multi-source: synthetic path 'content:<hash>' → ChunkScheduler. When set,
95 // _onChunkHashes/_onChunkData delegate to the scheduler (parallel fetch
96 // across peers) instead of the legacy single-peer auto-flow.
97 this._schedulers = new Map()
98 // 'content:<hash>' cancelled before its scheduler existed (the caller's
99 // peer-wait window) — fetchContent honors it the moment it creates the scheduler.
100 this._cancelPending = new Set()
101
102 // serve authorization. _serveAuthorizer(peer, from, hash) →
103 // boolean gates every inbound content-request; deny is a silent drop
104 // (indistinguishable from "I don't hold it" — no membership oracle).
105 // _localProfileKey is stamped on outbound content-requests as msg.from.
106 this._serveAuthorizer = opts.serveAuthorizer || null
107 this._localProfileKey = opts.localProfileKey || null
108 }
109
110 /**
111 * Multi-source fetch of a file by content hash from several peers in
112 * parallel (torrent-style). The caller pre-registers _filePaths['content:'
113 * + contentHash] = destPath. Resolves when the file is assembled + verified.
114 *
115 * @param {string} contentHash
116 * @param {Array} peers peer objects to fan the request to
117 * @param {object} [opts] { destPath, cap, timeout, onProgress }
118 * @returns {Promise<void>}
119 */
120 fetchContent (contentHash, peers, opts = {}) {
121 const p = 'content:' + contentHash
122 // Content-addressed dedup: a concurrent fetch of the SAME hash (a
123 // different file/transfer with identical bytes) joins the in-flight one instead
124 // of failing — await its verified bytes, then copy to our destPath. If the leader
125 // was cancelled, re-issue our own fetch (the scheduler entry is already gone).
126 const inflight = this._schedulers.get(p)
127 if (inflight) {
128 return inflight.shared.then(
129 (res) => {
130 if (opts.destPath && opts.destPath !== inflight.destPath) {
131 try { fs.copyFileSync(inflight.destPath, opts.destPath) } catch (err) { throw err }
132 }
133 return res
134 },
135 (err) => { if (err?.code === 'ECANCELLED') return this.fetchContent(contentHash, peers, opts); throw err },
136 )
137 }
138 const sched = new ChunkScheduler({
139 path: p,
140 destPath: opts.destPath,
141 transfer: this._transferManager,
142 sendNeed: (peer, indices) => peer.msgs.chunkNeed.send({ path: p, indices }),
143 cap: opts.cap,
144 timeout: opts.timeout,
145 onProgress: opts.onProgress,
146 onVerify: opts.onVerify,
147 onEnd: opts.onEnd,
148 onBaseline: (have) => this.sendTransferProgress(contentHash, have),
149 contentHash // verify the whole-file hash incrementally during the transfer
150 })
151 this._schedulers.set(p, sched)
152 // shared promise so joiners (above) can await the same completion —
153 // calling sched.promise() twice would overwrite its resolve/reject handlers.
154 sched.shared = sched.promise().finally(() => this._schedulers.delete(p))
155 // honor a cancel/pause that arrived before this scheduler existed.
156 if (this._cancelPending.delete(p)) { sched.cancel(); return sched.shared }
157 for (const peer of peers) this.requestContent(peer, contentHash, null)
158 return sched.shared
159 }
160
161 // Stop an in-flight multi-source fetch. discardPartial:true unlinks the
162 // partial (cancel); false keeps it for resume (pause). The scheduler's promise
163 // rejects with ECANCELLED and its .finally() removes it from _schedulers. When no
164 // scheduler exists yet (the caller is still in its peer-wait window), remember the
165 // intent so fetchContent cancels the scheduler the instant it is created.
166 async cancelContent (contentHash, { discardPartial = false, signal = true } = {}) {
167 const p = 'content:' + contentHash
168 const sched = this._schedulers.get(p)
169 // No scheduler → the fetch never sent a content-request, so no holder has a serve
170 // row for us; skip the (wasted) broadcast and just record the pre-scheduler cancel.
171 if (!sched) { this._cancelPending.add(p); return false }
172 // Tell holders we paused (keep partial) or stopped (discard) BEFORE local teardown,
173 // so their sender-side indicator reacts now, not after the idle sweep. signal:false
174 // suppresses it (a supersede is a restart, not a user stop).
175 if (signal) this._sendTransferControl(contentHash, discardPartial ? CONTROL_STOPPED : CONTROL_PAUSED)
176 // Stop accepting chunks first so the pause hash-drain converges on a stable frontier.
177 sched.cancel()
178 if (discardPartial) this._transferManager.cancel(sched.destPath)
179 else await this._transferManager.pause(sched.destPath)
180 return true
181 }
182
183 // Tell holders we stopped pulling a hash whose local fetch is already gone (e.g.
184 // discarding an already-paused transfer) — no scheduler to tear down here.
185 sendStopControl (contentHash) { this._sendTransferControl(contentHash, CONTROL_STOPPED) }
186
187 // Best-effort broadcast to all connected holders. A holder with no serve-ledger entry
188 // for us (or no slot for this message) just drops it; wrapped per-peer so a closing
189 // channel or an old peer can't throw.
190 _broadcast (msgName, payload) {
191 for (const [, peer] of this._peers) {
192 try { peer.msgs[msgName]?.send(payload) } catch {}
193 }
194 }
195
196 _sendTransferControl (contentHash, state) { this._broadcast('transferControl', { contentHash, state }) }
197
198 // Tell holders our current on-disk have-bytes for a hash we're (re)fetching, so their
199 // sender-side bar mirrors our TRUE progress rather than only the bytes they re-serve.
200 sendTransferProgress (contentHash, have) { this._broadcast('transferProgress', { contentHash, have }) }
201
202 // Drop a cancel recorded before a scheduler existed when the fetch is
203 // abandoned without ever reaching fetchContent (no peer connected) — otherwise the
204 // marker would cancel the next fetch of the same content.
205 clearCancelPending (contentHash) { this._cancelPending.delete('content:' + contentHash) }
206
207 setContentHashPaths (map) { this._contentHashPaths = map }
208
209 attach (mux) {
210 if (this._peers.has(mux)) return this._peers.get(mux)
211
212 const self = this
213
214 const channel = mux.createChannel({
215 protocol: PROTOCOL,
216 id: this._overlayKey || null,
217 onopen () {
218 const peer = self._peers.get(mux)
219 if (peer) self._onOpen(peer)
220 },
221 onclose () {
222 const peer = self._peers.get(mux)
223 self._peers.delete(mux)
224 // Failover: let any active multi-source fetch reassign this peer's
225 // inflight chunks to the remaining peers.
226 if (peer) for (const sched of self._schedulers.values()) sched.removePeer(peer)
227 // Fail in-flight tree requests now, so a mid-stream disconnect rejects
228 // promptly instead of stranding the request (and its buffered pages)
229 // until the idle timeout fires.
230 if (peer) self._failPendingTrees(peer, new Error('channel closed'))
231 // tell the sender-side indicator this peer is gone for every file
232 // it was pulling, so a disconnect mid-download clears its row promptly.
233 if (peer && self._serveEndCb) {
234 for (const [synthPath, from] of peer.authorizedServe) {
235 try { self._serveEndCb({ path: synthPath, peer, from }) } catch {}
236 }
237 }
238 }
239 })
240
241 if (!channel) return null
242
243 const peer = {
244 mux, channel, remoteFeedKey: null, remoteSeq: 0, msgs: {},
245 pendingTrees: new Map(), // hash → { resolve, reject, timer, timeout, nonce, pages, pageBytes }
246 // synthetic serve-paths this peer was authorized for via a
247 // GATED _onContentRequest, mapped to the requester profileKey (msg.from).
248 // _onChunkNeed serves bytes only for paths in here, so a peer cannot pull
249 // bytes by sending chunkNeed/fileRequest directly for a registered path it
250 // was never authorized to fetch; the value flows the requester identity to
251 // the serve telemetry.
252 authorizedServe: new Map()
253 }
254
255 peer.msgs.syncState = channel.addMessage({ encoding: messages.syncState, onmessage: (msg) => self._onSyncState(peer, msg) })
256 peer.msgs.fileOffer = channel.addMessage({ encoding: messages.fileOffer, onmessage: (msg) => self._onFileOffer(peer, msg) })
257 peer.msgs.fileRequest = channel.addMessage({ encoding: messages.fileRequest, onmessage: (msg) => self._onFileRequest(peer, msg) })
258 peer.msgs.chunkHashes = channel.addMessage({ encoding: messages.chunkHashes, onmessage: (msg) => self._onChunkHashes(peer, msg) })
259 peer.msgs.chunkNeed = channel.addMessage({ encoding: messages.chunkNeed, onmessage: (msg) => self._onChunkNeed(peer, msg) })
260 peer.msgs.chunkData = channel.addMessage({ encoding: messages.chunkData, onmessage: (msg) => self._onChunkData(peer, msg) })
261 peer.msgs.chunkCancel = channel.addMessage({ encoding: messages.chunkCancel, onmessage: (msg) => self._onChunkCancel(peer, msg) })
262 peer.msgs.transferComplete = channel.addMessage({ encoding: messages.transferComplete, onmessage: (msg) => self._onTransferComplete(peer, msg) })
263 peer.msgs.conflict = channel.addMessage({ encoding: messages.conflict, onmessage: (msg) => self._onConflictMsg(peer, msg) })
264
265 // 0.5a — tree messages (backward-compatible additions)
266 peer.msgs.treeRequest = channel.addMessage({ encoding: messages.treeRequest, onmessage: (msg) => self._onTreeRequest(peer, msg) })
267 peer.msgs.treeResponse = channel.addMessage({ encoding: messages.treeResponse, onmessage: (msg) => self._onTreeResponse(peer, msg) })
268 peer.msgs.contentRequest = channel.addMessage({ encoding: messages.contentRequest, onmessage: (msg) => self._onContentRequest(peer, msg) })
269 // Appended-last slots: ids 0-11 stay fixed for peers without 12/13, which just
270 // never register these channels and ignore the frames.
271 peer.msgs.transferControl = channel.addMessage({ encoding: messages.transferControl, onmessage: (msg) => self._onTransferControl(peer, msg) })
272 peer.msgs.transferProgress = channel.addMessage({ encoding: messages.transferProgress, onmessage: (msg) => self._onTransferProgress(peer, msg) })
273
274 this._peers.set(mux, peer)
275 channel.open({ version: VERSION, capabilities: CAP_LOCAL_FILES | CAP_ADAPTIVE_CHUNKS })
276 return peer
277 }
278
279 detach (mux) {
280 const peer = this._peers.get(mux)
281 if (!peer) return
282 peer.channel.close()
283 this._peers.delete(mux)
284 }
285
286 setFilePaths (map) { this._filePaths = map }
287
288 broadcastOffer (overlayPath, meta, op) {
289 const msg = {
290 path: overlayPath,
291 contentHash: meta.contentHash || '0'.repeat(64),
292 size: meta.size || 0,
293 mtime: meta.mtime || 0,
294 op
295 }
296 for (const [, peer] of this._peers) {
297 peer.msgs.fileOffer.send(msg)
298 }
299 }
300
301 requestFile (peer, overlayPath, contentHash, chunksHave) {
302 peer.msgs.fileRequest.send({
303 path: overlayPath,
304 contentHash,
305 chunksHave: chunksHave || null
306 })
307 }
308
309 /**
310 * 0.5a: Request a tree by its hash. Returns a promise resolving to
311 * the entries array (or [] if the peer doesn't have it).
312 *
313 * @param {object} peer
314 * @param {string} hash - hex tree hash
315 * @param {object} [opts]
316 * @param {number} [opts.timeout] - override default timeout (ms)
317 * @returns {Promise<Array<{kind, exec, name, childHash, size}>>}
318 */
319 requestTree (peer, hash, opts = {}) {
320 return new Promise((resolve, reject) => {
321 // If a prior request is already in-flight for the same hash, reject
322 // this caller rather than silently overwrite.
323 if (peer.pendingTrees.has(hash)) {
324 return reject(new Error('tree request already pending for hash: ' + hash))
325 }
326 // Per-request nonce (per-peer monotonic, non-zero) lets the requester drop
327 // pages echoed for a superseded attempt. pages/pageBytes buffer this
328 // request's own paged response, so a fresh attempt never inherits an old one.
329 const nonce = peer._treeReqSeq = (peer._treeReqSeq || 0) + 1
330 const timeout = opts.timeout || this._treeRequestTimeout
331 const pending = { resolve, reject, timer: null, timeout, nonce, pages: null, pageBytes: 0 }
332 this._armTreeTimeout(peer, hash, pending)
333 peer.pendingTrees.set(hash, pending)
334 peer.msgs.treeRequest.send({ hash, nonce })
335 })
336 }
337
338 // Arm (or re-arm) a pending tree request's idle timeout. Every buffered page
339 // re-arms it (a large tree on a slow link mustn't time out mid-stream); the
340 // request still settles because the accumulator is byte-capped in _onTreeResponse.
341 _armTreeTimeout (peer, hash, pending) {
342 pending.timer = setTimeout(() => {
343 if (peer.pendingTrees.get(hash) === pending) {
344 peer.pendingTrees.delete(hash)
345 pending.reject(new Error('tree request timed out: ' + hash))
346 }
347 }, pending.timeout)
348 }
349
350 // Reject every in-flight tree request on this peer (channel close / destroy),
351 // clearing timers so the promise fails promptly and its buffered pages release.
352 _failPendingTrees (peer, err) {
353 for (const [hash, pending] of peer.pendingTrees) {
354 clearTimeout(pending.timer)
355 peer.pendingTrees.delete(hash)
356 pending.reject(err)
357 }
358 }
359
360 /**
361 * 0.5a: Request a file by its content hash. Peer looks up any file
362 * matching that hash locally and streams chunks via the existing flow
363 * (chunkHashes → chunkNeed → chunkData). Returns immediately; chunks
364 * arrive asynchronously.
365 *
366 * Note: receiver must pre-register a target path mapping via
367 * _filePaths keyed by the contentHash before calling, OR this method
368 * is a no-op on the receive side. Full wiring deferred to 0.5c.
369 */
370 requestContent (peer, contentHash, chunksHave) {
371 peer.msgs.contentRequest.send({
372 contentHash,
373 chunksHave: chunksHave || null,
374 from: this._localProfileKey || '' // authenticate the asker to the serve gate
375 })
376 }
377
378 get peerCount () { return this._peers.size }
379
380 destroy () {
381 for (const [, peer] of this._peers) {
382 this._failPendingTrees(peer, new Error('protocol destroyed'))
383 peer.channel.close()
384 }
385 this._peers.clear()
386 }
387
388 // ── Handlers ────────────────────────────────────────────────
389
390 _onOpen (peer) {
391 peer.msgs.syncState.send({
392 feedKey: this._syncEngine.feedKey,
393 localSeq: this._syncEngine.feed.length,
394 remoteSeq: 0
395 })
396 }
397
398 async _onSyncState (peer, msg) {
399 peer.remoteFeedKey = msg.feedKey
400 peer.remoteSeq = msg.localSeq
401
402 if (!this._autoSync) return
403
404 try {
405 await this._offerLocalFiles(peer)
406 } catch {
407 // ignore — the HyperOverlay facade runs with autoSync off, so this path is dormant
408 }
409 }
410
411 async _onFileOffer (peer, msg) {
412 // In gated mode the overlay flow is contentRequest→scheduler ONLY; gated
413 // consumers never act on path-based file-offers (autoSync is off). Ignoring
414 // them closes the offer→requestFile→legacy-receive path a peer could drive.
415 if (this._serveAuthorizer) return
416 if (this._fileOfferCb) this._fileOfferCb(peer, msg)
417
418 if (msg.op === 0) {
419 const local = await this._syncEngine._fileIndex.getFile(msg.path)
420 if (!local || local.contentHash !== msg.contentHash) {
421 const diskPath = this._resolveFilePath(msg.path)
422 if (diskPath) {
423 this._filePaths.set(msg.path, diskPath)
424 this.requestFile(peer, msg.path, msg.contentHash)
425 }
426 }
427 }
428 }
429
430 async _onFileRequest (peer, msg) {
431 // path-based serve is UNGATED and would bypass the serve gate (the only
432 // gated entry point is _onContentRequest). Gated consumers never send
433 // fileRequest, so refuse it outright in gated mode.
434 if (this._serveAuthorizer) return
435 const diskPath = this._filePaths.get(msg.path)
436 if (!diskPath) return
437
438 const prepared = await this._transferManager.prepareFile(diskPath, msg.path)
439 if (!prepared) return
440
441 if (!await this._transferManager._fileIndex.hasChunkMap(msg.path)) {
442 await this._transferManager._fileIndex.putChunkMap(msg.path, prepared.chunks)
443 }
444
445 this._sendChunkHashes(peer, msg.path, prepared.tier, prepared.chunks)
446 }
447
448 // Send a file's chunk list, paged so no single frame exceeds
449 // the Noise transport's 16 MiB-1 atomic-write limit. A list that fits in one
450 // frame ships as one (more:0) — identical on the wire to the pre-paging format
451 // bar a trailing 0 byte. _onChunkHashes reassembles the pages on the receiver.
452 _sendChunkHashes (peer, filePath, tier, chunks) {
453 const total = chunks.length
454 if (total <= MAX_CHUNKS_PER_MSG) {
455 peer.msgs.chunkHashes.send({ path: filePath, tier, chunks, more: 0 })
456 return
457 }
458 for (let i = 0; i < total; i += MAX_CHUNKS_PER_MSG) {
459 const more = i + MAX_CHUNKS_PER_MSG < total ? 1 : 0
460 peer.msgs.chunkHashes.send({ path: filePath, tier, chunks: chunks.slice(i, i + MAX_CHUNKS_PER_MSG), more })
461 }
462 }
463
464 // Reassemble a paged chunk list. Returns the full list once
465 // the final page (more:0) arrives, or null while pages are still pending. A
466 // lone complete frame passes straight through (no copy) — the small-file norm.
467 // Pages for a path arrive in order on this channel, so arrival-order
468 // concatenation is correct even when several files interleave on it.
469 _reassembleChunkHashes (peer, msg) {
470 const buffered = peer._chunkHashPages && peer._chunkHashPages.get(msg.path)
471 if (!buffered && !msg.more) return msg.chunks
472 if (!peer._chunkHashPages) peer._chunkHashPages = new Map()
473 let acc = buffered
474 if (!acc) { acc = []; peer._chunkHashPages.set(msg.path, acc) }
475 for (const ch of msg.chunks) acc.push(ch)
476 if (msg.more) return null
477 peer._chunkHashPages.delete(msg.path)
478 return acc
479 }
480
481 // Upper-bound wire size of one tree entry: kind(1)+exec(1) + string(len-prefix ≤4
482 // + utf8 name bytes) + childHash(32) + size varint(≤9, c.uint is 9 bytes for a
483 // >4 GiB size).
484 _treeEntryWireSize (e) {
485 return 1 + 1 + 4 + Buffer.byteLength(e.name || '', 'utf8') + 32 + 9
486 }
487
488 // Send a tree's entries, paged so no single frame exceeds the Noise atomic-write
489 // limit; every page but the last is more:1. A tree that fits in one frame ships
490 // as a single more:0 frame. `nonce` echoes the requester's per-request id.
491 _sendTreeResponse (peer, hash, entries, nonce) {
492 let page = []
493 let bytes = 40 // hash(32) + uint32 count prefix + more/nonce + slack
494 for (const e of entries) {
495 const sz = this._treeEntryWireSize(e)
496 if (page.length && bytes + sz > MAX_TREE_BYTES_PER_MSG) {
497 peer.msgs.treeResponse.send({ hash, entries: page, more: 1, nonce })
498 page = []
499 bytes = 40
500 }
501 page.push(e)
502 bytes += sz
503 }
504 peer.msgs.treeResponse.send({ hash, entries: page, more: 0, nonce })
505 }
506
507 async _onChunkHashes (peer, msg) {
508 // Reassemble paged chunkHashes frames before dispatching.
509 const chunks = this._reassembleChunkHashes(peer, msg)
510 if (chunks === null) {
511 // A buffered page is forward progress — keep the scheduler's no-progress
512 // watchdog from tripping while a large map streams across several frames.
513 const pending = this._schedulers.get(msg.path)
514 if (pending && pending.notePageProgress) pending.notePageProgress()
515 return
516 }
517 // Multi-source: if a scheduler owns this path, let it distribute chunk
518 // requests across peers instead of the legacy single-peer auto-flow.
519 const sched = this._schedulers.get(msg.path)
520 if (sched) { Promise.resolve(sched.onChunkHashes(peer, chunks)).catch(() => {}); return }
521 // the legacy single-peer receive below writes incoming bytes
522 // to _filePaths.get(path) and finalizes a rename OVER it. For a path that
523 // maps to one of OUR own files (e.g. a content-register entry), an
524 // unsolicited chunkHashes would let a peer overwrite the owner's source
525 // file. Gated consumers only ever receive via a scheduler (the branch
526 // above), so refuse any other receive.
527 if (this._serveAuthorizer) return
528
529 const diskPath = this._filePaths.get(msg.path)
530 if (!diskPath) return
531
532 // Empty-file fast path: a 0-byte file has no chunks to fetch, but we
533 // still need to materialize it on disk and fire the same completion
534 // flow as _onChunkData's "transfer complete" branch — otherwise the
535 // requester (e.g. NetworkFetcher.getContent) never resolves and times
536 // out. Empty files are common in practice: .keep, __init__.py, build
537 // stamps, touch'd markers. Bug symptom before this: node-gyp's
538 // postinstall.stamp (0 bytes, hash 5187b7a8…) hung every install.
539 if (chunks.length === 0) {
540 try {
541 fs.mkdirSync(path.dirname(diskPath), { recursive: true })
542 fs.writeFileSync(diskPath, Buffer.alloc(0))
543 } catch {
544 return
545 }
546 const contentHash = hashChunk(Buffer.alloc(0))
547 const stat = fs.statSync(diskPath)
548 await this._syncEngine._fileIndex.putFile(msg.path, {
549 contentHash, size: 0, mtime: stat.mtimeMs
550 })
551 await this._syncEngine.logPut(msg.path, {
552 contentHash, size: 0, mtime: stat.mtimeMs
553 })
554 if (peer.remoteFeedKey) {
555 await this._syncEngine.markSynced(peer.remoteFeedKey, msg.path, contentHash, this._syncEngine.feed.length - 1)
556 }
557 peer.msgs.transferComplete.send({ path: msg.path, contentHash })
558 if (this._syncedCb) {
559 this._syncedCb({ path: msg.path, contentHash, size: 0, direction: 'pull' })
560 }
561 return
562 }
563
564 const needed = this._transferManager.computeNeeded(chunks, new Set())
565 if (needed.length === 0) return
566
567 let size = 0
568 for (const c of chunks) size += c.length
569
570 // startReceive can throw (fd open, partial recovery) — drop this transfer
571 // instead of rejecting protomux's onmessage promise, which would destroy the
572 // whole Noise stream.
573 try {
574 await this._transferManager.startReceive(diskPath, {
575 size,
576 chunks: chunks.map((c, i) => {
577 let offset = 0
578 for (let j = 0; j < i; j++) offset += chunks[j].length
579 return { hash: c.hash, offset, length: c.length }
580 })
581 })
582 } catch {
583 return
584 }
585
586 peer.msgs.chunkNeed.send({ path: msg.path, indices: needed })
587 }
588
589 async _onChunkNeed (peer, msg) {
590 // Serve bytes ONLY for a synthetic path THIS peer was
591 // authorized for via a gated _onContentRequest. Blocks a peer pulling bytes
592 // by sending chunkNeed directly for a registered path (e.g. a content-
593 // register entry) without passing the serve gate.
594 if (this._serveAuthorizer && !peer.authorizedServe.has(msg.path)) return
595 const diskPath = this._filePaths.get(msg.path)
596 if (!diskPath) return
597
598 const fileIndex = this._transferManager._fileIndex
599 const chunkMap = msg.path.startsWith('content:')
600 ? await fileIndex.getChunkMapByHash(msg.path.slice('content:'.length))
601 : await fileIndex.getChunkMap(msg.path)
602 if (!chunkMap) return
603
604 // Single-flight serve pump per peer (mirrors hypercore's `receiverBusy`).
605 // Enqueue this request; if no pump is running for this peer, drain the queue
606 // one job at a time. protomux dispatches every frame of a socket read
607 // synchronously, so two chunkNeed frames from the same peer would otherwise
608 // start rival serve loops — interleaving sends and admitting N chunks per
609 // drain instead of honouring backpressure once (the discipline our 0.2.13
610 // serialized pump had, which Ok's inline loop dropped).
611 if (!peer._serveQueue) peer._serveQueue = []
612 peer._serveQueue.push({ path: msg.path, diskPath, chunkMap, indices: msg.indices })
613 if (peer._serving) return
614 peer._serving = true
615 try {
616 while (peer._serveQueue.length) {
617 if (peer.channel && peer.channel.closed) { peer._serveQueue.length = 0; break }
618 try { await this._serveJob(peer, peer._serveQueue.shift()) } catch {}
619 }
620 } finally {
621 peer._serving = false
622 }
623 }
624
625 // Serve one chunkNeed job: stream the requested chunk indices, honouring
626 // stream backpressure — stop producing when send() reports the write buffer is
627 // full and resume on drain, so sibling protocols multiplexed on the same Noise
628 // stream (Corestore replication, app handshakes; protomux MAX_BACKLOG is
629 // Infinity) aren't starved. Runs under the per-peer single-flight guard in
630 // _onChunkNeed, so exactly one job streams at a time per peer.
631 async _serveJob (peer, job) {
632 const { path, diskPath, chunkMap, indices } = job
633 for (let i = 0; i < indices.length; i++) {
634 const index = indices[i]
635 if (index >= chunkMap.length) continue
636 const c = chunkMap[index]
637 const data = this._transferManager.readChunk(diskPath, c.offset, c.length)
638 if (!data) continue
639 const flushed = peer.msgs.chunkData.send({ path, index, data })
640 if (this._chunkServeCb) {
641 try { this._chunkServeCb({ path, index, bytes: data.length, peer, from: peer.authorizedServe.get(path) || null }) } catch {}
642 }
643 if (flushed === false && i < indices.length - 1) {
644 const alive = await this._waitForDrain(peer)
645 if (!alive) return
646 if (this._serveAuthorizer && !peer.authorizedServe.has(path)) return
647 }
648 }
649 }
650
651 // One shared drain waiter per peer so concurrent serve loops (the
652 // same peer pulling several files) don't each attach listeners and trip the stream's
653 // max-listeners warning. Resolves true on drain, false if the channel closes first.
654 _waitForDrain (peer) {
655 const channel = peer.channel
656 const stream = peer.mux && peer.mux.stream
657 if (!stream || !channel || channel.closed) return Promise.resolve(false)
658 if (channel.drained) return Promise.resolve(true)
659 if (peer._drainWait) return peer._drainWait
660 peer._drainWait = new Promise((resolve) => {
661 let timer = null
662 const done = (alive) => {
663 if (timer) clearTimeout(timer)
664 stream.removeListener('drain', onDrain)
665 stream.removeListener('close', onClose)
666 peer._drainWait = null
667 resolve(alive)
668 }
669 const onDrain = () => done(!channel.closed)
670 const onClose = () => done(false)
671 timer = setTimeout(() => done(false), DRAIN_TIMEOUT_MS)
672 stream.on('drain', onDrain)
673 stream.on('close', onClose)
674 })
675 return peer._drainWait
676 }
677
678 async _onChunkData (peer, msg) {
679 // Silent-peer watchdog hook: fire BEFORE any processing so a slow disk
680 // doesn't get blamed for a peer-side stall.
681 if (this._chunkProgressCb) {
682 try {
683 this._chunkProgressCb({
684 path: msg.path,
685 index: msg.index,
686 bytes: msg.data ? msg.data.length : 0,
687 peer
688 })
689 } catch {}
690 }
691
692 // Multi-source: scheduler owns the write + reschedule + finalize.
693 const sched = this._schedulers.get(msg.path)
694 if (sched) { Promise.resolve(sched.onChunkData(peer, msg.index, msg.data)).catch(() => {}); return }
695 // refuse the legacy single-peer receive (writes+renames over
696 // _filePaths.get(path), i.e. potentially our own source file). Gated
697 // consumers only receive via a scheduler (above).
698 if (this._serveAuthorizer) return
699
700 const diskPath = this._filePaths.get(msg.path)
701 if (!diskPath) return
702
703 const result = this._transferManager.writeChunk(diskPath, msg.index, msg.data)
704
705 if (result.ok && this._transferManager.isComplete(diskPath)) {
706 const fin = await this._transferManager.finalize(diskPath)
707 if (fin.ok) {
708 const fileData = fs.readFileSync(diskPath)
709 const contentHash = hashChunk(fileData)
710 const stat = fs.statSync(diskPath)
711
712 await this._syncEngine._fileIndex.putFile(msg.path, {
713 contentHash, size: stat.size, mtime: stat.mtimeMs
714 })
715 await this._syncEngine.logPut(msg.path, {
716 contentHash, size: stat.size, mtime: stat.mtimeMs
717 })
718
719 if (peer.remoteFeedKey) {
720 await this._syncEngine.markSynced(peer.remoteFeedKey, msg.path, contentHash, this._syncEngine.feed.length - 1)
721 }
722
723 peer.msgs.transferComplete.send({ path: msg.path, contentHash })
724 if (this._syncedCb) {
725 this._syncedCb({ path: msg.path, contentHash, size: stat.size, direction: 'pull' })
726 }
727 }
728 }
729 }
730
731 _onChunkCancel (peer, msg) {
732 const diskPath = this._filePaths.get(msg.path)
733 if (diskPath) this._transferManager.cancel(diskPath)
734 }
735
736 // Resolve a serve-message's downloader identity from the authenticated record we kept at
737 // serve start (peer.authorizedServe), NOT the message — so a peer can only affect its own
738 // ledger row, never spoof another. Returns null `from` when the peer was never authorized.
739 _authorizedFrom (peer, contentHash) {
740 const synthPath = 'content:' + contentHash
741 return { synthPath, from: peer.authorizedServe.get(synthPath) || null }
742 }
743
744 // A downloader told us it paused/stopped pulling a hash WE serve.
745 _onTransferControl (peer, msg) {
746 if (!this._serveControlCb) return
747 const { synthPath, from } = this._authorizedFrom(peer, msg.contentHash)
748 if (!from) return
749 const state = msg.state === CONTROL_STOPPED ? 'stopped' : 'paused'
750 try { this._serveControlCb({ path: synthPath, peer, from, state }) } catch {}
751 }
752
753 // A downloader reported its on-disk have-bytes for a hash WE serve.
754 _onTransferProgress (peer, msg) {
755 if (!this._serveProgressCb) return
756 const { synthPath, from } = this._authorizedFrom(peer, msg.contentHash)
757 if (!from) return
758 try { this._serveProgressCb({ path: synthPath, peer, from, have: msg.have || 0 }) } catch {}
759 }
760
761 async _onTransferComplete (peer, msg) {
762 if (peer.remoteFeedKey) {
763 await this._syncEngine.markSynced(peer.remoteFeedKey, msg.path, msg.contentHash, this._syncEngine.feed.length - 1)
764 }
765 if (this._syncedCb) {
766 this._syncedCb({ path: msg.path, contentHash: msg.contentHash, direction: 'push' })
767 }
768 }
769
770 _onConflictMsg (peer, msg) {
771 if (this._conflictCb) this._conflictCb(msg, peer)
772 }
773
774 // ── Tree handlers (0.5a) ────────────────────────────────────
775
776 async _onTreeRequest (peer, msg) {
777 // ungated metadata oracle (returns fileIndex tree entries to
778 // any peer). Gated consumers never use tree-requests; refuse in gated mode.
779 if (this._serveAuthorizer) return
780 let entries = []
781 try {
782 const fileIndex = this._syncEngine && this._syncEngine._fileIndex
783 if (fileIndex && typeof fileIndex.getTree === 'function') {
784 const tree = await fileIndex.getTree(msg.hash)
785 if (tree && Array.isArray(tree.entries)) entries = tree.entries
786 }
787 } catch {
788 // fall through — reply with empty entries on lookup failure
789 }
790 this._sendTreeResponse(peer, msg.hash, entries, msg.nonce || 0)
791 }
792
793 _onTreeResponse (peer, msg) {
794 const pending = peer.pendingTrees.get(msg.hash)
795 if (!pending) return // no request in flight — drop
796 // Drop a page echoing a superseded request's nonce (a stale page from a
797 // timed-out attempt must not settle a later same-hash retry). nonce 0 = a
798 // pre-nonce holder that can't echo it, so fall through (unverifiable,
799 // best-effort).
800 if (msg.nonce && msg.nonce !== pending.nonce) return
801 const settle = (fn, arg) => { clearTimeout(pending.timer); peer.pendingTrees.delete(msg.hash); fn(arg) }
802 // Lone complete frame (the small-tree norm) resolves with no accumulation.
803 if (!pending.pages && !msg.more) return settle(pending.resolve, msg.entries)
804 // Accumulate into THIS request's own buffer, bounding total bytes so a peer
805 // streaming pages endlessly rejects the request instead of growing memory
806 // without limit.
807 if (!pending.pages) pending.pages = []
808 for (const e of msg.entries) { pending.pages.push(e); pending.pageBytes += this._treeEntryWireSize(e) }
809 if (pending.pageBytes > this._maxTreeResponseBytes) {
810 return settle(pending.reject, new Error('tree response too large: ' + msg.hash))
811 }
812 if (msg.more) {
813 // Forward progress — re-arm the idle timeout so a legitimately large
814 // multi-page tree on a slow link doesn't time out mid-stream.
815 clearTimeout(pending.timer)
816 this._armTreeTimeout(peer, msg.hash, pending)
817 return
818 }
819 settle(pending.resolve, pending.pages)
820 }
821
822 async _onContentRequest (peer, msg) {
823 // serve gate. BEFORE any path resolution or streaming,
824 // ask the injected authorizer whether this peer may receive this hash.
825 // Deny == silent return (same observable behavior as "I don't hold it"
826 // below), so a non-member cannot distinguish denial from absence.
827 if (this._serveAuthorizer) {
828 let ok = false
829 try { ok = await this._serveAuthorizer(peer, msg.from || null, msg.contentHash) } catch { ok = false }
830 if (!ok) return
831 }
832
833 // Resolve msg.contentHash → an on-disk path. Three lookup paths, in order:
834 // 1. Fast path: _contentHashPaths map (warm — just-registered / consumer-set)
835 // 2. Content-addressed spool (RESTART-DURABLE): <syncBaseDir>/<contentHash>
836 // 3. Slow path: scan fileIndex for any file:* entry with that hash
837 let diskPath = this._contentHashPaths.get(msg.contentHash)
838 if (diskPath && !fileExists(diskPath)) diskPath = null
839
840 if (!diskPath) {
841 // Content-addressed spool fallback. The canonical bytes are stored at
842 // <syncBaseDir>/<contentHash> (fetchFile's mirrorSpool option writes them
843 // there). The in-memory map is empty after a restart and FileIndex
844 // persists no disk path, so resolve the hash straight from the
845 // content-addressed spool — a pure function of the hash + dir, like a
846 // git object store's blobs/<hash>. This is what lets a node keep serving
847 // its spooled files across restarts with no rehydration step.
848 for (const baseDir of this._syncBaseDirs) {
849 const candidate = path.join(baseDir, msg.contentHash)
850 if (fileExists(candidate)) {
851 diskPath = candidate
852 this._contentHashPaths.set(msg.contentHash, candidate) // warm the map
853 break
854 }
855 }
856 }
857
858 if (!diskPath) {
859 // Scan fileIndex for a file whose contentHash matches.
860 const fileIndex = this._syncEngine && this._syncEngine._fileIndex
861 if (fileIndex && typeof fileIndex.listFiles === 'function') {
862 try {
863 const files = await fileIndex.listFiles()
864 for (const f of files) {
865 if (f.contentHash === msg.contentHash) {
866 diskPath = this._filePaths.get(f.path)
867 if ((!diskPath || !fileExists(diskPath)) && f.path) diskPath = f.path
868 break
869 }
870 }
871 } catch {
872 // fall through
873 }
874 }
875 }
876
877 if (!diskPath) {
878 // Don't have it. Silently drop; requester will time out.
879 return
880 }
881
882 // Use a SYNTHETIC path keyed by contentHash so the requester can
883 // pre-register a destPath and the chunk flow routes correctly on
884 // both sides. Temporarily populate _filePaths so _onChunkNeed +
885 // chunkmap lookups resolve under this synthetic key.
886 const syntheticPath = 'content:' + msg.contentHash
887 this._filePaths.set(syntheticPath, diskPath)
888 peer.authorizedServe.set(syntheticPath, msg.from || null) // _onChunkNeed will now serve this path to THIS peer
889
890 // The chunk map is content-addressed — identical bytes always chunk
891 // identically. Reuse one computed at publish (or a prior serve, or before a
892 // restart) instead of re-reading the whole file to re-chunk it: re-chunking a
893 // multi-GB file adds a full-file-read of latency before the first byte ships.
894 const fileIndex = this._transferManager._fileIndex
895 let chunks = await fileIndex.getChunkMapByHash(msg.contentHash)
896 let tier
897 if (chunks) {
898 let size = 0
899 try { size = fs.statSync(diskPath).size } catch {}
900 tier = selectTier(size)
901 } else {
902 const prepared = await this._transferManager.prepareFile(diskPath, syntheticPath, { byHashOnly: true })
903 if (!prepared) {
904 this._filePaths.delete(syntheticPath)
905 peer.authorizedServe.delete(syntheticPath) // keep the authorize map in lockstep with _filePaths
906 return
907 }
908 chunks = prepared.chunks
909 tier = prepared.tier
910 // prepareFile persists the hash-keyed map only for large files; ensure it exists
911 // for small ones too, since _onChunkNeed resolves content:<hash> by that map.
912 if (!await fileIndex.hasChunkMapByHash(msg.contentHash)) await fileIndex.putChunkMapByHash(msg.contentHash, chunks)
913 }
914 this._sendChunkHashes(peer, syntheticPath, tier, chunks)
915 if (this._serveStartCb) {
916 let total = 0
917 for (const c of chunks) total += c.length || 0
918 try { this._serveStartCb({ path: syntheticPath, peer, from: msg.from || null, total }) } catch {}
919 }
920 }
921
922 // ── Sync logic ──────────────────────────────────────────────
923
924 /**
925 * Offer all local files to a peer.
926 * Reads from the local feed and sends file-offers for each entry.
927 * The peer will request any files it doesn't have.
928 */
929 async _offerLocalFiles (peer) {
930 const feed = this._syncEngine.feed
931 if (feed.length === 0) return
932
933 for (let i = 0; i < feed.length; i++) {
934 try {
935 const entry = await feed.get(i)
936 if (!entry) continue
937
938 // Register the file path mapping so _onFileRequest can find it
939 if (entry.op === 0) {
940 const diskPath = this._resolveFilePath(entry.path)
941 if (diskPath) this._filePaths.set(entry.path, diskPath)
942 }
943
944 peer.msgs.fileOffer.send({
945 path: entry.path,
946 contentHash: entry.hash || '0'.repeat(64),
947 size: entry.size || 0,
948 mtime: entry.mtime || 0,
949 op: entry.op
950 })
951 } catch {
952 // skip unreadable entries
953 }
954 }
955 }
956
957 /**
958 * Resolve an overlay path to a disk path.
959 */
960 _resolveFilePath (overlayPath) {
961 if (this._filePaths.has(overlayPath)) return this._filePaths.get(overlayPath)
962 for (const baseDir of this._syncBaseDirs) {
963 return path.join(baseDir, overlayPath)
964 }
965 return null
966 }
967}
968
969// True iff `p` is an existing regular file. Used to validate a resolved disk path
970// before serving (a cold-restart map entry or an overlay-path masquerading as a
971// disk path won't exist) and to probe the content-addressed spool.
972function fileExists (p) {
973 try { return fs.statSync(p).isFile() } catch { return false }
974}
lib/v3/sync-engine.js +0 -364
……@@ -1,364 +0,0 @@
1/**
2 * SyncEngine — Feed-based change tracking and conflict detection
3 *
4 * Each peer has a Hypercore feed logging file change operations.
5 * When peers connect, feeds replicate automatically (Hypercore handles this).
6 * The sync engine diffs feeds since the last-synced sequence number
7 * and detects conflicts using three-value comparison:
8 *
9 * my current hash vs their current hash vs last-synced hash (common ancestor)
10 *
11 * Conflict resolution is configurable per sync folder:
12 * prompt | newest-wins | keep-both | skip | mine-wins | theirs-wins
13 */
14
15import ReadyResource from 'ready-resource'
16
17// Operation types written to the change feed
18const OP_PUT = 0
19const OP_DEL = 1
20
21export const STRATEGIES = ['prompt', 'newest-wins', 'keep-both', 'skip', 'mine-wins', 'theirs-wins']
22
23export class SyncEngine extends ReadyResource {
24 constructor (fileIndex, corestore, opts = {}) {
25 super()
26 this._fileIndex = fileIndex
27 this._corestore = corestore
28 this._feed = null
29 this._peerFeeds = new Map() // peerKey hex → Hypercore
30 this._conflicts = [] // unresolved conflicts
31 this._opts = opts
32 }
33
34 async _open () {
35 // Local change feed — append-only log of file operations. Encrypted at rest when
36 // the caller passes opts.encryptionKey (peer feeds, opened by key in addPeerFeed,
37 // stay plaintext — their key is not ours to encrypt under).
38 const opts = { name: 'sync-feed', valueEncoding: 'json' }
39 if (this._opts.encryptionKey) opts.encryptionKey = this._opts.encryptionKey
40 this._feed = this._corestore.get(opts)
41 await this._feed.ready()
42 }
43
44 async _close () {
45 // Peer feeds are managed by corestore, no explicit close needed
46 }
47
48 /**
49 * The local change feed (Hypercore)
50 */
51 get feed () { return this._feed }
52
53 /**
54 * The local feed's public key as hex
55 */
56 get feedKey () { return this._feed.key.toString('hex') }
57
58 /**
59 * Current unresolved conflicts
60 */
61 get conflicts () { return this._conflicts }
62
63 // ── Change logging ──────────────────────────────────────────
64
65 /**
66 * Log a file put/update operation
67 * @param {string} path - file path
68 * @param {{ contentHash: string, size: number, mtime: number }} meta
69 */
70 async logPut (path, meta) {
71 await this._feed.append({
72 op: OP_PUT,
73 path,
74 hash: meta.contentHash,
75 size: meta.size,
76 mtime: meta.mtime,
77 timestamp: Date.now()
78 })
79 }
80
81 /**
82 * Log a file deletion
83 * @param {string} path - file path
84 */
85 async logDel (path) {
86 await this._feed.append({
87 op: OP_DEL,
88 path,
89 hash: null,
90 size: 0,
91 mtime: 0,
92 timestamp: Date.now()
93 })
94 }
95
96 // ── Peer feeds ──────────────────────────────────────────────
97
98 /**
99 * Add a remote peer's feed for syncing
100 * @param {Buffer|string} key - peer's feed public key
101 * @returns {Hypercore} the peer feed
102 */
103 addPeerFeed (key) {
104 const keyHex = typeof key === 'string' ? key : key.toString('hex')
105 if (this._peerFeeds.has(keyHex)) return this._peerFeeds.get(keyHex)
106
107 const feed = this._corestore.get({ key: typeof key === 'string' ? Buffer.from(key, 'hex') : key, valueEncoding: 'json' })
108 this._peerFeeds.set(keyHex, feed)
109 return feed
110 }
111
112 /**
113 * Get a peer feed by key
114 * @param {string} keyHex
115 * @returns {Hypercore|null}
116 */
117 getPeerFeed (keyHex) {
118 return this._peerFeeds.get(keyHex) || null
119 }
120
121 // ── Feed diffing ────────────────────────────────────────────
122
123 /**
124 * Get new operations from a feed since a given sequence number
125 * @param {Hypercore} feed
126 * @param {number} sinceSeq - start reading from this seq (exclusive)
127 * @returns {Array<{ seq: number, op: number, path: string, hash: string|null, size: number, mtime: number }>}
128 */
129 async getChanges (feed, sinceSeq) {
130 await feed.ready()
131 await feed.update()
132
133 const changes = []
134 const start = sinceSeq + 1
135 const end = feed.length
136
137 for (let i = start; i < end; i++) {
138 const entry = await feed.get(i)
139 changes.push({ seq: i, ...entry })
140 }
141
142 return changes
143 }
144
145 /**
146 * Build a map of latest state per path from a list of changes.
147 * Later entries override earlier ones for the same path.
148 * @param {Array} changes
149 * @returns {Map<string, { seq, op, hash, size, mtime }>}
150 */
151 buildChangeMap (changes) {
152 const map = new Map()
153 for (const c of changes) {
154 map.set(c.path, { seq: c.seq, op: c.op, hash: c.hash, size: c.size, mtime: c.mtime })
155 }
156 return map
157 }
158
159 // ── Sync diff ───────────────────────────────────────────────
160
161 /**
162 * Compute the sync diff between local and a remote peer.
163 * Returns actions to take: pull, push, conflict, or skip.
164 *
165 * @param {string} peerKey - hex key of the remote peer's feed
166 * @returns {Array<{ path, action, local?, remote?, ancestor? }>}
167 * action: 'pull' | 'push' | 'conflict' | 'skip' | 'delete-local' | 'delete-remote'
168 */
169 async diff (peerKey) {
170 const peerFeed = this._peerFeeds.get(peerKey)
171 if (!peerFeed) throw new Error('Unknown peer: ' + peerKey)
172
173 await peerFeed.ready()
174 await peerFeed.update()
175
176 // Get last-synced state for all files with this peer
177 const syncStates = await this._fileIndex.listSyncStates(peerKey)
178 const syncMap = new Map()
179 for (const s of syncStates) {
180 syncMap.set(s.path, s)
181 }
182
183 // Get my changes since last sync
184 const myLastSeq = this._getMaxSyncedSeq(syncStates, 'local')
185 const myChanges = await this.getChanges(this._feed, -1) // read all, filter by syncMap
186 const myMap = this.buildChangeMap(myChanges)
187
188 // Get their changes since last sync
189 const theirChanges = await this.getChanges(peerFeed, -1) // read all
190 const theirMap = this.buildChangeMap(theirChanges)
191
192 // Collect all paths mentioned by either side
193 const allPaths = new Set([...myMap.keys(), ...theirMap.keys()])
194
195 const actions = []
196
197 for (const path of allPaths) {
198 const mine = myMap.get(path)
199 const theirs = theirMap.get(path)
200 const ancestor = syncMap.get(path)
201
202 const myHash = mine ? mine.hash : null
203 const theirHash = theirs ? theirs.hash : null
204 const ancestorHash = ancestor ? ancestor.lastHash : null
205
206 // Both have it and hashes match — in sync
207 if (myHash && theirHash && myHash === theirHash) {
208 actions.push({ path, action: 'skip', local: mine, remote: theirs })
209 continue
210 }
211
212 // Only they have it (or they changed, I haven't)
213 if (theirHash && !mine) {
214 actions.push({ path, action: 'pull', remote: theirs })
215 continue
216 }
217
218 // Only I have it (or I changed, they haven't)
219 if (myHash && !theirs) {
220 actions.push({ path, action: 'push', local: mine })
221 continue
222 }
223
224 // Both null — both deleted
225 if (!myHash && !theirHash) {
226 actions.push({ path, action: 'skip' })
227 continue
228 }
229
230 // One deleted, one has content
231 if (mine && mine.op === OP_DEL && theirHash) {
232 // I deleted, they still have it
233 if (ancestorHash && ancestorHash === theirHash) {
234 // They haven't changed since ancestor — propagate my delete
235 actions.push({ path, action: 'delete-remote', local: mine, remote: theirs })
236 } else {
237 // They changed it after ancestor — conflict
238 actions.push({ path, action: 'conflict', local: mine, remote: theirs, ancestor })
239 }
240 continue
241 }
242
243 if (theirs && theirs.op === OP_DEL && myHash) {
244 // They deleted, I still have it
245 if (ancestorHash && ancestorHash === myHash) {
246 // I haven't changed since ancestor — accept their delete
247 actions.push({ path, action: 'delete-local', local: mine, remote: theirs })
248 } else {
249 // I changed it after ancestor — conflict
250 actions.push({ path, action: 'conflict', local: mine, remote: theirs, ancestor })
251 }
252 continue
253 }
254
255 // Both have different content — use ancestor to determine conflict
256 if (myHash && theirHash && myHash !== theirHash) {
257 if (ancestorHash === myHash) {
258 // Only they changed — pull
259 actions.push({ path, action: 'pull', local: mine, remote: theirs })
260 } else if (ancestorHash === theirHash) {
261 // Only I changed — push
262 actions.push({ path, action: 'push', local: mine, remote: theirs })
263 } else {
264 // Both changed — conflict
265 actions.push({ path, action: 'conflict', local: mine, remote: theirs, ancestor })
266 }
267 continue
268 }
269
270 // Fallback — shouldn't reach here
271 actions.push({ path, action: 'skip', local: mine, remote: theirs })
272 }
273
274 return actions
275 }
276
277 // ── Conflict resolution ─────────────────────────────────────
278
279 /**
280 * Resolve a conflict using a strategy
281 * @param {{ path, local, remote, ancestor }} conflict
282 * @param {string} strategy - one of STRATEGIES
283 * @returns {{ action: 'pull'|'push'|'keep-both'|'skip', winner?: 'local'|'remote' }}
284 */
285 resolveConflict (conflict, strategy) {
286 switch (strategy) {
287 case 'mine-wins':
288 return { action: 'push', winner: 'local' }
289
290 case 'theirs-wins':
291 return { action: 'pull', winner: 'remote' }
292
293 case 'newest-wins': {
294 const mySeq = conflict.local ? conflict.local.seq : -1
295 const theirSeq = conflict.remote ? conflict.remote.seq : -1
296 if (mySeq >= theirSeq) {
297 return { action: 'push', winner: 'local' }
298 }
299 return { action: 'pull', winner: 'remote' }
300 }
301
302 case 'keep-both':
303 return { action: 'keep-both' }
304
305 case 'skip':
306 return { action: 'skip' }
307
308 case 'prompt':
309 default:
310 // Add to conflict queue for user resolution
311 this._conflicts.push(conflict)
312 return { action: 'skip' }
313 }
314 }
315
316 /**
317 * Resolve a queued conflict by user choice
318 * @param {string} path
319 * @param {string} choice - 'overwrite' | 'keep-both' | 'skip'
320 * @returns {{ action: string } | null}
321 */
322 resolveQueued (path, choice) {
323 const idx = this._conflicts.findIndex(c => c.path === path)
324 if (idx === -1) return null
325
326 const conflict = this._conflicts.splice(idx, 1)[0]
327
328 switch (choice) {
329 case 'overwrite':
330 return { action: 'pull', winner: 'remote', conflict }
331 case 'keep-mine':
332 return { action: 'push', winner: 'local', conflict }
333 case 'keep-both':
334 return { action: 'keep-both', conflict }
335 case 'skip':
336 default:
337 return { action: 'skip', conflict }
338 }
339 }
340
341 /**
342 * Update sync state after a successful sync operation
343 * @param {string} peerKey
344 * @param {string} path
345 * @param {string} hash - the hash both sides now agree on
346 * @param {number} seq - the sequence number of the operation
347 */
348 async markSynced (peerKey, path, hash, seq) {
349 await this._fileIndex.putSyncState(peerKey, path, {
350 lastSeq: seq,
351 lastHash: hash
352 })
353 }
354
355 // ── Internal ────────────────────────────────────────────────
356
357 _getMaxSyncedSeq (syncStates, _side) {
358 let max = -1
359 for (const s of syncStates) {
360 if (s.lastSeq > max) max = s.lastSeq
361 }
362 return max
363 }
364}
lib/v3/transfer.js +0 -808
……@@ -1,808 +0,0 @@
1/**
2 * TransferManager — On-demand chunking and streaming file transfer
3 *
4 * Sender side:
5 * 1. Read file from disk
6 * 2. Chunk on demand (adaptive FastCDC)
7 * 3. Send chunk hashes → peer says which it needs → stream those chunks
8 * 4. Release chunks from memory after send
9 *
10 * Receiver side:
11 * 1. Receive chunk hashes, respond with which are missing
12 * 2. Receive chunks, write to <file>.overlay-partial
13 * 3. Verify each chunk hash
14 * 4. Atomic rename on completion
15 *
16 * No blob storage. Chunks exist in memory during transfer only.
17 */
18
19import fs from 'bare-fs'
20import path from 'bare-path'
21import os from 'bare-os'
22import c from 'compact-encoding'
23import sodium from 'sodium-universal'
24import { chunk as chunkBuffer, chunkStream, hashChunk, createStreamingHasher, selectTier, getTierParams } from './chunker.js'
25
26export const PARTIAL_SUFFIX = '.overlay-partial'
27
28export const partialPathFor = (targetPath) => path.join(path.dirname(targetPath), path.basename(targetPath) + PARTIAL_SUFFIX)
29
30// Resume verification + whole-file hashing run async + yielding so a multi-GB pass
31// never blocks the event loop (IPC, pause/cancel, progress stay live).
32const VERIFY_YIELD_EVERY = 64
33const yieldToLoop = () => new Promise((resolve) => setTimeout(resolve, 0))
34
35// Receive journal: bitmap + whole-file hash snapshot, stored in app-private storage
36// (opts.journalDir) keyed by a digest of the destination path — never next to the
37// download — so resume is O(1) and the incremental verify continues with no re-read.
38const JOURNAL_SUFFIX = '.journal'
39const JOURNAL_MAGIC = 0x4d4a5633
40const JOURNAL_FLUSH_EVERY = 512
41const JOURNAL_FLUSH_INTERVAL_MS = 1500
42export const journalNameFor = (targetPath) => hashChunk(Buffer.from(targetPath)) + JOURNAL_SUFFIX
43
44// Block-read prepare — host-adaptive read block size.
45const READ_BLOCK_DEFAULT = 8 * 1024 * 1024
46const READ_BLOCK_LOW_RAM = 4 * 1024 * 1024
47const LOW_RAM_THRESHOLD = 6 * 1024 * 1024 * 1024
48
49// Cap on stashed just-received chunk buffers (per transfer, insert-refusal), and the
50// hash pump's yield budget — stash-fed iterations never await, so the pump yields by
51// bytes hashed, not chunk count.
52const MEM_STASH_BYTES_DEFAULT = 16 * 1024 * 1024
53const DRAIN_YIELD_BYTES = 2 * 1024 * 1024
54
55// Emit file blocks >= tier maxSize so chunkStream drains every block
56// instead of re-copying a growing pending buffer (64 KiB createReadStream reads force
57// ~56x memcpy at tier 3). Owns its fd and closes it on every terminal path: normal
58// end, consumer break/throw (for-await calls .return() → finally), read error.
59async function * readFileBlocks (filePath, size, blockSize) {
60 const fd = await fs.open(filePath, 'r')
61 try {
62 let pos = 0
63 while (pos < size) {
64 const want = Math.min(blockSize, size - pos)
65 const buf = Buffer.allocUnsafe(want)
66 let filled = 0
67 while (filled < want) {
68 const n = await fs.read(fd, buf, filled, want - filled, pos + filled)
69 if (n <= 0) break
70 filled += n
71 }
72 if (filled === 0) break
73 pos += filled
74 yield filled === buf.length ? buf : buf.subarray(0, filled)
75 }
76 } finally {
77 try { await fs.close(fd) } catch {}
78 }
79}
80
81function bitmapOf (received, total) {
82 const bm = Buffer.alloc((total + 7) >> 3)
83 for (const i of received) bm[i >> 3] |= 1 << (i & 7)
84 return bm
85}
86
87// Drop journals whose partial is gone, are stale, or are corrupt/foreign. Each
88// journal self-describes its partial path (read cheaply from the header). Standalone
89// (no TransferManager instance) so a boot-time sweep can reclaim orphans without
90// the overlay stack being live.
91export function cleanupOrphanedJournals (journalDir, maxAge = 7 * 86400000) {
92 const cleaned = []
93 if (!journalDir) return cleaned
94 let entries
95 try { entries = fs.readdirSync(journalDir) } catch { return cleaned }
96 const now = Date.now()
97 for (const entry of entries) {
98 if (!entry.endsWith(JOURNAL_SUFFIX)) continue
99 const full = path.join(journalDir, entry)
100 let remove = false
101 try {
102 const raw = fs.readFileSync(full)
103 const s = { start: 0, end: raw.length, buffer: raw }
104 if (c.uint32.decode(s) !== JOURNAL_MAGIC) remove = true
105 else {
106 c.uint32.decode(s)
107 const partialPath = c.string.decode(s)
108 if (!fs.existsSync(partialPath)) remove = true
109 else if (now - fs.statSync(full).mtimeMs > maxAge) remove = true
110 }
111 } catch { remove = true }
112 if (remove) { try { fs.unlinkSync(full); cleaned.push(full) } catch {} }
113 }
114 return cleaned
115}
116
117export class TransferManager {
118 constructor (fileIndex, opts = {}) {
119 this._fileIndex = fileIndex
120 this._maxConcurrency = opts.maxConcurrency || 4
121 this._active = new Map() // path → transfer state
122 this._memStashBytes = opts.memStashBytes ?? MEM_STASH_BYTES_DEFAULT
123 this._journalDir = opts.journalDir || null
124 if (this._journalDir) { try { fs.mkdirSync(this._journalDir, { recursive: true }) } catch {} }
125 }
126
127 _journalPathFor (targetPath) {
128 if (!this._journalDir) return null
129 return path.join(this._journalDir, journalNameFor(targetPath))
130 }
131
132 // ── Sender side ─────────────────────────────────────────────
133
134 /**
135 * Prepare a file for sending — read from disk, chunk, return hashes.
136 * Chunk map is persisted for large files (>1MB).
137 *
138 * @param {string} filePath - absolute path on disk
139 * @param {string} overlayPath - path in the overlay namespace
140 * @returns {{ tier: number, chunks: Array<{ hash, offset, length }>, size: number } | null}
141 */
142 async prepareFile (filePath, overlayPath, opts = {}) {
143 // Check file exists and get stats
144 let stat
145 try {
146 stat = fs.statSync(filePath)
147 } catch {
148 return null
149 }
150
151 if (!stat.isFile()) return null
152
153 const mtimeBefore = stat.mtimeMs
154
155 // Check for existing chunk map (skip re-chunking if valid)
156 const existing = await this._fileIndex.getChunkMap(overlayPath)
157 const fileMeta = await this._fileIndex.getFile(overlayPath)
158
159 if (existing && fileMeta && fileMeta.mtime === mtimeBefore) {
160 // Chunk map still valid — file hasn't changed.
161 // Backfill the content-addressed alias so the serve path (a
162 // different overlay key) reuses this map instead of re-reading the file —
163 // covers maps chunked before content-addressed caching existed.
164 if (fileMeta.contentHash && stat.size >= 1048576 && !await this._fileIndex.hasChunkMapByHash(fileMeta.contentHash)) {
165 await this._fileIndex.putChunkMapByHash(fileMeta.contentHash, existing)
166 }
167 return {
168 tier: selectTier(stat.size),
169 chunks: existing,
170 size: stat.size,
171 contentHash: fileMeta.contentHash // surfaced so the publish path gets the hash from one pass
172 }
173 }
174
175 // Read in blocks >= tier maxSize and chunk from views (copy:false).
176 // Cut points are content-stable across block sizes, so peers still match hashes.
177 // size is passed to the hasher so the streaming digest matches hashChunk(buffer).
178 const tier = selectTier(stat.size)
179 const blockSize = Math.max(
180 getTierParams(tier).maxSize,
181 os.totalmem() < LOW_RAM_THRESHOLD ? READ_BLOCK_LOW_RAM : READ_BLOCK_DEFAULT
182 )
183 const fileHasher = createStreamingHasher({ size: stat.size })
184 const chunks = []
185
186 for await (const c of chunkStream(readFileBlocks(filePath, stat.size, blockSize), { tier, copy: false })) {
187 // honor a publish-cancel signal (per-chunk granularity); on throw the
188 // for-await calls readFileBlocks.return() which closes the fd.
189 if (opts.signal?.aborted) { const e = new Error('publish aborted'); e.code = 'ECANCELLED'; throw e }
190 fileHasher.update(c.data)
191 chunks.push({ hash: c.hash, offset: c.offset, length: c.length })
192 opts.onProgress?.(c.length)
193 }
194
195 // mtime guard — abort if file changed during the streaming read. A
196 // vanished source (moved out of the folder mid-read) is just another "changed
197 // during read" case — the open fd kept the read alive after a same-volume
198 // rename, so it is this path-resolving stat that trips ENOENT. Guard it like
199 // the pre-read stat above and re-queue rather than throw.
200 let statAfter
201 try {
202 statAfter = fs.statSync(filePath)
203 } catch {
204 return null // source vanished during read, caller should re-queue
205 }
206 if (statAfter.mtimeMs !== mtimeBefore) {
207 return null // file changed during read, caller should re-queue
208 }
209
210 // Post-read guard: a mid-read shrink can feed the digest fewer bytes than the
211 // size baked into its leaf prefix — re-queue instead of persisting a bad hash.
212 if (fileHasher.bytes !== stat.size) return null
213
214 // Update file index
215 const contentHash = fileHasher.digest()
216
217 // Persist chunk map for large files — both path-keyed (legacy serve lookups)
218 // and content-addressed, so the serve path reuses it across keys.
219 // byHashOnly (the publish prepareForServe pass) keeps ONLY the
220 // content-addressed map — the serve path resolves by hash, so the path-keyed map
221 // + putFile under a throwaway overlay key would be dead, never-read state.
222 if (stat.size >= 1048576) {
223 if (!opts.byHashOnly) await this._fileIndex.putChunkMap(overlayPath, chunks)
224 if (!await this._fileIndex.hasChunkMapByHash(contentHash)) await this._fileIndex.putChunkMapByHash(contentHash, chunks)
225 }
226
227 if (!opts.byHashOnly) {
228 await this._fileIndex.putFile(overlayPath, {
229 contentHash,
230 size: stat.size,
231 mtime: mtimeBefore
232 })
233 }
234
235 return { tier, chunks, size: stat.size, contentHash }
236 }
237
238 /**
239 * Read a specific chunk from disk by its offset and length.
240 * Returns raw bytes — caller sends them to the peer.
241 *
242 * @param {string} filePath - absolute path on disk
243 * @param {number} offset - byte offset
244 * @param {number} length - bytes to read
245 * @returns {Buffer | null}
246 */
247 readChunk (filePath, offset, length) {
248 try {
249 const fd = fs.openSync(filePath, 'r')
250 const buf = Buffer.alloc(length)
251 const bytesRead = fs.readSync(fd, buf, 0, length, offset)
252 fs.closeSync(fd)
253 if (bytesRead !== length) return null
254 return buf
255 } catch {
256 return null
257 }
258 }
259
260 /**
261 * Determine which chunks a peer needs (chunks they don't already have).
262 *
263 * @param {Array<{ hash, offset, length }>} offered - chunks the sender has
264 * @param {Set<string>} peerHas - set of chunk hashes the peer already has
265 * @returns {number[]} indices of chunks the peer needs
266 */
267 computeNeeded (offered, peerHas) {
268 const needed = []
269 for (let i = 0; i < offered.length; i++) {
270 if (!peerHas.has(offered[i].hash)) {
271 needed.push(i)
272 }
273 }
274 return needed
275 }
276
277 // ── Receiver side ───────────────────────────────────────────
278
279 // Start receiving a file. A same-size partial is resumed: the journal restores the
280 // received-set + whole-file hash snapshot in O(1) (no re-read); failing that, an
281 // async, yielding re-verify rebuilds them without blocking the event loop.
282 // opts.isCancelled aborts a long re-verify; opts.onVerifyProgress(0..1) drives the UI.
283 async startReceive (targetPath, meta, opts = {}) {
284 const isCancelled = opts.isCancelled || (() => false)
285 const dir = path.dirname(targetPath)
286 const base = path.basename(targetPath)
287 // Visible partial (no leading dot) so an in-progress download shows in the
288 // destination folder; the distinct ".overlay-partial" suffix keeps host-app
289 // sweeps keyed on a plain ".partial" suffix from deleting a paused overlay
290 // partial.
291 const partialPath = path.join(dir, base + PARTIAL_SUFFIX)
292 const journalPath = this._journalPathFor(targetPath)
293
294 fs.mkdirSync(dir, { recursive: true })
295
296 let resumed = false
297 try { resumed = fs.statSync(partialPath).size === meta.size } catch {}
298
299 let received = new Set()
300 let hasher = meta.contentHash ? createStreamingHasher({ size: meta.size }) : null
301 let hashFrontier = 0
302
303 if (resumed) {
304 const j = journalPath ? this._loadJournal(journalPath, partialPath, meta) : null
305 if (j) {
306 received = j.received
307 hashFrontier = j.hashFrontier
308 if (meta.contentHash) hasher = createStreamingHasher({ size: meta.size, restore: { state: j.hasherState, bytes: j.hasherBytes } })
309 } else {
310 const rec = await this._recoverPartialAsync(partialPath, meta, isCancelled, opts.onVerifyProgress)
311 received = rec.received; hasher = rec.hasher; hashFrontier = rec.hashFrontier
312 }
313 } else {
314 if (journalPath) { try { fs.unlinkSync(journalPath) } catch {} }
315 }
316
317 if (isCancelled()) { const e = new Error('receive cancelled during setup'); e.code = 'ECANCELLED'; throw e }
318
319 // ONE persistent fd for the whole transfer — positioned writeSync/fs.read never
320 // move a shared cursor, so a single handle serves the chunk writes, the hash
321 // pump's gap read-back, and the journal fsync. Replaces an openSync+closeSync per
322 // chunk; 'w+' creates+truncates the fresh partial in the same open.
323 const fd = fs.openSync(partialPath, resumed ? 'r+' : 'w+')
324 if (!resumed) {
325 try { fs.ftruncateSync(fd, meta.size) } catch (err) { try { fs.closeSync(fd) } catch {}; throw err }
326 }
327
328 const total = meta.chunks.length
329 const state = {
330 partialPath,
331 targetPath,
332 journalPath,
333 chunks: meta.chunks,
334 received,
335 bitmap: bitmapOf(received, total),
336 total,
337 size: meta.size,
338 contentHash: meta.contentHash || null,
339 hasher,
340 hashFrontier,
341 advancePromise: null,
342 draining: false,
343 sinceFlush: 0,
344 lastFlush: 0,
345 flushing: false,
346 flushPending: false,
347 fd,
348 // Bounded stash of just-received verified chunk buffers so the hash pump feeds
349 // from memory instead of reading the bytes back off disk. Never load-bearing:
350 // bytes hit disk first; a miss falls back to read-back.
351 memChunks: new Map(),
352 memBytes: 0,
353 stats: { readbacks: 0, stashHits: 0 }
354 }
355
356 // A prior transfer for this path that ended via _fail (stall / all-peers-
357 // gone / disk error) leaves its state here with an open fd — close it before
358 // overwriting so a retry/resume of the same file can't orphan the fd.
359 const prev = this._active.get(targetPath)
360 if (prev) this._closeFd(prev)
361
362 this._active.set(targetPath, state)
363 // A fresh transfer has nothing worth journaling yet; a resume that recovered
364 // chunks persists them now so an immediate re-pause is O(1).
365 if (received.size > 0) this._flushJournal(state)
366 return state
367 }
368
369 // Rebuild the received-set from a same-size partial without blocking the loop:
370 // async reads, a yield every VERIFY_YIELD_EVERY chunks, feeding the contiguous
371 // prefix into the whole-file hasher inline (single pass, no second read).
372 async _recoverPartialAsync (partialPath, meta, isCancelled, onVerifyProgress) {
373 const received = new Set()
374 const hasher = meta.contentHash ? createStreamingHasher({ size: meta.size }) : null
375 let hashFrontier = 0
376 const total = meta.chunks.length
377 const maxLen = meta.chunks.reduce((m, c) => (c.length > m ? c.length : m), 0)
378 const buf = Buffer.allocUnsafe(maxLen)
379 let lastPct = -1
380 onVerifyProgress?.(0)
381 const fd = await fs.open(partialPath, 'r')
382 try {
383 for (let i = 0; i < total; i++) {
384 if (isCancelled()) { const e = new Error('receive recovery cancelled'); e.code = 'ECANCELLED'; throw e }
385 const ci = meta.chunks[i]
386 let n = 0
387 try { n = await fs.read(fd, buf, 0, ci.length, ci.offset) } catch {}
388 const view = buf.subarray(0, ci.length)
389 if (n === ci.length && hashChunk(view) === ci.hash) {
390 received.add(i)
391 if (hasher && i === hashFrontier) { hasher.update(view); hashFrontier++ }
392 }
393 if ((i & (VERIFY_YIELD_EVERY - 1)) === VERIFY_YIELD_EVERY - 1) {
394 const pct = Math.floor(((i + 1) / total) * 100)
395 if (pct !== lastPct) { lastPct = pct; onVerifyProgress?.(pct / 100) }
396 await yieldToLoop()
397 }
398 }
399 } finally {
400 try { await fs.close(fd) } catch {}
401 }
402 onVerifyProgress?.(1)
403 return { received, hasher, hashFrontier }
404 }
405
406 _encodeJournal (state) {
407 const total = state.total
408 const bm = state.bitmap
409 const ch = state.contentHash ? Buffer.from(state.contentHash, 'hex') : Buffer.alloc(32)
410 const snap = state.hasher ? state.hasher.snapshot() : { state: Buffer.alloc(0), bytes: 0 }
411 const s = { start: 0, end: 0, buffer: null }
412 c.uint32.preencode(s, JOURNAL_MAGIC)
413 c.uint32.preencode(s, sodium.crypto_generichash_STATEBYTES)
414 c.string.preencode(s, state.partialPath)
415 c.uint64.preencode(s, state.size)
416 c.uint32.preencode(s, total)
417 c.fixed32.preencode(s, ch)
418 c.uint32.preencode(s, state.hashFrontier)
419 c.uint64.preencode(s, snap.bytes)
420 c.buffer.preencode(s, snap.state)
421 c.buffer.preencode(s, bm)
422 s.buffer = Buffer.allocUnsafe(s.end)
423 c.uint32.encode(s, JOURNAL_MAGIC)
424 c.uint32.encode(s, sodium.crypto_generichash_STATEBYTES)
425 c.string.encode(s, state.partialPath)
426 c.uint64.encode(s, state.size)
427 c.uint32.encode(s, total)
428 c.fixed32.encode(s, ch)
429 c.uint32.encode(s, state.hashFrontier)
430 c.uint64.encode(s, snap.bytes)
431 c.buffer.encode(s, snap.state)
432 c.buffer.encode(s, bm)
433 return s.buffer
434 }
435
436 // Returns the resume state only when the journal is for this partial, binds to
437 // this content, and its snapshot is restorable on this libsodium build; else null
438 // so the caller falls back to the async re-verify.
439 _loadJournal (journalPath, partialPath, meta) {
440 let raw
441 try { raw = fs.readFileSync(journalPath) } catch { return null }
442 try {
443 const s = { start: 0, end: raw.length, buffer: raw }
444 if (c.uint32.decode(s) !== JOURNAL_MAGIC) return null
445 const stateBytes = c.uint32.decode(s)
446 const storedPartial = c.string.decode(s)
447 const size = c.uint64.decode(s)
448 const total = c.uint32.decode(s)
449 const ch = c.fixed32.decode(s).toString('hex')
450 const hashFrontier = c.uint32.decode(s)
451 const hasherBytes = c.uint64.decode(s)
452 const hasherState = c.buffer.decode(s)
453 const bm = c.buffer.decode(s)
454 if (storedPartial !== partialPath) return null
455 if (size !== meta.size || total !== meta.chunks.length) return null
456 if (meta.contentHash && ch !== meta.contentHash) return null
457 if (meta.contentHash) {
458 if (stateBytes !== sodium.crypto_generichash_STATEBYTES) return null
459 if (hasherState.length !== sodium.crypto_generichash_STATEBYTES) return null
460 }
461 const received = new Set()
462 for (let i = 0; i < total; i++) if (bm[i >> 3] & (1 << (i & 7))) received.add(i)
463 return { received, hashFrontier, hasherState, hasherBytes }
464 } catch { return null }
465 }
466
467 // Best-effort async crash-hedge flush: never blocks the receive loop, single-flight
468 // (a flush requested while one runs coalesces into one more pass). fsync the partial
469 // BEFORE the journal references it so a power-loss can never leave the journal
470 // marking a chunk received whose bytes aren't durable.
471 _flushJournal (state) {
472 if (!state.contentHash || !state.journalPath || state.fd == null) return
473 if (state.flushing) { state.flushPending = true; return }
474 if (Date.now() - state.lastFlush < JOURNAL_FLUSH_INTERVAL_MS) return
475 state.flushing = true
476 this._drainFlush(state)
477 }
478
479 async _drainFlush (state) {
480 do {
481 state.flushPending = false
482 state.lastFlush = Date.now()
483 if (state.fd == null) break
484 let buf
485 try { buf = this._encodeJournal(state) } catch { break }
486 const tmp = state.journalPath + '.tmp'
487 try {
488 try { await fs.fsync(state.fd) } catch {}
489 await fs.writeFile(tmp, buf)
490 if (state.fd == null) break // paused/cancelled mid-flush — don't re-create the journal
491 await fs.rename(tmp, state.journalPath)
492 } catch { try { await fs.unlink(tmp) } catch {} }
493 } while (state.flushPending && state.fd != null && state.contentHash)
494 try { await fs.unlink(state.journalPath + '.tmp') } catch {}
495 state.flushing = false
496 }
497
498 // Synchronous durable flush for pause/shutdown, where the app may quit before an
499 // async flush would land.
500 _flushJournalSync (state) {
501 if (!state.contentHash || !state.journalPath) return
502 let buf
503 try { buf = this._encodeJournal(state) } catch { return }
504 if (state.fd != null) { try { fs.fsyncSync(state.fd) } catch {} }
505 try {
506 const tmp = state.journalPath + '.tmp'
507 fs.writeFileSync(tmp, buf)
508 fs.renameSync(tmp, state.journalPath)
509 state.lastFlush = Date.now()
510 } catch { try { fs.unlinkSync(state.journalPath + '.tmp') } catch {} }
511 }
512
513 cleanJournals (maxAge) { return cleanupOrphanedJournals(this._journalDir, maxAge) }
514
515 async _hashWholeFileAsync (filePath, size) {
516 const h = createStreamingHasher({ size })
517 for await (const buf of readFileBlocks(filePath, size, 8 * 1024 * 1024)) {
518 h.update(buf)
519 await yieldToLoop()
520 }
521 return h.digest()
522 }
523
524 // Advance the whole-file hash over the contiguous run of received chunks
525 // as a single-flight BACKGROUND pump: chunk index order == file offset order, so
526 // once chunk `hashFrontier` is present we feed its bytes and move on. The drain is
527 // async + yielding so a large contiguous run that suddenly becomes available (a
528 // gap filling after a low-frontier resume) cannot block the event loop.
529 // Single-flight is `state.draining`, set/cleared SYNCHRONOUSLY inside _drainHash —
530 // a stash-fed drain completes without awaiting, so a promise-based gate would stay
531 // stale until its microtask clear and block same-macrotask restarts (protomux
532 // delivers frame bursts synchronously). advancePromise is only the awaitable
533 // handle for finalize/pause; its clear is guarded against nulling a newer drain.
534 _advanceHash (state) {
535 if (!state.hasher || state.draining) return
536 if (!(state.hashFrontier < state.chunks.length && state.received.has(state.hashFrontier))) return
537 const p = this._drainHash(state)
538 state.advancePromise = p
539 p.catch(() => {}).then(() => { if (state.advancePromise === p) state.advancePromise = null })
540 }
541
542 async _drainHash (state) {
543 state.draining = true
544 try {
545 let sinceYield = 0
546 while (state.fd != null && state.hashFrontier < state.chunks.length && state.received.has(state.hashFrontier)) {
547 const i = state.hashFrontier
548 const c = state.chunks[i]
549 let buf = state.memChunks.get(i)
550 if (buf !== undefined) {
551 state.memChunks.delete(i)
552 state.memBytes -= buf.length
553 state.stats.stashHits++
554 } else {
555 buf = Buffer.alloc(c.length)
556 try { await fs.read(state.fd, buf, 0, c.length, c.offset) } catch { return }
557 state.stats.readbacks++
558 }
559 state.hasher.update(buf)
560 state.hashFrontier++
561 // Yield by BYTE volume: the stash path has no awaits, so a chunk-count
562 // cadence would let a gap-fill hash the whole stash synchronously inside
563 // the network event that delivered the gap chunk.
564 sinceYield += buf.length
565 if (sinceYield >= DRAIN_YIELD_BYTES) { sinceYield = 0; await yieldToLoop() }
566 }
567 } finally {
568 state.draining = false
569 }
570 }
571
572 // Close the persistent fd — MUST run before finalize's rename (an open handle
573 // blocks the rename on Windows) — and drop the chunk stash.
574 _closeFd (state) {
575 if (state.fd != null) {
576 try { fs.closeSync(state.fd) } catch {}
577 state.fd = null
578 }
579 state.memChunks.clear()
580 state.memBytes = 0
581 }
582
583 /**
584 * Write a received chunk to the partial file. Verifies the hash.
585 *
586 * @param {string} targetPath - the final file path (used as key)
587 * @param {number} index - chunk index
588 * @param {Buffer} data - raw chunk bytes
589 * @returns {{ ok: boolean, error?: string }}
590 */
591 writeChunk (targetPath, index, data) {
592 const state = this._active.get(targetPath)
593 if (!state) return { ok: false, error: 'no active transfer' }
594 if (index < 0 || index >= state.chunks.length) return { ok: false, error: 'index out of range' }
595 // Terminal race (chunk landed after close) — codeless so the scheduler
596 // re-assigns instead of failing the fetch.
597 if (state.fd == null) return { ok: false, error: 'transfer closed' }
598
599 const expected = state.chunks[index]
600
601 // Verify hash
602 const actual = hashChunk(data)
603 if (actual !== expected.hash) {
604 return { ok: false, error: `hash mismatch at index ${index}: expected ${expected.hash.slice(0, 16)}... got ${actual.slice(0, 16)}...` }
605 }
606
607 // Verify length
608 if (data.length !== expected.length) {
609 return { ok: false, error: `length mismatch at index ${index}: expected ${expected.length} got ${data.length}` }
610 }
611
612 // Write to partial at the chunk's offset through the persistent fd. A local
613 // I/O failure here (ENOSPC / EACCES / EROFS) is fatal and non-retryable —
614 // return its code so the scheduler fails the fetch instead of looping over
615 // other peers. Loop on a SHORT count: libuv swallows an error after partial
616 // progress and returns short without throwing; retrying the remainder
617 // surfaces the real coded error instead of leaving a silent hole the
618 // stash-fed digest would never see.
619 try {
620 let written = 0
621 while (written < data.length) {
622 const n = fs.writeSync(state.fd, data, written, data.length - written, expected.offset + written)
623 if (!(n > 0)) { const e = new Error(`short write at index ${index}: ${written}/${data.length} bytes`); e.code = 'EIO'; throw e }
624 written += n
625 }
626 } catch (err) {
627 return { ok: false, error: err.message, code: err.code }
628 }
629
630 state.received.add(index)
631 state.bitmap[index >> 3] |= 1 << (index & 7)
632 // Stash only what the pump can consume cleanly: skip when already hashed or
633 // stashed, when the pump is mid-drain on exactly this index, or over cap.
634 // Copied, not aliased — wire buffers are views into shared network slabs (udx
635 // read slab, decrypted in place by secret-stream), so retaining `data` would
636 // pin whole slabs and bet the digest on their write-once behavior.
637 if (state.hasher && !state.memChunks.has(index) && state.memBytes + data.length <= this._memStashBytes &&
638 (index > state.hashFrontier || (index === state.hashFrontier && !state.draining))) {
639 state.memChunks.set(index, Buffer.from(data))
640 state.memBytes += data.length
641 }
642 this._advanceHash(state)
643 if (++state.sinceFlush >= JOURNAL_FLUSH_EVERY) { state.sinceFlush = 0; this._flushJournal(state) }
644
645 return { ok: true }
646 }
647
648 /**
649 * Check if a transfer is complete (all chunks received).
650 *
651 * @param {string} targetPath
652 * @returns {boolean}
653 */
654 isComplete (targetPath) {
655 const state = this._active.get(targetPath)
656 if (!state) return false
657 return state.received.size === state.total
658 }
659
660 /**
661 * Finalize a completed transfer — atomic rename from partial to target.
662 *
663 * @param {string} targetPath
664 * @returns {{ ok: boolean, error?: string }}
665 */
666 async finalize (targetPath) {
667 const state = this._active.get(targetPath)
668 if (!state) return { ok: false, error: 'no active transfer' }
669 if (state.received.size !== state.total) {
670 return { ok: false, error: `incomplete: ${state.received.size}/${state.total} chunks` }
671 }
672
673 // whole-file integrity — computed incrementally by the background hash
674 // pump during the transfer. Kick the pump and await it to quiescence, then read
675 // the digest (no full re-read). On a mismatch, drop the corrupt partial so a
676 // retry starts clean.
677 if (state.hasher) {
678 this._advanceHash(state)
679 while (state.advancePromise) { try { await state.advancePromise } catch {} }
680 }
681 if (state.contentHash) {
682 const actual = state.hashFrontier === state.chunks.length ? state.hasher.digest() : null
683 if (actual !== state.contentHash) {
684 this._closeFd(state)
685 try { fs.unlinkSync(state.partialPath) } catch {}
686 if (state.journalPath) { try { fs.unlinkSync(state.journalPath) } catch {} }
687 this._active.delete(targetPath)
688 return { ok: false, error: 'content-hash mismatch', code: 'EHASHMISMATCH' }
689 }
690 }
691 this._closeFd(state)
692
693 try {
694 fs.renameSync(state.partialPath, state.targetPath)
695 } catch (err) {
696 // Carry the code (routes through the coded local-I/O rethrow) and clear the
697 // state — the partial + journal stay for a clean resume-retry.
698 this._active.delete(targetPath)
699 return { ok: false, error: 'rename failed: ' + err.message, code: err.code }
700 }
701
702 if (state.journalPath) { try { fs.unlinkSync(state.journalPath) } catch {} }
703 this._active.delete(targetPath)
704 return { ok: true }
705 }
706
707 // Stop receiving but KEEP the partial on disk so a later startReceive
708 // verifies and resumes it. Distinct from cancel(), which unlinks the partial.
709 // Drains the in-flight hash advance to the full contiguous frontier first, so the
710 // journal captures it and resume stays O(1) (no re-read) and deterministic.
711 async pause (targetPath) {
712 const state = this._active.get(targetPath)
713 if (!state) return
714 if (state.hasher) {
715 this._advanceHash(state)
716 while (state.advancePromise) { try { await state.advancePromise } catch {} }
717 }
718 this._flushJournalSync(state)
719 this._closeFd(state)
720 this._active.delete(targetPath)
721 }
722
723 /**
724 * Cancel an active transfer and clean up the partial file.
725 *
726 * @param {string} targetPath
727 */
728 cancel (targetPath) {
729 const state = this._active.get(targetPath)
730 if (!state) return
731
732 this._closeFd(state)
733 try {
734 fs.unlinkSync(state.partialPath)
735 } catch {
736 // partial file may not exist
737 }
738 if (state.journalPath) { try { fs.unlinkSync(state.journalPath) } catch {} }
739
740 this._active.delete(targetPath)
741 }
742
743 /**
744 * Get transfer progress for an active transfer.
745 *
746 * @param {string} targetPath
747 * @returns {{ received: number, total: number, percentage: number } | null}
748 */
749 getProgress (targetPath) {
750 const state = this._active.get(targetPath)
751 if (!state) return null
752
753 return {
754 received: state.received.size,
755 total: state.total,
756 percentage: state.total > 0 ? Math.round(state.received.size / state.total * 100) : 0
757 }
758 }
759
760 /**
761 * List all active transfers.
762 *
763 * @returns {Array<{ path: string, received: number, total: number, percentage: number }>}
764 */
765 listActive () {
766 const result = []
767 for (const [targetPath, state] of this._active) {
768 result.push({
769 path: targetPath,
770 received: state.received.size,
771 total: state.total,
772 percentage: state.total > 0 ? Math.round(state.received.size / state.total * 100) : 0
773 })
774 }
775 return result
776 }
777
778 /**
779 * Clean up stale partial files in a directory (older than maxAge).
780 *
781 * @param {string} dir - directory to scan
782 * @param {number} [maxAge=86400000] - max age in ms (default 24h)
783 * @returns {string[]} paths of cleaned files
784 */
785 cleanPartials (dir, maxAge = 86400000) {
786 const cleaned = []
787 try {
788 const entries = fs.readdirSync(dir)
789 const now = Date.now()
790 for (const entry of entries) {
791 if (!entry.endsWith(PARTIAL_SUFFIX)) continue
792 const full = path.join(dir, entry)
793 try {
794 const stat = fs.statSync(full)
795 if (now - stat.mtimeMs > maxAge) {
796 fs.unlinkSync(full)
797 cleaned.push(full)
798 }
799 } catch {
800 // skip unreadable files
801 }
802 }
803 } catch {
804 // directory may not exist
805 }
806 return cleaned
807 }
808}
lib/watch-manager.js +0 -279
……@@ -1,279 +0,0 @@
1/**
2 * WatchManager — Tiered filesystem watching with localwatch
3 *
4 * Hot directories: real-time inotify via localwatch
5 * Cold directories: periodic mtime scan as fallback
6 * Rename detection: debounced delete+create with same hash = rename
7 *
8 * Emits change events that the sync engine consumes to update the Hypercore feed.
9 */
10
11import Localwatch from 'localwatch'
12import fs from '#fs'
13import path from '#path'
14import ReadyResource from 'ready-resource'
15import { hashChunk } from './chunker.js'
16import { createMatcher, loadIgnoreFile } from './ignore.js'
17
18export class WatchManager extends ReadyResource {
19 constructor (opts = {}) {
20 super()
21 this._watchers = new Map() // dir → Localwatch
22 this._scanDirs = new Map() // dir → { interval, timer }
23 this._listeners = [] // onChange callbacks
24 this._matchers = new Map() // dir → isIgnored function
25 this._extraIgnore = opts.ignore || [] // additional patterns from opts
26 this._scanInterval = opts.scanInterval || 30000 // 30s default
27 this._pendingDeletes = new Map() // path → { hash, timeout }
28 this._renameWindow = opts.renameWindow || 200 // ms
29 }
30
31 async _open () {}
32
33 async _close () {
34 for (const [, watcher] of this._watchers) {
35 try { watcher.destroy() } catch {}
36 }
37 this._watchers.clear()
38
39 for (const [, entry] of this._scanDirs) {
40 clearInterval(entry.timer)
41 }
42 this._scanDirs.clear()
43
44 for (const [, pending] of this._pendingDeletes) {
45 clearTimeout(pending.timeout)
46 }
47 this._pendingDeletes.clear()
48 }
49
50 /**
51 * Register a callback for file changes
52 * @param {function} fn - called with { type: 'update'|'delete'|'rename', path, oldPath? }
53 */
54 onChange (fn) {
55 this._listeners.push(fn)
56 }
57
58 /**
59 * Watch a directory with real-time localwatch (inotify).
60 * Use for hot/actively-edited directories.
61 *
62 * @param {string} dir - absolute path to watch
63 */
64 watchLive (dir) {
65 if (this._watchers.has(dir)) return
66
67 const matcher = this._getOrCreateMatcher(dir)
68
69 const filter = (filename) => {
70 const base = path.basename(filename)
71 return !matcher(filename, base)
72 }
73
74 const watcher = new Localwatch(dir, { filter, settle: true })
75 this._watchers.set(dir, watcher)
76
77 this._drainWatcher(dir, watcher)
78 }
79
80 /**
81 * Watch a directory with periodic mtime scanning.
82 * Use for cold/archive directories or when inotify limits are hit.
83 *
84 * @param {string} dir - absolute path to scan
85 * @param {number} [interval] - scan interval in ms (default: 30s)
86 */
87 watchPeriodic (dir, interval) {
88 if (this._scanDirs.has(dir)) return
89
90 const ms = interval || this._scanInterval
91 const state = { mtimes: new Map() }
92
93 // Initial scan
94 this._scanDir(dir, state)
95
96 const timer = setInterval(() => this._scanDir(dir, state), ms)
97 this._scanDirs.set(dir, { timer, state })
98 }
99
100 /**
101 * Stop watching a directory (both live and periodic).
102 *
103 * @param {string} dir
104 */
105 unwatch (dir) {
106 const watcher = this._watchers.get(dir)
107 if (watcher) {
108 watcher.destroy()
109 this._watchers.delete(dir)
110 }
111
112 const scan = this._scanDirs.get(dir)
113 if (scan) {
114 clearInterval(scan.timer)
115 this._scanDirs.delete(dir)
116 }
117 }
118
119 /**
120 * List watched directories.
121 * @returns {Array<{ dir: string, mode: 'live'|'periodic' }>}
122 */
123 listWatched () {
124 const result = []
125 for (const dir of this._watchers.keys()) {
126 result.push({ dir, mode: 'live' })
127 }
128 for (const dir of this._scanDirs.keys()) {
129 result.push({ dir, mode: 'periodic' })
130 }
131 return result
132 }
133
134 // ── Internal ────────────────────────────────────────────────
135
136 async _drainWatcher (dir, watcher) {
137 try {
138 for await (const diff of watcher) {
139 for (const entry of diff) {
140 this._handleChange(entry.type, entry.filename)
141 }
142 }
143 } catch {
144 // Watcher destroyed or errored
145 }
146 }
147
148 _handleChange (type, filePath) {
149 if (type === 'delete') {
150 this._handleDelete(filePath)
151 } else if (type === 'update') {
152 this._handleUpdate(filePath)
153 }
154 }
155
156 _handleUpdate (filePath) {
157 // Check if this matches a pending delete (rename detection)
158 // A rename looks like: delete old path, then create new path with same content
159 for (const [oldPath, pending] of this._pendingDeletes) {
160 try {
161 const data = fs.readFileSync(filePath)
162 const hash = hashChunk(data)
163 if (hash === pending.hash) {
164 // Same content — this is a rename
165 clearTimeout(pending.timeout)
166 this._pendingDeletes.delete(oldPath)
167 this._emit({ type: 'rename', path: filePath, oldPath })
168 return
169 }
170 } catch {
171 // Can't read the new file — treat as normal update
172 }
173 }
174
175 this._emit({ type: 'update', path: filePath })
176 }
177
178 _handleDelete (filePath) {
179 // Try to detect rename: hold the delete for a short window
180 // If an update with the same hash arrives, it's a rename
181 let hash = null
182 // We can't hash a deleted file, but we might have cached the hash
183 // For now, emit delete immediately if we can't detect rename
184 // The rename detection works from the update side (checking pending deletes)
185
186 const timeout = setTimeout(() => {
187 this._pendingDeletes.delete(filePath)
188 this._emit({ type: 'delete', path: filePath })
189 }, this._renameWindow)
190
191 this._pendingDeletes.set(filePath, { hash, timeout })
192 }
193
194 _scanDir (dir, state) {
195 let entries
196 try {
197 entries = this._readdirRecursive(dir)
198 } catch {
199 return
200 }
201
202 const currentPaths = new Set()
203
204 for (const { filePath, mtime } of entries) {
205 currentPaths.add(filePath)
206 const prevMtime = state.mtimes.get(filePath)
207
208 if (prevMtime === undefined) {
209 // New file
210 state.mtimes.set(filePath, mtime)
211 this._emit({ type: 'update', path: filePath })
212 } else if (mtime > prevMtime) {
213 // Modified file
214 state.mtimes.set(filePath, mtime)
215 this._emit({ type: 'update', path: filePath })
216 }
217 }
218
219 // Check for deleted files
220 for (const [filePath] of state.mtimes) {
221 if (!currentPaths.has(filePath)) {
222 state.mtimes.delete(filePath)
223 this._emit({ type: 'delete', path: filePath })
224 }
225 }
226 }
227
228 _readdirRecursive (dir) {
229 const matcher = this._getOrCreateMatcher(dir)
230 const results = []
231 const stack = [dir]
232
233 while (stack.length > 0) {
234 const current = stack.pop()
235 let entries
236 try {
237 entries = fs.readdirSync(current)
238 } catch {
239 continue
240 }
241
242 for (const name of entries) {
243 const full = path.join(current, name)
244 if (matcher(full, name)) continue
245
246 let stat
247 try {
248 stat = fs.statSync(full)
249 } catch {
250 continue
251 }
252
253 if (stat.isDirectory()) {
254 stack.push(full)
255 } else if (stat.isFile()) {
256 results.push({ filePath: full, mtime: stat.mtimeMs })
257 }
258 }
259 }
260
261 return results
262 }
263
264 _getOrCreateMatcher (dir) {
265 if (this._matchers.has(dir)) return this._matchers.get(dir)
266
267 // Load .overlayignore from the watched directory
268 const filePatterns = loadIgnoreFile(dir)
269 const matcher = createMatcher([...this._extraIgnore, ...filePatterns])
270 this._matchers.set(dir, matcher)
271 return matcher
272 }
273
274 _emit (event) {
275 for (const fn of this._listeners) {
276 fn(event)
277 }
278 }
279}
package.json +26 -36
……@@ -1,7 +1,7 @@
11{
22 "name": "hyper-overlay",
33 "version": "0.3.0",
4 "description": "Content-addressed union filesystem — P2P file distribution with FastCDC chunking and zero duplication",
4 "description": "Content-addressed P2P file distribution over Protomux — FastCDC chunking, multi-source fetch, resumable transfers, zero blob duplication. Files live on disk; the overlay is a thin index and transfer protocol.",
55 "type": "module",
66 "main": "index.js",
77 "imports": {
……@@ -20,78 +20,68 @@
2020 "#zlib": {
2121 "bare": "bare-zlib",
2222 "default": "zlib"
23 },
24 "#http": {
25 "bare": "bare-http1",
26 "default": "http"
27 },
28 "#process": {
29 "bare": "bare-process",
30 "default": "process"
3123 }
3224 },
3325 "exports": {
3426 ".": "./index.js",
35 "./chunker": "./lib/chunker.js",
36 "./pointer": "./lib/pointer.js",
37 "./manifest": "./lib/manifest.js",
38 "./registry": "./lib/chunk-registry.js",
27 "./overlay": "./lib/overlay.js",
3928 "./protocol": "./lib/protocol.js",
4029 "./messages": "./lib/messages.js",
41 "./http": "./lib/http.js",
42 "./fuse": "./lib/fuse.js",
30 "./chunker": "./lib/chunker.js",
31 "./chunk-scheduler": "./lib/chunk-scheduler.js",
32 "./transfer": "./lib/transfer.js",
4333 "./file-index": "./lib/file-index.js",
4434 "./sync-engine": "./lib/sync-engine.js",
45 "./transfer": "./lib/transfer.js",
46 "./protocol-v2": "./lib/protocol-v2.js",
47 "./overlay-v2": "./lib/overlay-v2.js",
48 "./messages-v2": "./lib/messages-v2.js",
49 "./watch-manager": "./lib/watch-manager.js",
35 "./tree": "./lib/tree.js",
36 "./pointer": "./lib/pointer.js",
5037 "./ignore": "./lib/ignore.js",
51 "./tree": "./lib/tree.js"
38 "./manifest": "./lib/manifest.js",
39 "./protocol-v2": "./lib/protocol.js",
40 "./overlay-v2": "./lib/overlay.js",
41 "./messages-v2": "./lib/messages.js",
42 "./package.json": "./package.json"
5243 },
5344 "scripts": {
5445 "test": "brittle-bare test/*.test.js",
5546 "test:node": "brittle-node test/*.test.js"
5647 },
48 "engines": {
49 "bare": ">=1.16.0"
50 },
5751 "license": "AGPL-3.0",
5852 "keywords": [
5953 "p2p",
6054 "hypercore",
55 "hyperbee",
56 "protomux",
6157 "overlay",
6258 "content-addressed",
6359 "deduplication",
64 "filesystem",
65 "hyperswarm",
60 "file-transfer",
6661 "fastcdc",
67 "chunking"
62 "chunking",
63 "multi-source",
64 "resumable",
65 "bare"
6866 ],
6967 "dependencies": {
70 "@hyperswarm/testnet": "^3.1.4",
7168 "b4a": "^1.0.0",
7269 "bare-fs": "^4.7.2",
73 "bare-http1": "^4.5.5",
74 "bare-native": "^0.1.2",
7570 "bare-os": "^3.8.7",
7671 "bare-path": "^3.0.0",
77 "bare-process": "^4.4.1",
7872 "bare-zlib": "^1.3.3",
79 "blind-pairing": "^2.3.1",
8073 "compact-encoding": "^3.2.0",
8174 "corestore": "^7.0.0",
8275 "hyperbee": "^2.0.0",
8376 "hypercore-crypto": "^3.0.0",
84 "hyperswarm": "^4.0.0",
85 "localwatch": "^1.4.3",
8677 "protomux": "^3.0.0",
87 "quickbit-universal": "^2.0.0",
8878 "ready-resource": "^1.0.0",
79 "sodium-universal": "^5.0.1",
8980 "streamx": "^2.0.0"
9081 },
91 "optionalDependencies": {
92 "fuse-native": "^2.2.6"
93 },
9482 "devDependencies": {
95 "brittle": "^3.0.0"
83 "brittle": "^3.0.0",
84 "hyperswarm": "^4.0.0",
85 "@hyperswarm/testnet": "^3.1.4"
9686 }
9787}
test/adapter.test.js +0 -223
……@@ -1,223 +0,0 @@
1import test from 'brittle'
2import crypto from 'hypercore-crypto'
3import { tmpStore } from './helpers.js'
4import { HyperOverlay } from '../index.js'
5
6test('stat — file', async (t) => {
7 const overlay = new HyperOverlay(tmpStore())
8 await overlay.ready()
9
10 const data = Buffer.from('stat test content')
11 await overlay.put('/doc.txt', data)
12
13 const s = await overlay.stat('/doc.txt')
14 t.ok(s, 'stat returned')
15 t.is(s.type, 'file')
16 t.is(s.size, data.length)
17 t.ok(s.contentHash)
18 t.is(s.chunkCount, 1)
19 t.is(s.hydrated, true, 'locally stored = hydrated')
20 t.ok(s.mtime > 0)
21
22 await overlay.close()
23})
24
25test('stat — directory', async (t) => {
26 const overlay = new HyperOverlay(tmpStore())
27 await overlay.ready()
28
29 await overlay.put('/docs/readme.md', Buffer.from('# Hello'))
30 await overlay.put('/docs/guide.md', Buffer.from('# Guide'))
31
32 const s = await overlay.stat('/docs')
33 t.ok(s, 'stat returned for directory')
34 t.is(s.type, 'dir')
35
36 await overlay.close()
37})
38
39test('stat — root', async (t) => {
40 const overlay = new HyperOverlay(tmpStore())
41 await overlay.ready()
42
43 const s = await overlay.stat('/')
44 t.ok(s)
45 t.is(s.type, 'dir')
46
47 await overlay.close()
48})
49
50test('stat — missing returns null', async (t) => {
51 const overlay = new HyperOverlay(tmpStore())
52 await overlay.ready()
53
54 const s = await overlay.stat('/nope')
55 t.is(s, null)
56
57 await overlay.close()
58})
59
60test('readdir — lists immediate children', async (t) => {
61 const overlay = new HyperOverlay(tmpStore())
62 await overlay.ready()
63
64 await overlay.put('/docs/readme.md', Buffer.from('# Hello'))
65 await overlay.put('/docs/guide.md', Buffer.from('# Guide'))
66 await overlay.put('/docs/api/endpoints.md', Buffer.from('# API'))
67 await overlay.put('/src/index.js', Buffer.from('export default {}'))
68
69 const root = await overlay.readdir('/')
70 t.ok(root.includes('docs'), 'has docs')
71 t.ok(root.includes('src'), 'has src')
72 t.is(root.length, 2, 'only 2 top-level entries')
73
74 const docs = await overlay.readdir('/docs')
75 t.ok(docs.includes('readme.md'))
76 t.ok(docs.includes('guide.md'))
77 t.ok(docs.includes('api'), 'has api subdirectory')
78 t.is(docs.length, 3)
79
80 await overlay.close()
81})
82
83test('read — exact byte range within single chunk', async (t) => {
84 const overlay = new HyperOverlay(tmpStore())
85 await overlay.ready()
86
87 const data = Buffer.from('Hello, world! This is a test of byte range reading.')
88 await overlay.put('/range.txt', data)
89
90 const slice = await overlay.read('/range.txt', 7, 5)
91 t.ok(slice)
92 t.alike(slice, Buffer.from('world'))
93
94 await overlay.close()
95})
96
97test('read — spanning multiple chunks', async (t) => {
98 const overlay = new HyperOverlay(tmpStore())
99 await overlay.ready()
100
101 // 128KB = multiple chunks (~8 at 16KB avg)
102 const data = crypto.randomBytes(128 * 1024)
103 await overlay.put('/multi.bin', data)
104
105 // Read 1000 bytes starting at offset 50000 (crosses chunk boundary)
106 const slice = await overlay.read('/multi.bin', 50000, 1000)
107 t.ok(slice)
108 t.is(slice.length, 1000)
109 t.alike(slice, data.subarray(50000, 51000))
110
111 await overlay.close()
112})
113
114test('read — from start', async (t) => {
115 const overlay = new HyperOverlay(tmpStore())
116 await overlay.ready()
117
118 const data = Buffer.from('abcdefghijklmnop')
119 await overlay.put('/abc.txt', data)
120
121 const slice = await overlay.read('/abc.txt', 0, 5)
122 t.alike(slice, Buffer.from('abcde'))
123
124 await overlay.close()
125})
126
127test('read — past end of file', async (t) => {
128 const overlay = new HyperOverlay(tmpStore())
129 await overlay.ready()
130
131 const data = Buffer.from('short')
132 await overlay.put('/short.txt', data)
133
134 const slice = await overlay.read('/short.txt', 3, 100)
135 t.ok(slice)
136 t.alike(slice, Buffer.from('rt'), 'clamped to file end')
137
138 await overlay.close()
139})
140
141test('read — offset beyond file size', async (t) => {
142 const overlay = new HyperOverlay(tmpStore())
143 await overlay.ready()
144
145 await overlay.put('/tiny.txt', Buffer.from('x'))
146 const slice = await overlay.read('/tiny.txt', 100, 10)
147 t.is(slice.length, 0, 'empty buffer for offset past EOF')
148
149 await overlay.close()
150})
151
152test('createReadStream — full file', async (t) => {
153 const overlay = new HyperOverlay(tmpStore())
154 await overlay.ready()
155
156 const data = crypto.randomBytes(64 * 1024)
157 await overlay.put('/stream.bin', data)
158
159 const stream = overlay.createReadStream('/stream.bin')
160 const parts = []
161 for await (const chunk of stream) parts.push(chunk)
162 const result = Buffer.concat(parts)
163
164 t.is(result.length, data.length)
165 t.alike(result, data)
166
167 await overlay.close()
168})
169
170test('createReadStream — byte range', async (t) => {
171 const overlay = new HyperOverlay(tmpStore())
172 await overlay.ready()
173
174 const data = crypto.randomBytes(128 * 1024)
175 await overlay.put('/rangestream.bin', data)
176
177 const stream = overlay.createReadStream('/rangestream.bin', { start: 10000, end: 20000 })
178 const parts = []
179 for await (const chunk of stream) parts.push(chunk)
180 const result = Buffer.concat(parts)
181
182 t.is(result.length, 10000)
183 t.alike(result, data.subarray(10000, 20000))
184
185 await overlay.close()
186})
187
188test('hydrate — local file already hydrated', async (t) => {
189 const overlay = new HyperOverlay(tmpStore())
190 await overlay.ready()
191
192 await overlay.put('/local.txt', Buffer.from('already here'))
193 const result = await overlay.hydrate('/local.txt')
194 t.is(result.fetched, 1)
195 t.is(result.total, 1)
196
197 await overlay.close()
198})
199
200test('getAvailability — fully local', async (t) => {
201 const overlay = new HyperOverlay(tmpStore())
202 await overlay.ready()
203
204 await overlay.put('/avail.txt', Buffer.from('available'))
205 const a = await overlay.getAvailability('/avail.txt')
206 t.is(a.total, 1)
207 t.is(a.local, 1)
208 t.is(a.ratio, 1)
209
210 await overlay.close()
211})
212
213test('getAvailability — missing file', async (t) => {
214 const overlay = new HyperOverlay(tmpStore())
215 await overlay.ready()
216
217 const a = await overlay.getAvailability('/nope')
218 t.is(a.total, 0)
219 t.is(a.local, 0)
220 t.is(a.ratio, 0)
221
222 await overlay.close()
223})
test/chunk-backpressure.test.js +2 -2
……@@ -16,7 +16,7 @@
1616
1717import test from 'brittle'
1818import b4a from 'b4a'
19import { OverlayProtocolV2 } from '../lib/protocol-v2.js'
19import { OverlayProtocol } from '../lib/protocol.js'
2020
2121// Minimal drain-controllable stream stub (only what _awaitDrain uses).
2222function makeStream () {
……@@ -35,7 +35,7 @@ function makeProto (n) {
3535 _fileIndex: { getChunkMap: async () => chunkMap },
3636 readChunk: (p, off, len) => b4a.alloc(len)
3737 }
38 return new OverlayProtocolV2(null, transferManager, { filePaths: new Map([['content:x', '/fake']]) })
38 return new OverlayProtocol(null, transferManager, { filePaths: new Map([['content:x', '/fake']]) })
3939}
4040
4141test('backpressure: seeder paces chunkData by drain instead of flooding', async (t) => {
test/fuse.test.js +0 -161
……@@ -1,161 +0,0 @@
1import test from 'brittle'
2import crypto from 'hypercore-crypto'
3import { tmpStore, tmpDir, fs as bareFs, path } from './helpers.js'
4import { HyperOverlay } from '../index.js'
5import { mount } from '../lib/fuse.js'
6
7const fs = bareFs.promises
8
9test('fuse mount — read small file', async (t) => {
10 const overlay = new HyperOverlay(tmpStore())
11 await overlay.ready()
12
13 const content = Buffer.from('Hello from hyper-overlay FUSE!')
14 await overlay.put('/hello.txt', content)
15
16 const mnt = tmpDir('overlay-mnt')
17 const handle = await mount(overlay, mnt, { readonly: true })
18
19 try {
20 const files = await fs.readdir(mnt)
21 t.ok(files.includes('hello.txt'), 'file visible in mount')
22
23 const data = await fs.readFile(path.join(mnt, 'hello.txt'))
24 t.alike(data, content, 'file content matches')
25
26 const s = await fs.stat(path.join(mnt, 'hello.txt'))
27 t.is(s.size, content.length, 'stat size correct')
28 t.ok(s.isFile(), 'is a file')
29 } finally {
30 await handle.unmount()
31 }
32
33 await overlay.close()
34})
35
36test('fuse mount — read large file (multi-chunk)', async (t) => {
37 const overlay = new HyperOverlay(tmpStore())
38 await overlay.ready()
39
40 const content = crypto.randomBytes(128 * 1024)
41 await overlay.put('/large.bin', content)
42
43 const mnt = tmpDir('overlay-mnt')
44 const handle = await mount(overlay, mnt, { readonly: true })
45
46 try {
47 const data = await fs.readFile(path.join(mnt, 'large.bin'))
48 t.is(data.length, content.length, 'correct size')
49 t.alike(data, content, 'content matches exactly')
50 } finally {
51 await handle.unmount()
52 }
53
54 await overlay.close()
55})
56
57test('fuse mount — readdir with subdirectories', async (t) => {
58 const overlay = new HyperOverlay(tmpStore())
59 await overlay.ready()
60
61 await overlay.put('/docs/readme.md', Buffer.from('# Hello'))
62 await overlay.put('/docs/guide.md', Buffer.from('# Guide'))
63 await overlay.put('/src/index.js', Buffer.from('export default {}'))
64
65 const mnt = tmpDir('overlay-mnt')
66 const handle = await mount(overlay, mnt, { readonly: true })
67
68 try {
69 const root = await fs.readdir(mnt)
70 t.ok(root.includes('docs'), 'docs dir visible')
71 t.ok(root.includes('src'), 'src dir visible')
72
73 const docs = await fs.readdir(path.join(mnt, 'docs'))
74 t.ok(docs.includes('readme.md'))
75 t.ok(docs.includes('guide.md'))
76
77 const rootStat = await fs.stat(mnt)
78 t.ok(rootStat.isDirectory(), 'root is directory')
79
80 const docsStat = await fs.stat(path.join(mnt, 'docs'))
81 t.ok(docsStat.isDirectory(), 'docs is directory')
82 } finally {
83 await handle.unmount()
84 }
85
86 await overlay.close()
87})
88
89test('fuse mount — write and read back', async (t) => {
90 const overlay = new HyperOverlay(tmpStore())
91 await overlay.ready()
92
93 const mnt = tmpDir('overlay-mnt')
94 const handle = await mount(overlay, mnt, { readonly: false })
95
96 try {
97 const content = 'Written through FUSE!'
98 await fs.writeFile(path.join(mnt, 'new-file.txt'), content)
99
100 // release is async — wait for overlay.put to complete
101 await new Promise(resolve => setTimeout(resolve, 100))
102
103 // Verify through overlay API
104 const data = await overlay.get('/new-file.txt')
105 t.ok(data, 'file exists in overlay')
106 t.is(data.toString(), content, 'content matches')
107
108 // Verify through FUSE read
109 const read = await fs.readFile(path.join(mnt, 'new-file.txt'), 'utf-8')
110 t.is(read, content, 'readable back through FUSE')
111 } finally {
112 await handle.unmount()
113 }
114
115 await overlay.close()
116})
117
118test('fuse mount — delete file', async (t) => {
119 const overlay = new HyperOverlay(tmpStore())
120 await overlay.ready()
121
122 await overlay.put('/deleteme.txt', Buffer.from('temp'))
123
124 const mnt = tmpDir('overlay-mnt')
125 const handle = await mount(overlay, mnt, { readonly: false })
126
127 try {
128 const before = await fs.readdir(mnt)
129 t.ok(before.includes('deleteme.txt'), 'file exists before delete')
130
131 await fs.unlink(path.join(mnt, 'deleteme.txt'))
132
133 const exists = await overlay.exists('/deleteme.txt')
134 t.is(exists, false, 'file removed from overlay')
135 } finally {
136 await handle.unmount()
137 }
138
139 await overlay.close()
140})
141
142test('fuse mount — stat reports correct size', async (t) => {
143 const overlay = new HyperOverlay(tmpStore())
144 await overlay.ready()
145
146 const content = crypto.randomBytes(50000)
147 await overlay.put('/sized.bin', content)
148
149 const mnt = tmpDir('overlay-mnt')
150 const handle = await mount(overlay, mnt, { readonly: true })
151
152 try {
153 const s = await fs.stat(path.join(mnt, 'sized.bin'))
154 t.is(s.size, 50000, 'correct file size')
155 t.ok(s.isFile(), 'is a file')
156 } finally {
157 await handle.unmount()
158 }
159
160 await overlay.close()
161})
test/get-by-hash.test.js +0 -94
……@@ -1,94 +0,0 @@
1import test from 'brittle'
2import Protomux from 'protomux'
3import { Duplex } from 'streamx'
4import crypto from 'hypercore-crypto'
5import { tmpStore } from './helpers.js'
6import { HyperOverlay } from '../index.js'
7
8// Content-addressed P2P blob fetch — ask a peer for a chunk by its content
9// hash without going through a manifest. Covers the profile-picture and
10// small-attachment use cases in pear-git-v3 where path === /attachments/<hash>
11// and the single-chunk file has chunk.hash === fileHash.
12
13function makeDuplex () {
14 let aWrite, bWrite
15 const a = new Duplex({ write (data, cb) { bWrite(data); cb() }, read () {} })
16 const b = new Duplex({ write (data, cb) { aWrite(data); cb() }, read () {} })
17 aWrite = (data) => a.push(data)
18 bWrite = (data) => b.push(data)
19 return [a, b]
20}
21
22test('getByHash fetches a small blob over mux and caches it locally', async (t) => {
23 const overlayA = new HyperOverlay(tmpStore())
24 const overlayB = new HyperOverlay(tmpStore())
25 await overlayA.ready()
26 await overlayB.ready()
27
28 // Small file — single inline chunk whose hash equals the file hash
29 const data = crypto.randomBytes(32 * 1024) // 32KB, representative of a pfp
30 const { oid: fileHash } = await overlayA.put('/attachments/x.bin', data)
31
32 // B has never heard of this blob
33 t.is(await overlayB._registry.getChunkData(fileHash), null, 'B has no local chunk')
34
35 // Connect via Protomux
36 const [streamA, streamB] = makeDuplex()
37 const muxA = Protomux.from(streamA)
38 const muxB = Protomux.from(streamB)
39
40 overlayA.attachProtocol(muxA)
41 overlayB.attachProtocol(muxB)
42
43 // Allow channel open handshake
44 await new Promise(resolve => setTimeout(resolve, 100))
45
46 const retrieved = await overlayB.getByHash(fileHash)
47 t.ok(retrieved, 'getByHash returned bytes')
48 t.alike(retrieved, data, 'content matches original')
49
50 // Second call must be local — bytes should have been cached after the fetch
51 const cached = await overlayB._registry.getChunkData(fileHash)
52 t.ok(cached, 'B cached the chunk locally')
53 t.alike(cached, data, 'cached bytes match')
54
55 await overlayA.close()
56 await overlayB.close()
57})
58
59test('getByHash returns null when no peer has the chunk', async (t) => {
60 const overlayA = new HyperOverlay(tmpStore())
61 const overlayB = new HyperOverlay(tmpStore())
62 await overlayA.ready()
63 await overlayB.ready()
64
65 const [streamA, streamB] = makeDuplex()
66 const muxA = Protomux.from(streamA)
67 const muxB = Protomux.from(streamB)
68
69 overlayA.attachProtocol(muxA)
70 overlayB.attachProtocol(muxB)
71
72 await new Promise(resolve => setTimeout(resolve, 100))
73
74 const missing = await overlayB.getByHash('a'.repeat(64))
75 t.is(missing, null, 'no peer has this hash — returns null')
76
77 await overlayA.close()
78 await overlayB.close()
79})
80
81test('getByHash returns the chunk without peers when we already have it locally', async (t) => {
82 const overlay = new HyperOverlay(tmpStore())
83 await overlay.ready()
84
85 const data = crypto.randomBytes(16 * 1024)
86 const { oid: fileHash } = await overlay.put('/local.bin', data)
87
88 // No protocol attached, no peers — purely local lookup
89 const retrieved = await overlay.getByHash(fileHash)
90 t.ok(retrieved, 'local fetch succeeded with no peers')
91 t.alike(retrieved, data, 'content matches')
92
93 await overlay.close()
94})
test/http.test.js +0 -146
……@@ -1,146 +0,0 @@
1import test from 'brittle'
2import http from 'bare-http1'
3import crypto from 'hypercore-crypto'
4import { tmpStore, tmpDir } from './helpers.js'
5import { HyperOverlay } from '../index.js'
6
7function request (method, url, body) {
8 return new Promise((resolve, reject) => {
9 const u = new URL(url)
10 const headers = {}
11 if (body) headers['content-length'] = body.length
12 const opts = { method, hostname: u.hostname, port: u.port, path: u.pathname, headers }
13 const req = http.request(opts, (res) => {
14 const parts = []
15 res.on('data', d => parts.push(d))
16 res.on('end', () => {
17 const buf = Buffer.concat(parts)
18 resolve({ statusCode: res.statusCode, headers: res.headers, body: buf })
19 })
20 })
21 req.on('error', reject)
22 if (body && body.length > 0) req.write(body)
23 req.end()
24 })
25}
26
27async function setup () {
28 const overlay = new HyperOverlay(tmpStore())
29 await overlay.ready()
30
31 const server = http.createServer()
32 // uploadSpoolDir is now required for POST /overlay/upload — bytes
33 // are stored as type:'local' pointers into this directory.
34 overlay.serveHTTP(server, '/overlay', { uploadSpoolDir: tmpDir('upload-spool') })
35
36 await new Promise(resolve => server.listen(0, resolve))
37 const port = server.address().port
38 const base = `http://127.0.0.1:${port}`
39
40 return { overlay, server, port, base, teardown: async () => { server.close(); await overlay.close() } }
41}
42
43test('POST /overlay/upload — small file', async (t) => {
44 const { base, teardown } = await setup()
45
46 const data = Buffer.from('Hello, overlay upload!')
47 const res = await request('POST', `${base}/overlay/upload/test.txt`, data)
48
49 t.is(res.statusCode, 200)
50 const json = JSON.parse(res.body.toString())
51 t.ok(json.oid, 'has oid')
52 t.is(json.size, data.length, 'correct size')
53 t.is(json.chunkCount, 1, 'single chunk')
54 t.is(json.path, '/test.txt', 'correct path')
55 t.ok(json.pointer.startsWith('hyper-overlay v1'), 'valid pointer')
56
57 await teardown()
58})
59
60test('POST /overlay/upload — large file with multiple chunks', async (t) => {
61 const { base, teardown } = await setup()
62
63 const data = crypto.randomBytes(128 * 1024) // 128 KB
64 const res = await request('POST', `${base}/overlay/upload/large.bin`, data)
65
66 t.is(res.statusCode, 200)
67 const json = JSON.parse(res.body.toString())
68 t.ok(json.chunkCount > 1, `${json.chunkCount} chunks`)
69 t.is(json.size, data.length)
70
71 await teardown()
72})
73
74test('POST /overlay/upload — empty body returns 400', async (t) => {
75 const { base, teardown } = await setup()
76
77 const res = await request('POST', `${base}/overlay/upload/empty`, Buffer.alloc(0))
78
79 t.is(res.statusCode, 400)
80 const json = JSON.parse(res.body.toString())
81 t.is(json.error, 'Empty body')
82
83 await teardown()
84})
85
86test('upload then GET chunk by hash', async (t) => {
87 const { base, overlay, teardown } = await setup()
88
89 const data = Buffer.from('round-trip test content')
90 const uploadRes = await request('POST', `${base}/overlay/upload/rt.txt`, data)
91 t.is(uploadRes.statusCode, 200)
92
93 const json = JSON.parse(uploadRes.body.toString())
94
95 // File is small enough to be a single chunk — its oid IS the chunk hash
96 const chunkRes = await request('GET', `${base}/overlay/chunks/${json.oid}`)
97 t.is(chunkRes.statusCode, 200)
98 t.alike(chunkRes.body, data, 'chunk data matches original')
99 t.is(chunkRes.headers['cache-control'], 'public, immutable, max-age=31536000')
100
101 await teardown()
102})
103
104test('GET /overlay/status', async (t) => {
105 const { base, teardown } = await setup()
106
107 const res = await request('GET', `${base}/overlay/status`)
108 t.is(res.statusCode, 200)
109 const json = JSON.parse(res.body.toString())
110 t.is(json.chunkCount, 0, 'starts empty')
111
112 await teardown()
113})
114
115test('GET /overlay/chunks/invalid returns 400', async (t) => {
116 const { base, teardown } = await setup()
117
118 const res = await request('GET', `${base}/overlay/chunks/not-a-hash`)
119 t.is(res.statusCode, 400)
120
121 await teardown()
122})
123
124test('GET /overlay/chunks/missing returns 404', async (t) => {
125 const { base, teardown } = await setup()
126
127 const hash = 'a'.repeat(64)
128 const res = await request('GET', `${base}/overlay/chunks/${hash}`)
129 t.is(res.statusCode, 404)
130
131 await teardown()
132})
133
134test('upload auto-generates path when none given', async (t) => {
135 const { base, teardown } = await setup()
136
137 const data = Buffer.from('no path specified')
138 const res = await request('POST', `${base}/overlay/upload`, data)
139
140 t.is(res.statusCode, 200)
141 const json = JSON.parse(res.body.toString())
142 t.ok(json.path, 'auto-generated path')
143 t.ok(json.oid)
144
145 await teardown()
146})
test/integration.test.js +0 -164
……@@ -1,164 +0,0 @@
1import test from 'brittle'
2import crypto from 'hypercore-crypto'
3import { tmpStore } from './helpers.js'
4import { HyperOverlay } from '../index.js'
5
6test('put and get file', async (t) => {
7 const overlay = new HyperOverlay(tmpStore())
8 await overlay.ready()
9
10 const data = Buffer.from('Hello, hyper-overlay!')
11 const result = await overlay.put('/test.txt', data)
12
13 t.ok(result.oid, 'has oid')
14 t.is(result.chunkCount, 1, 'small file = 1 chunk')
15 t.ok(result.pointer, 'has pointer')
16
17 const retrieved = await overlay.get('/test.txt')
18 t.ok(retrieved, 'file found')
19 t.alike(retrieved, data, 'content matches')
20
21 await overlay.close()
22})
23
24test('put large file — multiple chunks', async (t) => {
25 const overlay = new HyperOverlay(tmpStore())
26 await overlay.ready()
27
28 const data = crypto.randomBytes(128 * 1024) // 128KB = multiple chunks
29 const result = await overlay.put('/large.bin', data)
30
31 t.ok(result.chunkCount > 1, `${result.chunkCount} chunks`)
32
33 const retrieved = await overlay.get('/large.bin')
34 t.ok(retrieved, 'file found')
35 t.is(retrieved.length, data.length, 'same size')
36 t.alike(retrieved, data, 'content matches exactly')
37
38 await overlay.close()
39})
40
41test('dedup — same file produces same chunks', async (t) => {
42 const overlay = new HyperOverlay(tmpStore())
43 await overlay.ready()
44
45 const data = crypto.randomBytes(64 * 1024)
46 const result1 = await overlay.put('/file-a.bin', data)
47 const result2 = await overlay.put('/file-b.bin', data)
48
49 t.is(result1.oid, result2.oid, 'same content hash')
50 t.is(result1.chunkCount, result2.chunkCount, 'same chunk count')
51
52 // Both files retrievable
53 const a = await overlay.get('/file-a.bin')
54 const b = await overlay.get('/file-b.bin')
55 t.alike(a, data)
56 t.alike(b, data)
57
58 // Stats: chunks should not be doubled (dedup)
59 const stats = await overlay.stats()
60 t.is(stats.chunkCount, result1.chunkCount, 'chunks stored once, not twice')
61
62 await overlay.close()
63})
64
65test('list files', async (t) => {
66 const overlay = new HyperOverlay(tmpStore())
67 await overlay.ready()
68
69 await overlay.put('/docs/readme.md', Buffer.from('# Hello'))
70 await overlay.put('/docs/guide.md', Buffer.from('# Guide'))
71 await overlay.put('/src/index.js', Buffer.from('export default {}'))
72
73 const all = await overlay.list()
74 t.is(all.length, 3)
75
76 const docs = await overlay.list('/docs/')
77 t.is(docs.length, 2)
78
79 await overlay.close()
80})
81
82test('entry returns metadata without content', async (t) => {
83 const overlay = new HyperOverlay(tmpStore())
84 await overlay.ready()
85
86 await overlay.put('/file.txt', Buffer.from('content'))
87 const entry = await overlay.entry('/file.txt')
88 t.ok(entry)
89 t.is(entry.size, 7)
90 t.ok(entry.contentHash)
91 t.ok(entry.chunkCount)
92
93 await overlay.close()
94})
95
96test('exists returns boolean', async (t) => {
97 const overlay = new HyperOverlay(tmpStore())
98 await overlay.ready()
99
100 t.is(await overlay.exists('/nope'), false)
101 await overlay.put('/yes', Buffer.from('x'))
102 t.is(await overlay.exists('/yes'), true)
103
104 await overlay.close()
105})
106
107test('del removes file', async (t) => {
108 const overlay = new HyperOverlay(tmpStore())
109 await overlay.ready()
110
111 await overlay.put('/temp.txt', Buffer.from('temporary'))
112 t.is(await overlay.exists('/temp.txt'), true)
113
114 await overlay.del('/temp.txt')
115 t.is(await overlay.exists('/temp.txt'), false)
116
117 await overlay.close()
118})
119
120test('get returns null for missing file', async (t) => {
121 const overlay = new HyperOverlay(tmpStore())
122 await overlay.ready()
123
124 const result = await overlay.get('/missing')
125 t.is(result, null)
126
127 await overlay.close()
128})
129
130test('pointer format for small file', async (t) => {
131 const overlay = new HyperOverlay(tmpStore())
132 await overlay.ready()
133
134 const data = Buffer.from('small file content')
135 const result = await overlay.put('/small.txt', data)
136
137 t.ok(result.pointer.startsWith('hyper-overlay v1\n'), 'valid pointer header')
138 t.ok(result.pointer.includes('oid blake2b:'), 'has oid')
139 t.ok(result.pointer.includes(`size ${data.length}`), 'has size')
140 t.ok(result.pointer.includes('chunks 1'), 'has chunk count')
141
142 await overlay.close()
143})
144
145test('large file uses manifest', async (t) => {
146 // Create a file large enough for >64 chunks
147 // At ~16KB average: 64 chunks ≈ 1MB. Use 2MB to be safe.
148 const overlay = new HyperOverlay(tmpStore())
149 await overlay.ready()
150
151 const data = crypto.randomBytes(2 * 1024 * 1024) // 2MB
152 const result = await overlay.put('/huge.bin', data)
153
154 t.ok(result.chunkCount > 64, `${result.chunkCount} chunks (>64 = manifest)`)
155 t.ok(result.pointer.includes('manifest blake2b:'), 'pointer has manifest reference')
156
157 // Verify round-trip with manifest
158 const retrieved = await overlay.get('/huge.bin')
159 t.ok(retrieved, 'file retrieved via manifest')
160 t.is(retrieved.length, data.length, 'size matches')
161 t.alike(retrieved, data, 'content matches')
162
163 await overlay.close()
164})
test/messages-v2.test.js +1 -1
……@@ -1,6 +1,6 @@
11import test from 'brittle'
22import c from 'compact-encoding'
3import * as m from '../lib/messages-v2.js'
3import * as m from '../lib/messages.js'
44import crypto from 'hypercore-crypto'
55
66function roundTrip (t, codec, value) {
test/messages.test.js +0 -111
……@@ -1,111 +0,0 @@
1import test from 'brittle'
2import c from 'compact-encoding'
3import * as m from '../lib/messages.js'
4
5function roundTrip (t, codec, value, label) {
6 const state = { buffer: null, start: 0, end: 0 }
7 codec.preencode(state, value)
8 state.buffer = Buffer.alloc(state.end)
9 state.start = 0
10 codec.encode(state, value)
11 state.start = 0
12 const decoded = codec.decode(state)
13 t.alike(decoded, value, label || 'round-trip')
14}
15
16const HASH = 'a'.repeat(64)
17const HASH2 = 'b'.repeat(64)
18
19test('handshake round-trip', (t) => {
20 roundTrip(t, m.handshake, { version: 1, manifestCount: 42, capabilities: 0x01 })
21})
22
23test('manifestHave round-trip', (t) => {
24 roundTrip(t, m.manifestHave, { hashes: [HASH, HASH2] })
25})
26
27test('manifestHave empty', (t) => {
28 roundTrip(t, m.manifestHave, { hashes: [] })
29})
30
31test('manifestRequest round-trip', (t) => {
32 roundTrip(t, m.manifestRequest, { hash: HASH, requestId: 7 })
33})
34
35test('manifestResponse round-trip (found)', (t) => {
36 roundTrip(t, m.manifestResponse, {
37 hash: HASH,
38 requestId: 7,
39 data: Buffer.from('compressed manifest data'),
40 found: true
41 })
42})
43
44test('manifestResponse round-trip (not found)', (t) => {
45 roundTrip(t, m.manifestResponse, {
46 hash: HASH,
47 requestId: 8,
48 data: null, // compact-encoding decodes empty buffer as null
49 found: false
50 })
51})
52
53test('chunkBitfield round-trip', (t) => {
54 roundTrip(t, m.chunkBitfield, {
55 manifestHash: HASH,
56 bitfield: Buffer.from([0xff, 0x0f, 0x00, 0xaa])
57 })
58})
59
60test('chunkRequest round-trip', (t) => {
61 roundTrip(t, m.chunkRequest, {
62 manifestHash: HASH,
63 index: 42,
64 requestId: 100,
65 priority: 1
66 })
67})
68
69test('chunkResponse round-trip (found)', (t) => {
70 roundTrip(t, m.chunkResponse, {
71 requestId: 100,
72 data: Buffer.from('chunk bytes here'),
73 found: true
74 })
75})
76
77test('chunkResponse round-trip (not found)', (t) => {
78 roundTrip(t, m.chunkResponse, {
79 requestId: 101,
80 data: null, // compact-encoding decodes empty buffer as null
81 found: false
82 })
83})
84
85test('chunkCancel round-trip', (t) => {
86 roundTrip(t, m.chunkCancel, { requestId: 99 })
87})
88
89test('chunkHave round-trip', (t) => {
90 roundTrip(t, m.chunkHave, {
91 manifestHash: HASH,
92 startIndex: 1024,
93 bitfield: Buffer.from([0xff, 0xff])
94 })
95})
96
97test('large manifestHave (100 hashes)', (t) => {
98 const hashes = Array.from({ length: 100 }, (_, i) =>
99 i.toString(16).padStart(64, '0')
100 )
101 roundTrip(t, m.manifestHave, { hashes })
102})
103
104test('max uint values', (t) => {
105 roundTrip(t, m.chunkRequest, {
106 manifestHash: HASH,
107 index: 4294967295,
108 requestId: 4294967295,
109 priority: 255
110 })
111})
test/overlay-v2-fetch.test.js +2 -2
……@@ -3,7 +3,7 @@ import Protomux from 'protomux'
33import { Duplex } from 'streamx'
44import crypto from 'hypercore-crypto'
55import { tmpStore, tmpDir, fs, path } from './helpers.js'
6import { HyperOverlayV2 } from '../lib/overlay-v2.js'
6import { HyperOverlay } from '../lib/overlay.js'
77
88// In-memory duplex pair (same pattern as e2e-two-peers.js).
99function makeDuplex () {
……@@ -16,7 +16,7 @@ function makeDuplex () {
1616}
1717
1818async function makePeer (label) {
19 const ov = new HyperOverlayV2(tmpStore('ov2-' + label), {
19 const ov = new HyperOverlay(tmpStore('ov2-' + label), {
2020 namespace: 'overlay-v2',
2121 destDir: tmpDir('ov2-' + label + '-dest')
2222 })
test/overlay-v2-mirror-spool.test.js +5 -5
……@@ -3,7 +3,7 @@ import Protomux from 'protomux'
33import crypto from 'hypercore-crypto'
44import { Duplex } from 'streamx'
55import { tmpStore, tmpDir, fs, path } from './helpers.js'
6import { HyperOverlayV2 } from '../lib/overlay-v2.js'
6import { HyperOverlay } from '../lib/overlay.js'
77
88// Phase 2 mirror primitive: a forge backend (no working tree) replicates a
99// repo's large files by FETCHING each by oid and storing it CONTENT-ADDRESSED at
……@@ -27,11 +27,11 @@ test('mirror fetch stores the blob content-addressed at <mirrorSpool>/<oid> and
2727 const content = crypto.randomBytes(256 * 1024)
2828 const oid = crypto.data(content).toString('hex')
2929 fs.writeFileSync(path.join(pubSpool, oid), content)
30 const pub = new HyperOverlayV2(tmpStore('ms-pub'), { namespace: 'overlay-v2', syncBaseDirs: [pubSpool], destDir: tmpDir('ms-pub-dest') })
30 const pub = new HyperOverlay(tmpStore('ms-pub'), { namespace: 'overlay-v2', syncBaseDirs: [pubSpool], destDir: tmpDir('ms-pub-dest') })
3131
3232 // Mirror has its own content-addressed mirror-spool (no working tree).
3333 const mirrorSpool = tmpDir('ms-mirror-spool')
34 const mir = new HyperOverlayV2(tmpStore('ms-mir'), { namespace: 'overlay-v2', syncBaseDirs: [mirrorSpool], destDir: tmpDir('ms-mir-dest') })
34 const mir = new HyperOverlay(tmpStore('ms-mir'), { namespace: 'overlay-v2', syncBaseDirs: [mirrorSpool], destDir: tmpDir('ms-mir-dest') })
3535
3636 await pub.ready(); await mir.ready()
3737 t.teardown(async () => { try { await pub.close() } catch {}; try { await mir.close() } catch {} })
……@@ -53,8 +53,8 @@ test('mirror fetch stores the blob content-addressed at <mirrorSpool>/<oid> and
5353
5454 // The mirror is now a SECOND source: a cold third node (only the mirror's
5555 // spool, empty maps) re-serves it to a fresh consumer.
56 const reserver = new HyperOverlayV2(tmpStore('ms-re'), { namespace: 'overlay-v2', syncBaseDirs: [mirrorSpool], destDir: tmpDir('ms-re-dest') })
57 const con2 = new HyperOverlayV2(tmpStore('ms-con2'), { namespace: 'overlay-v2', destDir: tmpDir('ms-con2-dest') })
56 const reserver = new HyperOverlay(tmpStore('ms-re'), { namespace: 'overlay-v2', syncBaseDirs: [mirrorSpool], destDir: tmpDir('ms-re-dest') })
57 const con2 = new HyperOverlay(tmpStore('ms-con2'), { namespace: 'overlay-v2', destDir: tmpDir('ms-con2-dest') })
5858 await reserver.ready(); await con2.ready()
5959 t.teardown(async () => { try { await reserver.close() } catch {}; try { await con2.close() } catch {} })
6060 const [sc, sd] = makeDuplex()
test/overlay-v2-restart-durability.test.js +3 -3
……@@ -3,7 +3,7 @@ import Protomux from 'protomux'
33import crypto from 'hypercore-crypto'
44import { Duplex } from 'streamx'
55import { tmpStore, tmpDir, fs, path } from './helpers.js'
6import { HyperOverlayV2 } from '../lib/overlay-v2.js'
6import { HyperOverlay } from '../lib/overlay.js'
77
88// Phase 2: a forge serves hyper-tps' large files (NOT stored in git) by content
99// hash over hyper-overlay — the deliberate "no hyperblobs" design. The canonical
……@@ -34,8 +34,8 @@ test('RESTART DURABILITY: a cold publisher serves by content hash from the conte
3434
3535 // Cold publisher: fresh store, EMPTY in-memory maps, NEVER registerFile'd —
3636 // exactly the post-restart state (the spool persists on disk; the maps do not).
37 const pub = new HyperOverlayV2(tmpStore('ov2-rd-pub'), { namespace: 'overlay-v2', syncBaseDirs: [spool], destDir: tmpDir('ov2-rd-pub-dest') })
38 const con = new HyperOverlayV2(tmpStore('ov2-rd-con'), { namespace: 'overlay-v2', destDir: tmpDir('ov2-rd-con-dest') })
37 const pub = new HyperOverlay(tmpStore('ov2-rd-pub'), { namespace: 'overlay-v2', syncBaseDirs: [spool], destDir: tmpDir('ov2-rd-pub-dest') })
38 const con = new HyperOverlay(tmpStore('ov2-rd-con'), { namespace: 'overlay-v2', destDir: tmpDir('ov2-rd-con-dest') })
3939 await pub.ready(); await con.ready()
4040 t.teardown(async () => { try { await pub.close() } catch {}; try { await con.close() } catch {} })
4141
test/pairing.test.js +0 -147
……@@ -1,147 +0,0 @@
1import test from 'brittle'
2import createTestnet from 'hyperdht/testnet'
3import Hyperswarm from 'hyperswarm'
4import BlindPairing from 'blind-pairing'
5import crypto from 'hypercore-crypto'
6import { createInvite, listenForPair, joinWithInvite, readDevices, addDevice } from '../lib/pairing.js'
7import { tmpDir } from './helpers.js'
8
9test('blind-pairing: core primitives (no network)', async (t) => {
10 // Verify the crypto flow works before testing network
11 const { CandidateRequest, MemberRequest, createInvite: coreCreateInvite } = await import('blind-pairing-core')
12
13 const pairingKey = crypto.keyPair()
14 const { invite, publicKey: invitePublicKey, seed: inviteSeed } = coreCreateInvite(pairingKey.publicKey)
15 const inviteKeyPair = crypto.keyPair(inviteSeed)
16
17 const userData = Buffer.from('stortron', 'utf-8')
18 const candidateReq = new CandidateRequest(invite, userData)
19
20 const memberReq = MemberRequest.from(candidateReq.encode())
21 const receivedUserData = memberReq.open(invitePublicKey)
22 t.alike(receivedUserData, userData, 'member receives candidate userData')
23
24 const payload = Buffer.from(JSON.stringify({ topic: 'abc123' }), 'utf-8')
25 memberReq.confirm({
26 key: pairingKey.publicKey,
27 encryptionKey: null,
28 additional: {
29 data: payload,
30 signature: crypto.sign(payload, inviteKeyPair.secretKey)
31 }
32 })
33
34 const auth = candidateReq.handleResponse(memberReq.response)
35 t.ok(auth, 'candidate received auth')
36 t.alike(auth.data, payload, 'payload transferred correctly')
37})
38
39test('blind-pairing: two swarms over testnet DHT', async (t) => {
40 t.timeout(30000)
41
42 const testnet = await createTestnet(10, t.teardown)
43 const bootstrap = testnet.bootstrap
44
45 const memberSwarm = new Hyperswarm({ bootstrap })
46 const candidateSwarm = new Hyperswarm({ bootstrap })
47 t.teardown(() => memberSwarm.destroy())
48 t.teardown(() => candidateSwarm.destroy())
49
50 // Member side: create invite
51 const pairingKey = crypto.keyPair()
52 const inviteResult = BlindPairing.createInvite(pairingKey.publicKey)
53 const { invite, publicKey: invitePublicKey, discoveryKey, seed: inviteSeed } = inviteResult
54 const inviteKeyPair = crypto.keyPair(inviteSeed)
55
56 const sharedTopic = crypto.randomBytes(32)
57 const payload = Buffer.from(JSON.stringify({ topic: sharedTopic.toString('hex') }), 'utf-8')
58
59 // Member listens
60 const memberBP = new BlindPairing(memberSwarm)
61 t.teardown(() => memberBP.close())
62
63 const memberGotCandidate = new Promise((resolve, reject) => {
64 const timeout = setTimeout(() => reject(new Error('member timeout')), 20000)
65 memberBP.addMember({
66 discoveryKey,
67 onadd (request) {
68 clearTimeout(timeout)
69 const userData = request.open(invitePublicKey)
70 request.confirm({
71 key: pairingKey.publicKey,
72 encryptionKey: null,
73 additional: {
74 data: payload,
75 signature: crypto.sign(payload, inviteKeyPair.secretKey)
76 }
77 })
78 resolve(userData ? userData.toString('utf-8') : null)
79 }
80 })
81 })
82
83 // Candidate joins
84 const candidateBP = new BlindPairing(candidateSwarm)
85 t.teardown(() => candidateBP.close())
86
87 const candidateGotAuth = new Promise((resolve, reject) => {
88 const timeout = setTimeout(() => reject(new Error('candidate timeout')), 20000)
89 candidateBP.addCandidate({
90 invite,
91 userData: Buffer.from('stortron', 'utf-8'),
92 onadd (auth) {
93 clearTimeout(timeout)
94 resolve(auth)
95 }
96 })
97 })
98
99 // Both should complete
100 const [memberResult, candidateResult] = await Promise.all([memberGotCandidate, candidateGotAuth])
101
102 t.is(memberResult, 'stortron', 'member received candidate name')
103 t.ok(candidateResult.data, 'candidate received auth data')
104
105 const received = JSON.parse(candidateResult.data.toString('utf-8'))
106 t.is(received.topic, sharedTopic.toString('hex'), 'topic transferred correctly')
107})
108
109test('blind-pairing: hyper-overlay pairing module over testnet', async (t) => {
110 t.timeout(30000)
111
112 const testnet = await createTestnet(10, t.teardown)
113 const bootstrap = testnet.bootstrap
114
115 const memberSwarm = new Hyperswarm({ bootstrap })
116 const candidateSwarm = new Hyperswarm({ bootstrap })
117 t.teardown(() => memberSwarm.destroy())
118 t.teardown(() => candidateSwarm.destroy())
119
120 const memberStorage = tmpDir('pair-member')
121 const candidateStorage = tmpDir('pair-candidate')
122
123 // Member creates invite
124 const inviteData = createInvite()
125 t.ok(inviteData.inviteHex, 'invite generated')
126
127 // Member listens
128 const { peerConnected, bp: memberBP } = listenForPair(memberSwarm, memberStorage, inviteData)
129 t.teardown(() => memberBP.close())
130
131 // Candidate joins
132 const joinResult = joinWithInvite(candidateSwarm, candidateStorage, inviteData.inviteHex, 'stortron')
133
134 // Both should complete
135 const [memberResult, candidateJoin] = await Promise.all([peerConnected, joinResult])
136
137 t.is(memberResult.peerName, 'stortron', 'member sees candidate name')
138 t.ok(candidateJoin.sharedTopic, 'candidate received topic')
139 t.alike(candidateJoin.sharedTopic, inviteData.sharedTopic, 'topics match')
140
141 // Both should have devices saved
142 const memberDevices = readDevices(memberStorage)
143 const candidateDevices = readDevices(candidateStorage)
144 t.is(memberDevices.length, 1, 'member saved device')
145 t.is(candidateDevices.length, 1, 'candidate saved device')
146 t.is(memberDevices[0].key, candidateDevices[0].key, 'same topic key saved')
147})
test/protocol-peer.test.js +0 -164
……@@ -1,164 +0,0 @@
1import test from 'brittle'
2import Protomux from 'protomux'
3import { Duplex } from 'streamx'
4import crypto from 'hypercore-crypto'
5import { tmpStore } from './helpers.js'
6import { HyperOverlay } from '../index.js'
7
8function makeDuplex () {
9 let aWrite, bWrite
10 const a = new Duplex({
11 write (data, cb) { bWrite(data); cb() },
12 read () {}
13 })
14 const b = new Duplex({
15 write (data, cb) { aWrite(data); cb() },
16 read () {}
17 })
18 aWrite = (data) => a.push(data)
19 bWrite = (data) => b.push(data)
20
21 return [a, b]
22}
23
24test('two peers — put on A, fetch manifest on B', async (t) => {
25 const overlayA = new HyperOverlay(tmpStore())
26 const overlayB = new HyperOverlay(tmpStore())
27 await overlayA.ready()
28 await overlayB.ready()
29
30 // Put a large file on A (>64 chunks → manifest)
31 const data = crypto.randomBytes(2 * 1024 * 1024) // 2MB
32 const result = await overlayA.put('/big.bin', data)
33
34 t.ok(result.chunkCount > 64, `${result.chunkCount} chunks`)
35 t.ok(result.pointer.includes('manifest blake2b:'), 'has manifest')
36
37 // Connect via Protomux
38 const [streamA, streamB] = makeDuplex()
39 const muxA = Protomux.from(streamA)
40 const muxB = Protomux.from(streamB)
41
42 overlayA.attachProtocol(muxA)
43 overlayB.attachProtocol(muxB)
44
45 // Wait for protocol open and manifest exchange
46 await new Promise(resolve => setTimeout(resolve, 200))
47
48 // B should know about A's manifests
49 const peersB = overlayB._protocol._peers
50 t.is(peersB.size, 1, 'B has 1 peer')
51
52 const peerState = peersB.values().next().value
53 t.ok(peerState.remoteManifests.size > 0, 'B knows about A manifests')
54
55 // B can request the manifest
56 const manifestHash = [...peerState.remoteManifests][0]
57 const manifestData = await overlayB._protocol.requestManifest(manifestHash)
58 t.ok(manifestData, 'manifest received')
59 t.ok(manifestData.length > 0, 'manifest has data')
60
61 await overlayA.close()
62 await overlayB.close()
63})
64
65test('two peers — full file transfer via chunks', async (t) => {
66 const overlayA = new HyperOverlay(tmpStore())
67 const overlayB = new HyperOverlay(tmpStore())
68 await overlayA.ready()
69 await overlayB.ready()
70
71 // Put a file on A that uses a manifest (>64 chunks)
72 const data = crypto.randomBytes(2 * 1024 * 1024)
73 const result = await overlayA.put('/transfer.bin', data)
74 t.ok(result.chunkCount > 64, `${result.chunkCount} chunks`)
75
76 // Connect
77 const [streamA, streamB] = makeDuplex()
78 const muxA = Protomux.from(streamA)
79 const muxB = Protomux.from(streamB)
80
81 overlayA.attachProtocol(muxA)
82 overlayB.attachProtocol(muxB)
83
84 await new Promise(resolve => setTimeout(resolve, 200))
85
86 // Copy the file entry and manifest to B so it knows what to fetch
87 const entry = await overlayA._registry._bee.get('file:/transfer.bin')
88 await overlayB._registry._bee.put('file:/transfer.bin', entry.value)
89
90 const manifestData = await overlayA._registry.getManifest(entry.value.manifestHash)
91 await overlayB._registry.putManifest(entry.value.manifestHash, manifestData)
92
93 // Now B should be able to get() the file — chunks fetched from A via protocol
94 const retrieved = await overlayB.get('/transfer.bin')
95 t.ok(retrieved, 'file retrieved on B')
96 t.is(retrieved.length, data.length, 'correct size')
97 t.alike(retrieved, data, 'content matches exactly')
98
99 await overlayA.close()
100 await overlayB.close()
101})
102
103test('two peers — small file with inline chunks', async (t) => {
104 const overlayA = new HyperOverlay(tmpStore())
105 const overlayB = new HyperOverlay(tmpStore())
106 await overlayA.ready()
107 await overlayB.ready()
108
109 // Small file — inline chunks, no manifest
110 const data = Buffer.from('Hello from peer A!')
111 await overlayA.put('/hello.txt', data)
112
113 // Connect
114 const [streamA, streamB] = makeDuplex()
115 const muxA = Protomux.from(streamA)
116 const muxB = Protomux.from(streamB)
117
118 overlayA.attachProtocol(muxA)
119 overlayB.attachProtocol(muxB)
120
121 await new Promise(resolve => setTimeout(resolve, 100))
122
123 // Copy file entry to B (has inline chunks)
124 const entry = await overlayA._registry._bee.get('file:/hello.txt')
125 await overlayB._registry._bee.put('file:/hello.txt', entry.value)
126
127 // Also copy chunk data (inline files have no manifest for P2P, so copy chunks directly)
128 for (const c of entry.value.chunks) {
129 const chunkEntry = await overlayA._registry._bee.get(`chunk:${c.hash}`)
130 if (chunkEntry) await overlayB._registry._bee.put(`chunk:${c.hash}`, chunkEntry.value)
131 }
132
133 const retrieved = await overlayB.get('/hello.txt')
134 t.ok(retrieved, 'file retrieved')
135 t.alike(retrieved, data, 'content matches')
136
137 await overlayA.close()
138 await overlayB.close()
139})
140
141test('chunk request for invalid manifest returns null', async (t) => {
142 const overlayA = new HyperOverlay(tmpStore())
143 await overlayA.ready()
144
145 const overlayB = new HyperOverlay(tmpStore())
146 await overlayB.ready()
147
148 const [streamA, streamB] = makeDuplex()
149 const muxA = Protomux.from(streamA)
150 const muxB = Protomux.from(streamB)
151
152 overlayA.attachProtocol(muxA)
153 overlayB.attachProtocol(muxB)
154
155 await new Promise(resolve => setTimeout(resolve, 100))
156
157 // Request chunk from non-existent manifest — should return null
158 const result = await overlayB._protocol.requestChunk('a'.repeat(64), 0)
159 t.is(result, null, 'no peer has this manifest')
160
161 await overlayA.close()
162 await overlayB.close()
163})
164
test/putfile-readback.test.js +0 -56
……@@ -1,56 +0,0 @@
1import test from 'brittle'
2import crypto from 'hypercore-crypto'
3import { tmpStore, tmpDir, fs, path } from './helpers.js'
4import { HyperOverlay } from '../index.js'
5
6// putFile stores chunks as { type:'local', path, offset } pointers — bytes
7// stay on the source disk, only metadata enters the corestore. Reads must
8// resolve those pointers back to the on-disk bytes. Without this round-trip
9// working, putFile is write-only and unsafe to wire to a serving endpoint.
10
11test('putFile stores type:local chunks; get() reads them back byte-equal', async (t) => {
12 const dir = tmpDir('putfile-readback')
13 const srcPath = path.join(dir, 'src.bin')
14 const data = crypto.randomBytes(2 * 1024 * 1024) // 2 MB → multi-chunk
15 fs.writeFileSync(srcPath, data)
16
17 const overlay = new HyperOverlay(tmpStore())
18 await overlay.ready()
19
20 const { oid, size, chunkCount } = await overlay.putFile('/test.bin', srcPath)
21 t.is(size, data.length)
22 t.ok(chunkCount > 1, 'expected multi-chunk file (chunkCount=' + chunkCount + ')')
23
24 const out = await overlay.get('/test.bin')
25 t.ok(out, 'overlay.get returned data')
26 t.is(out.length, data.length)
27 t.ok(out.equals(data), 'bytes are equal to source')
28
29 // Sanity: oid matches what hashChunk(buffer) would produce on the same
30 // input — exercised via the put/putStream/putFile equivalence in the bench
31 // and existing tests, just confirming we got something reasonable.
32 t.ok(/^[0-9a-f]{64}$/.test(oid))
33
34 await overlay.close()
35})
36
37test('putFile read returns null when source file is missing', async (t) => {
38 const dir = tmpDir('putfile-missing')
39 const srcPath = path.join(dir, 'src.bin')
40 const data = crypto.randomBytes(64 * 1024)
41 fs.writeFileSync(srcPath, data)
42
43 const overlay = new HyperOverlay(tmpStore())
44 await overlay.ready()
45
46 await overlay.putFile('/test.bin', srcPath)
47
48 // Move the source out from under the registry; reads should now fail
49 // gracefully (return null) rather than throw.
50 fs.unlinkSync(srcPath)
51
52 const out = await overlay.get('/test.bin')
53 t.is(out, null, 'reads return null when source bytes are gone')
54
55 await overlay.close()
56})
test/registry.test.js +0 -151
……@@ -1,151 +0,0 @@
1import test from 'brittle'
2import { tmpStore } from './helpers.js'
3import { ChunkRegistry } from '../lib/chunk-registry.js'
4
5test('putChunk + getChunk', async (t) => {
6 const store = tmpStore()
7 const registry = new ChunkRegistry(store)
8 await registry.ready()
9
10 await registry.putChunk('abc123', { size: 12288, type: 'local', path: '/tmp/file.bin', offset: 0 })
11
12 const chunk = await registry.getChunk('abc123')
13 t.ok(chunk)
14 t.is(chunk.size, 12288)
15 t.is(chunk.sources.length, 1)
16 t.is(chunk.sources[0].type, 'local')
17 t.is(chunk.sources[0].path, '/tmp/file.bin')
18
19 await registry.close()
20})
21
22test('hasChunk returns true/false', async (t) => {
23 const store = tmpStore()
24 const registry = new ChunkRegistry(store)
25 await registry.ready()
26
27 t.is(await registry.hasChunk('nonexistent'), false)
28 await registry.putChunk('exists', { size: 100, type: 'local' })
29 t.is(await registry.hasChunk('exists'), true)
30
31 await registry.close()
32})
33
34test('putChunk adds source without duplicating', async (t) => {
35 const store = tmpStore()
36 const registry = new ChunkRegistry(store)
37 await registry.ready()
38
39 await registry.putChunk('hash1', { size: 100, type: 'local', path: '/a', offset: 0 })
40 await registry.putChunk('hash1', { size: 100, type: 'local', path: '/a', offset: 0 }) // duplicate
41 await registry.putChunk('hash1', { size: 100, type: 'peer', path: undefined, offset: undefined })
42
43 const chunk = await registry.getChunk('hash1')
44 t.is(chunk.sources.length, 2, 'duplicate not added, new source added')
45
46 await registry.close()
47})
48
49test('cross-mode dedup: redundant cache copy not stored when local exists', async (t) => {
50 const store = tmpStore()
51 const registry = new ChunkRegistry(store)
52 await registry.ready()
53
54 await registry.putChunk('h', { size: 5, type: 'local', path: '/x', offset: 0 })
55 // Peer-fetch path would inline-cache the same bytes as base64 — must be a no-op.
56 await registry.putChunk('h', { size: 5, type: 'cache', data: Buffer.from('hello').toString('base64') })
57
58 const chunk = await registry.getChunk('h')
59 t.is(chunk.sources.length, 1, 'redundant cache copy not stored')
60 t.is(chunk.sources[0].type, 'local', 'kept the working-tree pointer')
61
62 await registry.close()
63})
64
65test('cross-mode dedup: local pointer supersedes an existing cache copy', async (t) => {
66 const store = tmpStore()
67 const registry = new ChunkRegistry(store)
68 await registry.ready()
69
70 await registry.putChunk('h', { size: 5, type: 'cache', data: Buffer.from('hello').toString('base64') })
71 t.is((await registry.getChunk('h')).sources.length, 1, 'cache copy present')
72
73 await registry.putChunk('h', { size: 5, type: 'local', path: '/x', offset: 0 })
74
75 const chunk = await registry.getChunk('h')
76 t.is(chunk.sources.length, 1, 'cache copy dropped in favor of the pointer')
77 t.is(chunk.sources[0].type, 'local', 'pointer is the surviving source')
78
79 await registry.close()
80})
81
82test('peer routing hints coexist with a byte source', async (t) => {
83 const store = tmpStore()
84 const registry = new ChunkRegistry(store)
85 await registry.ready()
86
87 await registry.putChunk('h', { size: 5, type: 'local', path: '/x', offset: 0 })
88 await registry.putChunk('h', { size: 5, type: 'peer' })
89
90 const chunk = await registry.getChunk('h')
91 t.is(chunk.sources.length, 2, 'peer hint carries no bytes, so it is kept')
92
93 await registry.close()
94})
95
96test('deleteChunk removes chunk', async (t) => {
97 const store = tmpStore()
98 const registry = new ChunkRegistry(store)
99 await registry.ready()
100
101 await registry.putChunk('todelete', { size: 50, type: 'local' })
102 t.is(await registry.hasChunk('todelete'), true)
103
104 await registry.deleteChunk('todelete')
105 t.is(await registry.hasChunk('todelete'), false)
106
107 await registry.close()
108})
109
110test('putManifest + getManifest', async (t) => {
111 const store = tmpStore()
112 const registry = new ChunkRegistry(store)
113 await registry.ready()
114
115 const data = Buffer.from('compressed manifest data here')
116 await registry.putManifest('mhash', data)
117
118 const result = await registry.getManifest('mhash')
119 t.ok(result)
120 t.ok(Buffer.isBuffer(result))
121 t.alike(result, data)
122
123 await registry.close()
124})
125
126test('getManifest returns null for unknown hash', async (t) => {
127 const store = tmpStore()
128 const registry = new ChunkRegistry(store)
129 await registry.ready()
130
131 t.is(await registry.getManifest('unknown'), null)
132
133 await registry.close()
134})
135
136test('stats returns correct counts', async (t) => {
137 const store = tmpStore()
138 const registry = new ChunkRegistry(store)
139 await registry.ready()
140
141 await registry.putChunk('c1', { size: 100, type: 'local' })
142 await registry.putChunk('c2', { size: 200, type: 'local' })
143 await registry.putManifest('m1', Buffer.from('data'))
144
145 const stats = await registry.stats()
146 t.is(stats.chunkCount, 2)
147 t.is(stats.manifestCount, 1)
148 t.is(stats.totalSize, 300)
149
150 await registry.close()
151})
test/spool-cache.test.js +0 -63
……@@ -1,63 +0,0 @@
1import test from 'brittle'
2import { tmpStore, tmpDir, fs, path } from './helpers.js'
3import { HyperOverlay } from '../index.js'
4
5test('peer-fetch spool: stores fetched chunk as a local pointer, not base64', async (t) => {
6 const spoolDir = tmpDir('ov-spool')
7 const overlay = new HyperOverlay(tmpStore(), { chunkSpoolDir: spoolDir })
8 await overlay.ready()
9
10 const hash = 'deadbeefchunkhash'
11 const bytes = Buffer.from('peer-fetched chunk bytes')
12 await overlay._cacheFetchedChunk(hash, bytes)
13
14 // Bytes written to a content-addressed spool file.
15 const dest = path.join(spoolDir, hash)
16 t.ok(fs.existsSync(dest), 'chunk spooled to <spoolDir>/<hash>')
17 t.alike(fs.readFileSync(dest), bytes, 'spool file holds the raw bytes')
18
19 // Registered as a local pointer — no base64 inline cache.
20 const rec = await overlay.registry.getChunk(hash)
21 t.is(rec.sources.length, 1, 'single source')
22 t.is(rec.sources[0].type, 'local', 'registered as a local pointer')
23 t.is(rec.sources[0].path, dest, 'pointer targets the spool file')
24
25 // Reads back through the pointer.
26 t.alike(await overlay.registry.getChunkData(hash), bytes, 'getChunkData reads bytes from spool')
27
28 await overlay.close()
29 fs.rmSync(spoolDir, { recursive: true, force: true })
30})
31
32test('peer-fetch spool: idempotent on repeat fetch of the same chunk', async (t) => {
33 const spoolDir = tmpDir('ov-spool')
34 const overlay = new HyperOverlay(tmpStore(), { chunkSpoolDir: spoolDir })
35 await overlay.ready()
36
37 const hash = 'repeatchunkhash'
38 const bytes = Buffer.from('same bytes twice')
39 await overlay._cacheFetchedChunk(hash, bytes)
40 await overlay._cacheFetchedChunk(hash, bytes)
41
42 const rec = await overlay.registry.getChunk(hash)
43 t.is(rec.sources.length, 1, 'no duplicate source on repeat')
44 t.alike(await overlay.registry.getChunkData(hash), bytes)
45
46 await overlay.close()
47 fs.rmSync(spoolDir, { recursive: true, force: true })
48})
49
50test('no spool dir: falls back to inline base64 cache (legacy default)', async (t) => {
51 const overlay = new HyperOverlay(tmpStore())
52 await overlay.ready()
53
54 const hash = 'inlinechunkhash'
55 const bytes = Buffer.from('inline bytes')
56 await overlay._cacheFetchedChunk(hash, bytes)
57
58 const rec = await overlay.registry.getChunk(hash)
59 t.is(rec.sources[0].type, 'cache', 'default still inline-caches (unchanged behavior)')
60 t.alike(await overlay.registry.getChunkData(hash), bytes, 'inline bytes read back')
61
62 await overlay.close()
63})
test/tree-peer.test.js +4 -4
……@@ -1,7 +1,7 @@
11/**
22 * Peer-to-peer tests for the 0.5a tree layer.
33 *
4 * Wires two OverlayProtocolV2 instances together via a Protomux over a
4 * Wires two OverlayProtocol instances together via a Protomux over a
55 * duplex pipe, then exercises requestTree end-to-end.
66 */
77
……@@ -10,7 +10,7 @@ import Protomux from 'protomux'
1010import { Duplex } from 'streamx'
1111import { tmpStore } from './helpers.js'
1212import { FileIndex } from '../lib/file-index.js'
13import { OverlayProtocolV2 } from '../lib/protocol-v2.js'
13import { OverlayProtocol } from '../lib/protocol.js'
1414import { computeTreeHash, KIND_FILE, KIND_DIR } from '../lib/tree.js'
1515
1616function makeDuplex () {
……@@ -23,7 +23,7 @@ function makeDuplex () {
2323}
2424
2525/**
26 * Minimal stub syncEngine — OverlayProtocolV2 only needs _fileIndex,
26 * Minimal stub syncEngine — OverlayProtocol only needs _fileIndex,
2727 * feedKey, feed (for _onOpen sync-state exchange), markSynced, logPut.
2828 */
2929function stubSyncEngine (fileIndex) {
……@@ -56,7 +56,7 @@ function stubTransferManager (fileIndex) {
5656async function makePeer (label) {
5757 const fileIndex = new FileIndex(tmpStore(label))
5858 await fileIndex.ready()
59 const protocol = new OverlayProtocolV2(
59 const protocol = new OverlayProtocol(
6060 stubSyncEngine(fileIndex),
6161 stubTransferManager(fileIndex),
6262 { key: Buffer.alloc(32, 0xab), autoSync: false }
test/watch-manager.test.js +0 -210
……@@ -1,210 +0,0 @@
1import test from 'brittle'
2import { tmpDir, fs, path } from './helpers.js'
3import { WatchManager } from '../lib/watch-manager.js'
4
5// ── Periodic scan tests (synchronous, no inotify dependency) ──
6
7test('periodic scan — detects new files', async (t) => {
8 const dir = tmpDir('watch')
9 fs.writeFileSync(path.join(dir, 'a.txt'), 'hello')
10
11 const wm = new WatchManager({ scanInterval: 50 })
12 await wm.ready()
13
14 const events = []
15 wm.onChange((e) => events.push(e))
16
17 wm.watchPeriodic(dir, 50)
18
19 // Initial scan should detect a.txt
20 t.is(events.length, 1)
21 t.is(events[0].type, 'update')
22 t.ok(events[0].path.endsWith('a.txt'))
23
24 await wm.close()
25})
26
27test('periodic scan — detects modified files', async (t) => {
28 const dir = tmpDir('watch')
29 fs.writeFileSync(path.join(dir, 'mod.txt'), 'v1')
30
31 const wm = new WatchManager({ scanInterval: 100000 }) // won't auto-fire
32 await wm.ready()
33
34 const events = []
35 wm.onChange((e) => events.push(e))
36
37 wm.watchPeriodic(dir)
38
39 // Initial: 1 event (new file)
40 t.is(events.length, 1)
41
42 // Modify file with a newer mtime
43 const filePath = path.join(dir, 'mod.txt')
44 // Ensure mtime changes by setting it to the future
45 fs.writeFileSync(filePath, 'v2')
46 const future = new Date(Date.now() + 10000)
47 fs.utimesSync(filePath, future, future)
48
49 // Manually trigger a scan
50 const state = wm._scanDirs.get(dir).state
51 wm._scanDir(dir, state)
52
53 t.is(events.length, 2)
54 t.is(events[1].type, 'update')
55
56 await wm.close()
57})
58
59test('periodic scan — detects deleted files', async (t) => {
60 const dir = tmpDir('watch')
61 const filePath = path.join(dir, 'del.txt')
62 fs.writeFileSync(filePath, 'temp')
63
64 const wm = new WatchManager({ scanInterval: 100000 })
65 await wm.ready()
66
67 const events = []
68 wm.onChange((e) => events.push(e))
69
70 wm.watchPeriodic(dir)
71 t.is(events.length, 1) // initial detect
72
73 // Delete the file
74 fs.unlinkSync(filePath)
75
76 // Manual scan
77 const state = wm._scanDirs.get(dir).state
78 wm._scanDir(dir, state)
79
80 t.is(events.length, 2)
81 t.is(events[1].type, 'delete')
82 t.ok(events[1].path.endsWith('del.txt'))
83
84 await wm.close()
85})
86
87test('periodic scan — ignores .git and node_modules', async (t) => {
88 const dir = tmpDir('watch')
89 fs.mkdirSync(path.join(dir, '.git'), { recursive: true })
90 fs.mkdirSync(path.join(dir, 'node_modules'), { recursive: true })
91 fs.writeFileSync(path.join(dir, '.git', 'config'), 'git stuff')
92 fs.writeFileSync(path.join(dir, 'node_modules', 'pkg.json'), '{}')
93 fs.writeFileSync(path.join(dir, 'real.txt'), 'keep')
94
95 const wm = new WatchManager()
96 await wm.ready()
97
98 const events = []
99 wm.onChange((e) => events.push(e))
100
101 wm.watchPeriodic(dir)
102
103 // Should only see real.txt
104 t.is(events.length, 1)
105 t.ok(events[0].path.endsWith('real.txt'))
106
107 await wm.close()
108})
109
110test('periodic scan — handles subdirectories', async (t) => {
111 const dir = tmpDir('watch')
112 fs.mkdirSync(path.join(dir, 'sub', 'deep'), { recursive: true })
113 fs.writeFileSync(path.join(dir, 'top.txt'), 'top')
114 fs.writeFileSync(path.join(dir, 'sub', 'mid.txt'), 'mid')
115 fs.writeFileSync(path.join(dir, 'sub', 'deep', 'bottom.txt'), 'bottom')
116
117 const wm = new WatchManager()
118 await wm.ready()
119
120 const events = []
121 wm.onChange((e) => events.push(e))
122
123 wm.watchPeriodic(dir)
124
125 t.is(events.length, 3, 'found all 3 files')
126
127 await wm.close()
128})
129
130// ── listWatched ───────────────────────────────────────────────
131
132test('listWatched returns watched dirs', async (t) => {
133 const dir1 = tmpDir('watch1')
134 const dir2 = tmpDir('watch2')
135
136 const wm = new WatchManager()
137 await wm.ready()
138
139 wm.watchPeriodic(dir1)
140 wm.watchPeriodic(dir2)
141
142 const watched = wm.listWatched()
143 t.is(watched.length, 2)
144 t.ok(watched.find(w => w.dir === dir1 && w.mode === 'periodic'))
145 t.ok(watched.find(w => w.dir === dir2 && w.mode === 'periodic'))
146
147 await wm.close()
148})
149
150// ── unwatch ───────────────────────────────────────────────────
151
152test('unwatch removes directory from watching', async (t) => {
153 const dir = tmpDir('watch')
154
155 const wm = new WatchManager()
156 await wm.ready()
157
158 wm.watchPeriodic(dir)
159 t.is(wm.listWatched().length, 1)
160
161 wm.unwatch(dir)
162 t.is(wm.listWatched().length, 0)
163
164 await wm.close()
165})
166
167// ── Custom ignore patterns ────────────────────────────────────
168
169test('custom ignore patterns', async (t) => {
170 const dir = tmpDir('watch')
171 fs.writeFileSync(path.join(dir, 'keep.txt'), 'keep')
172 fs.writeFileSync(path.join(dir, 'skip.log'), 'skip')
173 fs.writeFileSync(path.join(dir, 'skip.tmp'), 'skip')
174
175 const wm = new WatchManager({ ignore: ['skip.*'] })
176 await wm.ready()
177
178 const events = []
179 wm.onChange((e) => events.push(e))
180
181 wm.watchPeriodic(dir)
182
183 // Only keep.txt should be detected (skip.log and skip.tmp match 'skip.*' glob)
184 t.is(events.length, 1)
185 t.ok(events[0].path.endsWith('keep.txt'))
186
187 await wm.close()
188})
189
190// ── Ignore .env files ─────────────────────────────────────────
191
192test('default ignore blocks .env files', async (t) => {
193 const dir = tmpDir('watch')
194 fs.writeFileSync(path.join(dir, '.env'), 'SECRET=bad')
195 fs.writeFileSync(path.join(dir, '.env.local'), 'SECRET=worse')
196 fs.writeFileSync(path.join(dir, 'app.js'), 'code')
197
198 const wm = new WatchManager()
199 await wm.ready()
200
201 const events = []
202 wm.onChange((e) => events.push(e))
203
204 wm.watchPeriodic(dir)
205
206 t.is(events.length, 1)
207 t.ok(events[0].path.endsWith('app.js'))
208
209 await wm.close()
210})