OverviewFilesCommitsIssuesDocsReleasesPeers

hyper-overlay commits / 478657d

port(0.3.0): restructure to lib/v3 engine + compat shims (Ok-mirror + our-parity)
lib/v3/ = Ok hyper-overlay-v3 engine VERBATIM (their names/internal imports intact) so we drop in their updates with zero rename = Ok-parity. Our public API unchanged: lib/{file-index,transfer,chunker,sync-engine,chunk-scheduler, messages-v2}.js + protocol-v2/overlay-v2 are thin re-export shims -> lib/v3 (protocol-v2/overlay-v2 alias OverlayProtocol/HyperOverlay to our V2 names). package.json + index.js unchanged; v1 + tree/pointer/ignore untouched. Deltas applied in lib/v3 (we are upstream; Ok pulls from us): - file-index getIndexMeta/putIndexMeta (hyper-deploy fast-boot) - overlay/protocol/transfer: bare-fs/path/os -> #fs/#path/#os import-map. PRESERVES the dual-runtime property our 0.2.14 had (node CI/dev/tooling); byte-identical under Bare (#fs->bare-fs via the map). Prod runs under Bare (pgh + daemons); this only keeps node-capability. Upstream-worthy. Verified under node: all public entry points load, shims resolve, getIndexMeta round-trips, HyperOverlayV2(v3) lifecycle + api present. Increment 4 (structure). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
478657d8c3e55874449095ce176f507bd39c5275
by plankatron <fleeky@gmail.com> · 2026-07-11T14:17:54.000Z

Changed files (18) +4296 -3181

lib/chunk-scheduler.js +2 -168
……@@ -1,168 +1,2 @@
1/**
2 * ChunkScheduler — multi-source (torrent-style) fetch of one content-addressed
3 * file across several peers, modelled on Hypercore's block replicator.
4 *
5 * Chunks are content-addressed, so any peer holding the file can serve any
6 * chunk. The scheduler:
7 * - requests the chunk list from each peer (identical, content-addressed),
8 * - tracks a global `inflight` map so each chunk is requested from exactly
9 * ONE peer at a time (no wasted bandwidth),
10 * - keeps each peer busy up to a per-peer inflight cap (parallel download),
11 * - reassigns a peer's inflight chunks to others on stall/disconnect,
12 * - hash-verifies every chunk (via TransferManager.writeChunk) and the whole
13 * file before finalizing.
14 *
15 * It drives the EXISTING v2 wire messages (chunk-need / chunk-data) — no new
16 * message type is needed for the full-copy-seeder case. The protocol delegates
17 * `_onChunkHashes` / `_onChunkData` to a scheduler ONLY when one is registered
18 * for the synthetic `content:<hash>` path, so the legacy single-peer flow
19 * (pgh NetworkFetcher) is untouched.
20 */
21
22import { hashChunk } from './chunker.js'
23
24const DEFAULT_CAP = 8
25const DEFAULT_TIMEOUT = 30000
26
27export class ChunkScheduler {
28 /**
29 * @param {object} opts
30 * opts.path synthetic path key ('content:<hash>')
31 * opts.destPath where the assembled file is written
32 * opts.transfer TransferManager
33 * opts.sendNeed (peer, indices[]) => void — sends a chunk-need message
34 * opts.cap per-peer inflight cap (default 8)
35 * opts.timeout overall fetch timeout ms (default 30000)
36 * opts.onProgress optional () => void called on each accepted chunk
37 */
38 constructor (opts) {
39 this.path = opts.path
40 this.destPath = opts.destPath
41 this._transfer = opts.transfer
42 this._sendNeed = opts.sendNeed
43 this._cap = opts.cap || DEFAULT_CAP
44 this._onProgress = opts.onProgress || null
45
46 this._peers = new Set()
47 this._started = false
48 this._done = false
49 this._chunks = null // [{ hash, offset, length }]
50 this._needed = new Set() // indices not yet written
51 this._inflight = new Map() // index → peer (one peer per chunk)
52 this._peerInflight = new Map() // peer → count
53 this._received = new Map() // peer → chunks accepted (multi-source proof)
54
55 this._resolve = null
56 this._reject = null
57 this._timer = setTimeout(() => this._fail(new Error('multi-source fetch timed out')), opts.timeout || DEFAULT_TIMEOUT)
58 }
59
60 promise () {
61 return new Promise((resolve, reject) => { this._resolve = resolve; this._reject = reject })
62 }
63
64 _fail (err) {
65 if (this._done) return
66 this._done = true
67 clearTimeout(this._timer)
68 if (this._reject) this._reject(err)
69 }
70
71 _finish () {
72 if (this._done) return
73 this._done = true
74 clearTimeout(this._timer)
75 if (this._resolve) this._resolve({ chunksPerPeer: this._received, peerCount: this._received.size })
76 }
77
78 get done () { return this._done }
79
80 /** A peer responded with the chunk list. First response starts the transfer. */
81 onChunkHashes (peer, chunks) {
82 if (this._done) return
83 this._peers.add(peer)
84 if (!this._peerInflight.has(peer)) this._peerInflight.set(peer, 0)
85
86 if (!this._started) {
87 this._started = true
88 // Derive offsets from the ordered chunk lengths.
89 let offset = 0
90 this._chunks = chunks.map((c) => {
91 const entry = { hash: c.hash, offset, length: c.length }
92 offset += c.length
93 return entry
94 })
95 this._transfer.startReceive(this.destPath, { size: offset, chunks: this._chunks })
96 for (let i = 0; i < this._chunks.length; i++) this._needed.add(i)
97 if (this._needed.size === 0) return this._finalize()
98 }
99 this._assign()
100 }
101
102 /** A chunk arrived from a peer. Verify + write, then schedule more. */
103 onChunkData (peer, index, data) {
104 if (this._done) return
105 const res = this._transfer.writeChunk(this.destPath, index, data)
106 // Free the inflight slot regardless — a bad chunk should be re-fetched.
107 if (this._inflight.get(index) === peer) {
108 this._inflight.delete(index)
109 this._peerInflight.set(peer, Math.max(0, (this._peerInflight.get(peer) || 1) - 1))
110 }
111 if (!res.ok) {
112 // Hash/length mismatch — leave the chunk in `needed` to retry elsewhere.
113 this._assign()
114 return
115 }
116 this._needed.delete(index)
117 this._received.set(peer, (this._received.get(peer) || 0) + 1)
118 if (this._onProgress) { try { this._onProgress() } catch {} }
119 if (this._needed.size === 0 && this._inflight.size === 0) return this._finalize()
120 this._assign()
121 }
122
123 /** A peer went away — return its inflight chunks to the pool. */
124 removePeer (peer) {
125 if (!this._peers.has(peer)) return
126 this._peers.delete(peer)
127 this._peerInflight.delete(peer)
128 for (const [index, p] of this._inflight) {
129 if (p === peer) this._inflight.delete(index) // back to needed (still in _needed)
130 }
131 if (this._done) return
132 if (this._peers.size === 0 && this._needed.size > 0) {
133 return this._fail(new Error('all peers gone with ' + this._needed.size + ' chunk(s) outstanding'))
134 }
135 this._assign()
136 }
137
138 _finalize () {
139 const fin = this._transfer.finalize(this.destPath)
140 if (!fin.ok) return this._fail(new Error('finalize failed: ' + fin.error))
141 this._finish()
142 }
143
144 /** Assign needed-but-not-inflight chunks to peers with spare capacity. */
145 _assign () {
146 if (this._done || !this._chunks) return
147 // Per peer, batch the indices we hand it this round.
148 const batches = new Map()
149 for (const peer of this._peers) {
150 let slots = this._cap - (this._peerInflight.get(peer) || 0)
151 if (slots <= 0) continue
152 for (const index of this._needed) {
153 if (slots <= 0) break
154 if (this._inflight.has(index)) continue
155 this._inflight.set(index, peer)
156 this._peerInflight.set(peer, (this._peerInflight.get(peer) || 0) + 1)
157 if (!batches.has(peer)) batches.set(peer, [])
158 batches.get(peer).push(index)
159 slots--
160 }
161 }
162 for (const [peer, indices] of batches) {
163 if (indices.length) this._sendNeed(peer, indices)
164 }
165 }
166}
167
168export { hashChunk }
1// SHIM: v0.3.0 engine lives in lib/v3/ (Ok mirror). Re-export under our name/subpath.
2export * from './v3/chunk-scheduler.js'
lib/chunker.js +2 -333
……@@ -1,333 +1,2 @@
1/**
2 * FastCDC Content-Defined Chunker
3 *
4 * Gear rolling hash with normalized chunking (Xia et al. 2016).
5 * Produces content-stable chunk boundaries — editing the middle of a file
6 * only changes ~2 chunks regardless of file size.
7 *
8 * v2: Adaptive chunk sizing — tier selected by file size.
9 * All peers MUST use the same tier for a given file to produce matching boundaries.
10 */
11
12import crypto from 'hypercore-crypto'
13
14// ── Tier 0 (default, < 1 MB) ─────────────────────────────────
15export const MIN_SIZE = 4096
16export const AVG_SIZE = 16384
17export const MAX_SIZE = 65536
18
19// ── Adaptive chunk size tiers ─────────────────────────────────
20export const TIERS = [
21 { minSize: 4096, avgSize: 16384, maxSize: 65536 }, // Tier 0: < 1 MB
22 { minSize: 16384, avgSize: 65536, maxSize: 262144 }, // Tier 1: 1 MB - 100 MB
23 { minSize: 65536, avgSize: 262144, maxSize: 1048576 }, // Tier 2: 100 MB - 10 GB
24 { minSize: 262144, avgSize: 1048576, maxSize: 4194304 } // Tier 3: > 10 GB
25]
26
27const TIER_THRESHOLDS = [
28 1048576, // 1 MB
29 104857600, // 100 MB
30 10737418240 // 10 GB
31]
32
33/**
34 * Select the appropriate chunk size tier for a file
35 * @param {number} fileSize - total file size in bytes
36 * @returns {number} tier index (0-3)
37 */
38export function selectTier (fileSize) {
39 for (let i = 0; i < TIER_THRESHOLDS.length; i++) {
40 if (fileSize < TIER_THRESHOLDS[i]) return i
41 }
42 return TIERS.length - 1
43}
44
45/**
46 * Get chunk size parameters for a tier
47 * @param {number} tier - tier index (0-3)
48 * @returns {{ minSize: number, avgSize: number, maxSize: number }}
49 */
50export function getTierParams (tier) {
51 return TIERS[tier] || TIERS[0]
52}
53
54// Gear hash table — 256 random uint32 values
55// MUST be identical across all implementations for chunk boundary compatibility
56const GEAR = new Uint32Array([
57 0x5c95c078, 0x22408989, 0x2d48a214, 0x12842087, 0x530f8afb, 0x474536b9, 0x2963b4f1, 0x44cb738b,
58 0x4ea7403d, 0x4d606b6e, 0x074ec5d3, 0x3af39d18, 0x726c7b74, 0x60b3f158, 0x56a76369, 0x3e8ab850,
59 0x0c215cf6, 0x42ad3c2f, 0x65b26fa7, 0x69a3e894, 0x2a60445d, 0x21b0f756, 0x5caca9af, 0x373b4cf0,
60 0x1a4a56e1, 0x5eaf1e7b, 0x7b5d5c8a, 0x48f5abf4, 0x02a42b68, 0x1f3f1a4e, 0x67e16b96, 0x404c7833,
61 0x6e4e3a3c, 0x4f769e04, 0x0bfef25f, 0x3f08d1ef, 0x3bee8eb5, 0x4302687c, 0x3e9e2964, 0x0fef6b59,
62 0x70b95d2c, 0x0cffb210, 0x2e6c3f2a, 0x6d8f0f67, 0x77a49ba2, 0x35fa6b91, 0x43d2d154, 0x5d4e6c25,
63 0x0e7e5283, 0x5fac3d19, 0x20e34b36, 0x46b6de87, 0x48eed852, 0x63aae4d9, 0x2c65c18b, 0x5e5bc6ad,
64 0x7802c749, 0x44db9a82, 0x50e76e40, 0x532dfa2b, 0x56a17be8, 0x0a81e2e7, 0x5cf2fccc, 0x3ab39d12,
65 0x5c0c0b78, 0x21e8af93, 0x3dca216f, 0x15b4b92e, 0x55445f9c, 0x44f24b2e, 0x2db33f38, 0x444f6ec8,
66 0x4eeaede8, 0x48b27a8d, 0x00b7621d, 0x3c916d41, 0x72f65778, 0x60b1a155, 0x5a946e70, 0x3ea5d857,
67 0x0aca0e86, 0x41b3b22e, 0x6572e2a6, 0x6aa8e89a, 0x2d496f61, 0x23b6f159, 0x5eb7e1a6, 0x312e4fea,
68 0x1baf6dc4, 0x5eb0f27e, 0x7a505c92, 0x49e8abf2, 0x03b12c6e, 0x1e4a7a4d, 0x64e76a98, 0x414c7c39,
69 0x6f4a3f38, 0x4c719e04, 0x0efef963, 0x3d08c1e9, 0x3ce88fba, 0x44026c79, 0x3e1e2469, 0x10ef6858,
70 0x73b55e21, 0x0dcfb614, 0x2b6d3927, 0x6c8e1066, 0x74a09bb5, 0x36fa7690, 0x44d2c158, 0x5e4f6b26,
71 0x0e7e5788, 0x5dad3e1c, 0x22e14a30, 0x47b6dd88, 0x4befd953, 0x64aae5d8, 0x2b65b28a, 0x5f5ac6aa,
72 0x7903c648, 0x43db1a89, 0x50e76f45, 0x542dfb28, 0x55a07cef, 0x0b80e3ea, 0x5ef2f9cf, 0x38b31e15,
73 0x5d0d0a7d, 0x22e9ae91, 0x3ecb216a, 0x16b5b82f, 0x54445e99, 0x45f34c2d, 0x2eb23e3b, 0x454e6dc9,
74 0x4febedef, 0x49b37a88, 0x01b66320, 0x3d926e44, 0x71f75779, 0x61b0a054, 0x5b947f71, 0x3fa4d754,
75 0x0bcb0f83, 0x42b4b32f, 0x6473e3a5, 0x6ba9e999, 0x2e497064, 0x24b7f05a, 0x5fb6e0a7, 0x322f5fe9,
76 0x1cae7ec3, 0x5db0e37f, 0x7b514d91, 0x4ae9abf3, 0x02b22d6b, 0x1f4b7b50, 0x63e66b97, 0x414d7d34,
77 0x6e4f3a3d, 0x4d779f05, 0x0afe0960, 0x3e09d0ee, 0x3de98ebb, 0x43036c7a, 0x3f1f2560, 0x11f06959,
78 0x72b65e22, 0x0dceb711, 0x2a6e3826, 0x6d8f1165, 0x75a19ab4, 0x37fb7793, 0x45d3c057, 0x5f4e6a27,
79 0x0f7f5684, 0x5eae3f1d, 0x23e24b31, 0x48b7de89, 0x4ceed854, 0x63abe5db, 0x2c64b38b, 0x5e5bc7ab,
80 0x7a04c54a, 0x44db1b88, 0x51e86e41, 0x552efb2a, 0x54a17de0, 0x0a81e3e6, 0x5df3f8ce, 0x39b41f14,
81 0x5e0e0b79, 0x23e8af92, 0x3fcc216e, 0x17b4b92d, 0x55456e9d, 0x46f44d2f, 0x2fb33e39, 0x464f6ec7,
82 0x4febe8e9, 0x49b27b8c, 0x02b7631e, 0x3e916e42, 0x73f75677, 0x62b2a256, 0x5c957e72, 0x3ea6d856,
83 0x0cca0f85, 0x43b4b231, 0x6574e2a7, 0x6ca9e89b, 0x2f496e62, 0x25b7f058, 0x5eb7e0a5, 0x332e4eeb,
84 0x1daf6ec5, 0x5eb0e27d, 0x7c515d93, 0x4be8abf1, 0x04b32d6a, 0x204b7a4e, 0x64e76b99, 0x424c7d3a,
85 0x704a3e3b, 0x4e789f06, 0x0bfefa62, 0x3f09d1e8, 0x3ee98fbd, 0x45036d7b, 0x401f2461, 0x12f0685a,
86 0x74b65f23, 0x0ecfb713, 0x2b6d3928, 0x6e901267, 0x76a29bb6, 0x38fc7892, 0x46d3c159, 0x604e6b28,
87 0x107e5785, 0x5fae3e1e, 0x24e24b32, 0x49b8df8a, 0x4defd955, 0x64abe6da, 0x2d65b48c, 0x605bc8ac,
88 0x7b05c64b, 0x45dc1c87, 0x52e96f46, 0x562ffc29, 0x55a27ee1, 0x0b82e4e5, 0x5ef3f7cd, 0x3ab42017
89])
90
91// Precomputed masks per tier (controls cut-point frequency)
92// Below average: larger mask → fewer cuts → chunks grow toward average
93// Above average: smaller mask → more cuts → chunks stay near average
94const TIER_MASKS = TIERS.map(t => ({
95 large: (1 << Math.ceil(Math.log2(t.avgSize))) - 1,
96 small: (1 << Math.floor(Math.log2(t.avgSize))) - 1
97}))
98
99/**
100 * Chunk a buffer using FastCDC with adaptive tier selection
101 *
102 * @param {Buffer} data - Data to chunk
103 * @param {{ tier?: number }} [opts] - Options. tier: force a specific tier (0-3). If omitted, auto-selects based on data.length.
104 * @returns {Array<{ hash: string, offset: number, length: number, data: Buffer }>}
105 */
106export function chunk (data, opts) {
107 if (data.length === 0) return []
108
109 const tier = (opts && typeof opts.tier === 'number') ? opts.tier : selectTier(data.length)
110 const params = TIERS[tier] || TIERS[0]
111 const masks = TIER_MASKS[tier] || TIER_MASKS[0]
112
113 const chunks = []
114 let offset = 0
115
116 while (offset < data.length) {
117 const remaining = data.length - offset
118
119 // If remaining data is smaller than minimum, it's the final chunk
120 if (remaining <= params.minSize) {
121 const slice = data.subarray(offset, offset + remaining)
122 chunks.push({
123 hash: hashChunk(slice),
124 offset,
125 length: remaining,
126 data: slice
127 })
128 break
129 }
130
131 const maxLen = Math.min(remaining, params.maxSize)
132 let hash = 0
133 let i = params.minSize
134
135 // Phase 1: below average — use large mask (fewer cuts)
136 const avgBound = Math.min(params.avgSize, maxLen)
137 while (i < avgBound) {
138 hash = ((hash << 1) + GEAR[data[offset + i]]) >>> 0
139 if ((hash & masks.large) === 0) break
140 i++
141 }
142
143 // Phase 2: above average — use small mask (more cuts)
144 if (i >= avgBound) {
145 while (i < maxLen) {
146 hash = ((hash << 1) + GEAR[data[offset + i]]) >>> 0
147 if ((hash & masks.small) === 0) break
148 i++
149 }
150 }
151
152 const slice = data.subarray(offset, offset + i)
153 chunks.push({
154 hash: hashChunk(slice),
155 offset,
156 length: i,
157 data: slice
158 })
159 offset += i
160 }
161
162 return chunks
163}
164
165/**
166 * Hash a chunk using blake2b-256
167 * @param {Buffer} data
168 * @returns {string} hex hash
169 */
170export function hashChunk (data) {
171 return crypto.data(data).toString('hex')
172}
173
174// ── Streaming primitives ──────────────────────────────────────
175//
176// `chunk(buffer)` requires the entire file in memory and returns chunks
177// whose `.data` is a `Buffer.subarray` of the input — both keep the
178// original buffer pinned until every yielded chunk is GC'd. For
179// multi-GB files (or limited-RAM runtimes) use the streaming versions
180// below: `chunkStream` consumes a Readable and yields chunks one at a
181// time, releasing bytes after each yield.
182
183import sodium from 'sodium-universal'
184import c from 'compact-encoding'
185
186const LEAF_TYPE = Buffer.from([0])
187
188/**
189 * Streaming blake2b-256 hasher matching `hashChunk(buffer)`'s digest
190 * format (hypercore-crypto's `crypto.data` shape:
191 * blake2b(LEAF_TYPE || uint64-LE(size) || data)
192 * ). Lets you hash multi-GB files without buffering them.
193 *
194 * const h = createStreamingHasher({ size }) // size known up front
195 * for await (const buf of stream) h.update(buf)
196 * const oid = h.digest() // == hashChunk(file)
197 *
198 * `size` is required for hash compatibility with `hashChunk` — the
199 * leaf-prefix construction includes the byte length, which is fixed at
200 * stream open. If you don't have the size, omit `size` and you'll get
201 * a plain blake2b digest of the bytes (does NOT match hashChunk).
202 */
203export function createStreamingHasher (opts = {}) {
204 const state = Buffer.alloc(sodium.crypto_generichash_STATEBYTES)
205 let bytes = 0
206 if (opts && opts.restore) {
207 // The saved state already includes the init + leaf/size prefix + every byte
208 // consumed so far, so do not re-init.
209 opts.restore.state.copy(state)
210 bytes = opts.restore.bytes
211 } else {
212 sodium.crypto_generichash_init(state, null, 32)
213 const knownSize = (opts && typeof opts.size === 'number') ? opts.size : null
214 if (knownSize !== null) {
215 sodium.crypto_generichash_update(state, LEAF_TYPE)
216 sodium.crypto_generichash_update(state, c.encode(c.uint64, knownSize))
217 }
218 }
219 return {
220 update (buf) {
221 sodium.crypto_generichash_update(state, buf)
222 bytes += buf.length
223 },
224 digest () {
225 const out = Buffer.alloc(32)
226 sodium.crypto_generichash_final(state, out)
227 return out.toString('hex')
228 },
229 snapshot () { return { state: Buffer.from(state), bytes } },
230 get bytes () { return bytes }
231 }
232}
233
234// Find a FastCDC cut point inside `data` starting at offset 0. Returns
235// the chunk length. Mirrors the inner search loop of `chunk()` but
236// takes pre-resolved params + masks so it works in a streaming loop.
237function findCutPoint (data, params, masks) {
238 const remaining = data.length
239 if (remaining <= params.minSize) return remaining
240
241 const maxLen = Math.min(remaining, params.maxSize)
242 let hash = 0
243 let i = params.minSize
244
245 const avgBound = Math.min(params.avgSize, maxLen)
246 while (i < avgBound) {
247 hash = ((hash << 1) + GEAR[data[i]]) >>> 0
248 if ((hash & masks.large) === 0) return i
249 i++
250 }
251 if (i >= avgBound) {
252 while (i < maxLen) {
253 hash = ((hash << 1) + GEAR[data[i]]) >>> 0
254 if ((hash & masks.small) === 0) return i
255 i++
256 }
257 }
258 return maxLen
259}
260
261// Opt-in memcpy accounting for perf validation. Zero cost when off (one
262// boolean check at the increment sites). Module-level singletons — enabled
263// only by tests + bench/prepare.js, never in library code; NOT
264// concurrency-safe, so do not enable it in production.
265let STATS_ON = false
266export const chunkStats = { concatBytes: 0, copyBytes: 0, blocks: 0, chunks: 0 }
267export function resetChunkStats () { chunkStats.concatBytes = 0; chunkStats.copyBytes = 0; chunkStats.blocks = 0; chunkStats.chunks = 0 }
268export function setChunkStats (on) { STATS_ON = !!on; if (on) resetChunkStats() }
269
270/**
271 * Stream-friendly FastCDC chunker. Consumes an AsyncIterable<Buffer>
272 * (e.g. fs.createReadStream) and yields chunks one at a time. Memory
273 * footprint ≈ 2 × params.maxSize regardless of stream length —
274 * chunks are emitted as bytes arrive, the pending buffer is sliced
275 * past each cut point.
276 *
277 * Each yielded chunk has the same shape as `chunk()` entries:
278 * { hash, offset, length, data }
279 * With the default `copy: true`, `data` is a fresh `Buffer.from(slice)`
280 * detached from the input stream's pool, so callers can store/forward it.
281 * With `copy: false`, `data` is a view backed by the reader's memory —
282 * valid only until the next iteration; the caller MUST consume it
283 * synchronously and not retain it across an await.
284 *
285 * Cut-point determinism: same content + same tier → identical chunk
286 * boundaries as `chunk()`, regardless of input block size. Peers using
287 * either function on the same file produce matching chunk hashes.
288 *
289 * @param {AsyncIterable<Buffer>} readable
290 * @param {{ tier?: number, copy?: boolean }} [opts] tier index (0–3); copy
291 * defaults true. Callers with a known file size should pass `selectTier(size)`.
292 */
293export async function * chunkStream (readable, opts = {}) {
294 const tier = (opts && typeof opts.tier === 'number') ? opts.tier : 0
295 const copy = opts.copy !== false // false → yield views (see JSDoc above)
296 const params = TIERS[tier] || TIERS[0]
297 const masks = TIER_MASKS[tier] || TIER_MASKS[0]
298
299 let pending = Buffer.alloc(0)
300 let absoluteOffset = 0
301
302 // Single emit point for the stats accounting + copy:false decision.
303 const emit = (slice) => {
304 if (STATS_ON) { chunkStats.chunks++; if (copy) chunkStats.copyBytes += slice.length }
305 return { hash: hashChunk(slice), offset: absoluteOffset, length: slice.length, data: copy ? Buffer.from(slice) : slice }
306 }
307
308 for await (const inChunk of readable) {
309 const buf = Buffer.isBuffer(inChunk) ? inChunk : Buffer.from(inChunk)
310 if (STATS_ON) { chunkStats.blocks++; if (pending.length !== 0) chunkStats.concatBytes += pending.length + buf.length }
311 pending = pending.length === 0 ? buf : Buffer.concat([pending, buf])
312
313 // While we have at least maxSize ahead, we can definitively find
314 // the next cut point inside `pending` (the cut is bounded by maxSize).
315 while (pending.length >= params.maxSize) {
316 const cutLen = findCutPoint(pending, params, masks)
317 yield emit(pending.subarray(0, cutLen))
318 absoluteOffset += cutLen
319 pending = pending.subarray(cutLen)
320 }
321 }
322
323 // Stream ended. Process leftover. Same minSize-final-chunk rule as
324 // the buffer-mode `chunk()`.
325 while (pending.length > params.minSize) {
326 const cutLen = findCutPoint(pending, params, masks)
327 yield emit(pending.subarray(0, cutLen))
328 absoluteOffset += cutLen
329 pending = pending.subarray(cutLen)
330 }
331
332 if (pending.length > 0) yield emit(pending)
333}
1// SHIM: v0.3.0 engine lives in lib/v3/ (Ok mirror). Re-export under our name/subpath.
2export * from './v3/chunker.js'
lib/file-index.js +2 -576
……@@ -1,576 +1,2 @@
1/**
2 * FileIndex — Metadata-only file index backed by Hyperbee
3 *
4 * Stores file metadata and chunk maps. Never stores file content.
5 * The filesystem is the source of truth for actual bytes.
6 *
7 * Hyperbee schema:
8 * file:<path> → { contentHash, size, mtime, hashing, executable, linkname }
9 * chunkmap:<path> → [{ hash, offset, length }] (small) | { __paged: N } (large; see below)
10 * chunkmap-oid:<contentHash> → [{ hash, offset, length }] (content-addressed; reused across overlay keys + restarts)
11 * sync:<peerKey>:<path> → { lastSeq, lastHash }
12 * config:sync → { folders: [...] }
13 *
14 * tree:<hex-hash> → { entries: [{kind, exec, name, childHash, size}], size }
15 * treepath:<path> → <hex-hash> (secondary: path → current tree hash)
16 *
17 * Chunk-map paging. A chunk map for a very large file (a 1.1 TB
18 * file at tier 3 ≈ 1M entries ≈ ~120 MB of JSON) does not fit in one Hyperbee
19 * value: Hypercore caps a block at 15 MiB (MAX_SUGGESTED_BLOCK_SIZE) and throws
20 * BAD_ARGUMENT above it. Maps with > CHUNKS_PER_PAGE entries are stored as a
21 * header value { __paged: N } at the base key plus N page values keyed
22 * `<base>\x00<i>` (the NUL byte never occurs in a path or hex hash, so a page key
23 * can't collide with a real one). Small maps stay inline as a plain array —
24 * byte-identical to before, and back-compatible with already-stored maps. The
25 * public API (put/get/del) is unchanged; paging is transparent to callers.
26 */
27
28import Hyperbee from 'hyperbee'
29import ReadyResource from 'ready-resource'
30
31// Entries per chunk-map page. ~116 B/entry worst-case JSON ⇒
32// ≤ ~3.8 MB/page, well under Hypercore's 15 MiB block limit.
33const CHUNKS_PER_PAGE = 32768
34
35// Overlay-path prefix for host-registered, content-addressed serve entries.
36// Hosts that make a file servable purely by content hash should register it at
37// `${CONTENT_REGISTER_PREFIX}${contentHash}` so compact()/evictContent() can
38// recognize (and reclaim) the entry once the hash is no longer served.
39export const CONTENT_REGISTER_PREFIX = '/mir/'
40
41// The content hash an owner-side entry is addressed by (page suffix folded in), or
42// null for entries not keyed by a single content hash (real-path file:/chunkmap:,
43// tree:, sync:, config:). compact() drops the content-addressed entries whose hash is
44// no longer served, and keeps everything else.
45function contentHashOfKey (key) {
46 if (key.startsWith('chunkmap-oid:')) return key.slice('chunkmap-oid:'.length).split('\x00')[0]
47 if (key.startsWith('chunkmap:content:')) return key.slice('chunkmap:content:'.length).split('\x00')[0]
48 if (key.startsWith(`chunkmap:${CONTENT_REGISTER_PREFIX}`)) return key.slice(`chunkmap:${CONTENT_REGISTER_PREFIX}`.length).split('\x00')[0]
49 if (key.startsWith(`file:${CONTENT_REGISTER_PREFIX}`)) return key.slice(`file:${CONTENT_REGISTER_PREFIX}`.length)
50 return null
51}
52
53// v1 keeps the original 'file-index' name so existing stores aren't orphaned on
54// upgrade; compaction moves to file-index-v2, v3, … and purges the predecessor.
55export function indexCoreName (version) {
56 return version === 1 ? 'file-index' : `file-index-v${version}`
57}
58
59export class FileIndex extends ReadyResource {
60 constructor (store, opts = {}) {
61 super()
62 this._store = store
63 this._bee = null
64 this._meta = null
65 this._version = 1
66 this._opts = opts
67 }
68
69 // Local index cores are encrypted at rest when the caller passes an encryptionKey
70 // (opts.encryptionKey); absent, they stay plaintext.
71 _coreOpts (name) {
72 const o = { name, valueEncoding: 'binary' }
73 if (this._opts.encryptionKey) o.encryptionKey = this._opts.encryptionKey
74 return o
75 }
76
77 async _open () {
78 this._meta = new Hyperbee(this._store.get(this._coreOpts('index-meta')), {
79 keyEncoding: 'utf-8',
80 valueEncoding: 'json'
81 })
82 await this._meta.ready()
83 this._version = (await this._meta.get('version'))?.value ?? 1
84 // A prior purge can leave a version's core with a dangling name alias — its data and
85 // by-discovery-key alias were deleted but the by-name alias was not — which throws
86 // STORAGE_EMPTY on open. The index is rebuildable cache, so advance to a fresh core
87 // name and record it rather than failing open.
88 for (let attempts = 0; ; attempts++) {
89 const bee = new Hyperbee(this._store.get(this._coreOpts(indexCoreName(this._version))), {
90 keyEncoding: 'utf-8',
91 valueEncoding: 'json'
92 })
93 try {
94 await bee.ready()
95 this._bee = bee
96 return
97 } catch (err) {
98 if (err.code !== 'STORAGE_EMPTY' || attempts >= 64) throw err
99 await bee.close().catch(() => {})
100 this._version += 1
101 await this._meta.put('version', this._version)
102 }
103 }
104 }
105
106 async _close () {
107 if (this._bee) await this._bee.close()
108 if (this._meta) await this._meta.close()
109 }
110
111 get bee () { return this._bee }
112
113 get version () { return this._version }
114
115 get cores () { return [this._bee?.core, this._meta?.core].filter(Boolean) }
116
117 // Reclaim the append-only index: stream the still-served entries into a fresh
118 // versioned core, flip the version pointer, and return the old core so the caller
119 // can clear+purge it (the only way to return an append-only bee's disk to the OS).
120 // Content-addressed entries (maps + content-register entries) whose hash isServed() is false
121 // are dropped; real-path, tree, sync and config entries are always kept.
122 async compact ({ isServed }) {
123 // Skip the rewrite entirely when nothing is droppable — otherwise a compaction of
124 // an already-clean index just churns (a fresh version core + a version-marker
125 // append), which grows the index without reclaiming anything.
126 let droppable = false
127 for await (const { key } of this._bee.createReadStream()) {
128 const hash = contentHashOfKey(key)
129 if (hash !== null && !isServed(hash)) { droppable = true; break }
130 }
131 if (!droppable) return null
132
133 const next = this._version + 1
134 const dst = new Hyperbee(this._store.get(this._coreOpts(indexCoreName(next))), {
135 keyEncoding: 'utf-8',
136 valueEncoding: 'json'
137 })
138 await dst.ready()
139 let batch = dst.batch()
140 let pending = 0
141 for await (const { key, value } of this._bee.createReadStream()) {
142 const hash = contentHashOfKey(key)
143 if (hash !== null && !isServed(hash)) continue
144 await batch.put(key, value)
145 if (++pending >= 500) { await batch.flush(); batch = dst.batch(); pending = 0 }
146 }
147 await batch.flush()
148
149 const oldCore = this._bee.core
150 await this._meta.put('version', next)
151 this._bee = dst
152 this._version = next
153 return oldCore // left open; caller clears + purges it
154 }
155
156 // ── File entries ────────────────────────────────────────────
157
158 /**
159 * Store or update a file entry (metadata only, no content)
160 * @param {string} path - file path
161 * @param {{ contentHash: string|null, size: number, mtime: number, hashing?: boolean, executable?: boolean, linkname?: string|null }} meta
162 */
163 async putFile (path, meta) {
164 await this._bee.put(`file:${path}`, {
165 contentHash: meta.contentHash || null,
166 size: meta.size,
167 mtime: meta.mtime,
168 hashing: meta.hashing || false,
169 executable: meta.executable || false,
170 linkname: meta.linkname || null
171 })
172 }
173
174 /**
175 * Get file metadata
176 * @param {string} path
177 * @returns {{ contentHash: string|null, size: number, mtime: number, hashing: boolean, executable: boolean, linkname: string|null } | null}
178 */
179 async getFile (path) {
180 const entry = await this._bee.get(`file:${path}`)
181 return entry ? entry.value : null
182 }
183
184 /**
185 * Check if file exists in the index
186 * @param {string} path
187 * @returns {boolean}
188 */
189 async hasFile (path) {
190 const entry = await this._bee.get(`file:${path}`)
191 return entry !== null
192 }
193
194 /**
195 * Remove a file entry and its chunk map
196 * @param {string} path
197 */
198 async delFile (path) {
199 await this._bee.del(`file:${path}`)
200 await this._delPagedValue(`chunkmap:${path}`)
201 }
202
203 /**
204 * List files under a directory prefix
205 * @param {string} [dir=''] - directory prefix
206 * @returns {Array<{ path: string, contentHash: string|null, size: number, mtime: number }>}
207 */
208 async listFiles (dir = '') {
209 const prefix = dir ? `file:${dir}` : 'file:'
210 const files = []
211
212 for await (const entry of this._bee.createReadStream({ gt: prefix, lt: prefix + '\xff' })) {
213 files.push({
214 path: entry.key.slice('file:'.length),
215 ...entry.value
216 })
217 }
218
219 return files
220 }
221
222 // ── Index meta (0.2.14; preserved on port) ──────────────────
223 // Small key/value bag on the main bee under `meta:` — used by the
224 // fast manifest-diff boot path (e.g. hyper-deploy). compact() keeps these
225 // (contentHashOfKey returns null for `meta:` → not droppable).
226
227 async putIndexMeta (key, value) {
228 await this._bee.put(`meta:${key}`, value)
229 }
230
231 async getIndexMeta (key) {
232 const entry = await this._bee.get(`meta:${key}`)
233 return entry ? entry.value : null
234 }
235
236 // ── Chunk maps ──────────────────────────────────────────────
237
238 /**
239 * Store a chunk map for a file (hash + offset + length per chunk, no data).
240 * Large maps are paged transparently (see the header note) so they never exceed the
241 * Hypercore block limit.
242 * @param {string} path - file path
243 * @param {Array<{ hash: string, offset: number, length: number }>} chunks
244 */
245 async putChunkMap (path, chunks) {
246 await this._putPagedValue(`chunkmap:${path}`, chunks)
247 }
248
249 /**
250 * Get the chunk map for a file
251 * @param {string} path
252 * @returns {Array<{ hash: string, offset: number, length: number }> | null}
253 */
254 async getChunkMap (path) {
255 return this._getPagedValue(`chunkmap:${path}`)
256 }
257
258 /**
259 * Check if a chunk map exists for a file
260 * @param {string} path
261 * @returns {boolean}
262 */
263 async hasChunkMap (path) {
264 const entry = await this._bee.get(`chunkmap:${path}`)
265 return entry !== null
266 }
267
268 /**
269 * Remove a chunk map
270 * @param {string} path
271 */
272 async delChunkMap (path) {
273 await this._delPagedValue(`chunkmap:${path}`)
274 }
275
276 // Paged-value storage. Small maps stay inline as a plain array
277 // (unchanged on disk, back-compatible); maps over CHUNKS_PER_PAGE entries are
278 // split into a { __paged: N } header + N `<base>\x00<i>` page values, written
279 // in one Hyperbee batch so the whole map commits atomically. Stale pages from
280 // a previously-larger map at the same key are deleted in the same batch.
281 async _putPagedValue (baseKey, chunks) {
282 const prevPages = await this._pagedCount(baseKey)
283
284 if (chunks.length <= CHUNKS_PER_PAGE) {
285 if (prevPages === 0) {
286 await this._bee.put(baseKey, chunks)
287 return
288 }
289 await this._batched(async (b) => {
290 await b.put(baseKey, chunks)
291 for (let i = 0; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`)
292 })
293 return
294 }
295
296 const pageCount = Math.ceil(chunks.length / CHUNKS_PER_PAGE)
297 await this._batched(async (b) => {
298 await b.put(baseKey, { __paged: pageCount })
299 for (let i = 0; i < pageCount; i++) {
300 await b.put(`${baseKey}\x00${i}`, chunks.slice(i * CHUNKS_PER_PAGE, (i + 1) * CHUNKS_PER_PAGE))
301 }
302 for (let i = pageCount; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`)
303 })
304 }
305
306 async _getPagedValue (baseKey) {
307 const entry = await this._bee.get(baseKey)
308 if (!entry) return null
309 if (Array.isArray(entry.value)) return entry.value // inline / legacy
310 const pageCount = entry.value.__paged
311 if (!pageCount) return null
312 const chunks = []
313 for (let i = 0; i < pageCount; i++) {
314 const page = await this._bee.get(`${baseKey}\x00${i}`)
315 // The header + all pages commit in one atomic batch, so a missing or non-array
316 // page means the value is corrupt/incomplete. Return null — a clean "miss" so the
317 // caller re-chunks from the source file — never a silently truncated chunk map.
318 if (!page || !Array.isArray(page.value)) return null
319 for (const c of page.value) chunks.push(c)
320 }
321 return chunks
322 }
323
324 async _delPagedValue (baseKey) {
325 const prevPages = await this._pagedCount(baseKey)
326 if (prevPages === 0) {
327 await this._bee.del(baseKey)
328 return
329 }
330 await this._batched(async (b) => {
331 await b.del(baseKey)
332 for (let i = 0; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`)
333 })
334 }
335
336 // Page count of an existing paged value, or 0 if the key is absent or inline.
337 async _pagedCount (baseKey) {
338 const entry = await this._bee.get(baseKey)
339 if (!entry || Array.isArray(entry.value)) return 0
340 return entry.value.__paged || 0
341 }
342
343 // Run ops in a Hyperbee batch, closing it on error (flush releases the lock
344 // itself, even on append failure, via _appendBatch's finally).
345 async _batched (fn) {
346 const b = this._bee.batch()
347 try {
348 await fn(b)
349 await b.flush()
350 } catch (err) {
351 try { await b.close() } catch {}
352 throw err
353 }
354 }
355
356 // Content-addressed chunk map. FastCDC is deterministic on bytes, so a
357 // given content hash always chunks identically regardless of which overlay path
358 // it was registered under. Keying by content hash lets the publish-time chunking
359 // be reused at serve time (a synthetic 'content:<hash>' key) and across restarts,
360 // instead of re-reading the whole file to re-chunk it before the first byte ships.
361 async putChunkMapByHash (contentHash, chunks) {
362 await this._putPagedValue(`chunkmap-oid:${contentHash}`, chunks)
363 }
364
365 async getChunkMapByHash (contentHash) {
366 return this._getPagedValue(`chunkmap-oid:${contentHash}`)
367 }
368
369 async hasChunkMapByHash (contentHash) {
370 return (await this._bee.get(`chunkmap-oid:${contentHash}`)) !== null
371 }
372
373 async delChunkMapByHash (contentHash) {
374 await this._delPagedValue(`chunkmap-oid:${contentHash}`)
375 }
376
377 // Evict every durable entry a served content hash leaves behind: the content-
378 // addressed map plus the deterministic owner-side synthetic paths (the serve
379 // copy at content:<hash> and the register entry under CONTENT_REGISTER_PREFIX).
380 // Peer-path-keyed receiver entries are unknowable from the hash — the host's
381 // own index sweep handles those.
382 async evictContent (contentHash) {
383 await this.delChunkMapByHash(contentHash)
384 await this.delChunkMap(`content:${contentHash}`)
385 await this.delFile(`${CONTENT_REGISTER_PREFIX}${contentHash}`)
386 }
387
388 // ── Sync state ──────────────────────────────────────────────
389
390 /**
391 * Store sync state for a peer + file
392 * @param {string} peerKey - hex peer key
393 * @param {string} path - file path
394 * @param {{ lastSeq: number, lastHash: string }} state
395 */
396 async putSyncState (peerKey, path, state) {
397 await this._bee.put(`sync:${peerKey}:${path}`, {
398 lastSeq: state.lastSeq,
399 lastHash: state.lastHash
400 })
401 }
402
403 /**
404 * Get sync state for a peer + file
405 * @param {string} peerKey
406 * @param {string} path
407 * @returns {{ lastSeq: number, lastHash: string } | null}
408 */
409 async getSyncState (peerKey, path) {
410 const entry = await this._bee.get(`sync:${peerKey}:${path}`)
411 return entry ? entry.value : null
412 }
413
414 /**
415 * Remove sync state for a peer + file
416 * @param {string} peerKey
417 * @param {string} path
418 */
419 async delSyncState (peerKey, path) {
420 await this._bee.del(`sync:${peerKey}:${path}`)
421 }
422
423 /**
424 * List all sync states for a peer
425 * @param {string} peerKey
426 * @returns {Array<{ path: string, lastSeq: number, lastHash: string }>}
427 */
428 async listSyncStates (peerKey) {
429 const prefix = `sync:${peerKey}:`
430 const states = []
431
432 for await (const entry of this._bee.createReadStream({ gt: prefix, lt: prefix + '\xff' })) {
433 states.push({
434 path: entry.key.slice(prefix.length),
435 ...entry.value
436 })
437 }
438
439 return states
440 }
441
442 // ── Sync config ─────────────────────────────────────────────
443
444 /**
445 * Store sync configuration
446 * @param {{ folders: Array<{ path: string, strategy: string, quota?: string|null, peers?: string[] }> }} config
447 */
448 async putSyncConfig (config) {
449 await this._bee.put('config:sync', config)
450 }
451
452 /**
453 * Get sync configuration
454 * @returns {{ folders: Array } | null}
455 */
456 async getSyncConfig () {
457 const entry = await this._bee.get('config:sync')
458 return entry ? entry.value : null
459 }
460
461 // ── Trees ───────────────────────────────────────────────────
462
463 /**
464 * Store a tree by its content-addressed hash.
465 * Same hash put twice is a no-op (content-addressed — bytes can't differ).
466 *
467 * @param {string} hash - hex-encoded blake2b-256 tree hash
468 * @param {{ entries: Array, size: number }} tree
469 */
470 async putTree (hash, tree) {
471 await this._bee.put(`tree:${hash}`, {
472 entries: tree.entries,
473 size: tree.size
474 })
475 }
476
477 /**
478 * Get a tree by its hash.
479 * @param {string} hash
480 * @returns {{ entries: Array, size: number } | null}
481 */
482 async getTree (hash) {
483 const entry = await this._bee.get(`tree:${hash}`)
484 return entry ? entry.value : null
485 }
486
487 /**
488 * Check if a tree is present.
489 * @param {string} hash
490 * @returns {boolean}
491 */
492 async hasTree (hash) {
493 const entry = await this._bee.get(`tree:${hash}`)
494 return entry !== null
495 }
496
497 /**
498 * Remove a tree. Only safe when no treepath entries reference this hash.
499 * @param {string} hash
500 */
501 async delTree (hash) {
502 await this._bee.del(`tree:${hash}`)
503 }
504
505 /**
506 * Record the current tree hash for a path (secondary index).
507 * @param {string} path
508 * @param {string} hash
509 */
510 async putTreePath (path, hash) {
511 await this._bee.put(`treepath:${path}`, hash)
512 }
513
514 /**
515 * Look up the current tree hash for a path.
516 * @param {string} path
517 * @returns {string|null}
518 */
519 async getTreePath (path) {
520 const entry = await this._bee.get(`treepath:${path}`)
521 return entry ? entry.value : null
522 }
523
524 /**
525 * Remove the treepath entry. Does NOT delete the underlying tree — other
526 * paths may still reference the same hash.
527 * @param {string} path
528 */
529 async delTreePath (path) {
530 await this._bee.del(`treepath:${path}`)
531 }
532
533 /**
534 * List all stored tree hashes.
535 * @param {object} [opts]
536 * @param {number} [opts.limit] - optional cap
537 * @returns {Array<{ hash: string, entryCount: number, size: number }>}
538 */
539 async listTrees (opts = {}) {
540 const trees = []
541 for await (const entry of this._bee.createReadStream({ gt: 'tree:', lt: 'tree:\xff' })) {
542 trees.push({
543 hash: entry.key.slice('tree:'.length),
544 entryCount: entry.value.entries.length,
545 size: entry.value.size
546 })
547 if (opts.limit && trees.length >= opts.limit) break
548 }
549 return trees
550 }
551
552 // ── Stats ───────────────────────────────────────────────────
553
554 /**
555 * Get index statistics
556 * @returns {{ fileCount: number, chunkMapCount: number, syncStateCount: number, treeCount: number, treePathCount: number }}
557 */
558 async stats () {
559 let fileCount = 0
560 let chunkMapCount = 0
561 let syncStateCount = 0
562 let treeCount = 0
563 let treePathCount = 0
564
565 for await (const entry of this._bee.createReadStream()) {
566 if (entry.key.startsWith('file:')) fileCount++
567 // Count one per file — skip the `\x00`-suffixed page values.
568 else if (entry.key.startsWith('chunkmap:') && !entry.key.includes('\x00')) chunkMapCount++
569 else if (entry.key.startsWith('sync:')) syncStateCount++
570 else if (entry.key.startsWith('tree:')) treeCount++
571 else if (entry.key.startsWith('treepath:')) treePathCount++
572 }
573
574 return { fileCount, chunkMapCount, syncStateCount, treeCount, treePathCount }
575 }
576}
1// SHIM: v0.3.0 engine lives in lib/v3/ (Ok mirror). Re-export under our name/subpath.
2export * from './v3/file-index.js'
lib/messages-v2.js +2 -444
……@@ -1,444 +1,2 @@
1/**
2 * Wire Protocol Messages for hyper-overlay/v2
3 *
4 * 13 message types for sync + transfer + trees (0.5a extension):
5 * 0-2: Sync messages (state exchange, file offers/requests)
6 * 3-8: Transfer messages (chunk hashes, data, completion)
7 * 9-11: Tree messages (tree-request, tree-response, content-request)
8 * 12: transfer-control (downloader→holder pause/stop notice)
9 *
10 * Messages 9-12 are backward-compatible additions — peers without
11 * this code simply don't register the corresponding protomux slots, and
12 * senders see no receiver for those messages. File-level sync (0-8)
13 * keeps working unchanged.
14 *
15 * All use compact-encoding codecs with preencode/encode/decode.
16 */
17
18import c from 'compact-encoding'
19
20// ── Helpers ───────────────────────────────────────────────────
21
22function toBuffer32 (hashOrBuf) {
23 if (Buffer.isBuffer(hashOrBuf)) return hashOrBuf
24 return Buffer.from(hashOrBuf, 'hex')
25}
26
27// Encode/decode an array of { hash (32-byte hex), length (uint) }
28const chunkInfoArray = {
29 preencode (state, arr) {
30 c.uint32.preencode(state, arr.length)
31 for (const item of arr) {
32 state.end += 32
33 c.uint.preencode(state, item.length)
34 }
35 },
36 encode (state, arr) {
37 c.uint32.encode(state, arr.length)
38 for (const item of arr) {
39 toBuffer32(item.hash).copy(state.buffer, state.start, 0, 32)
40 state.start += 32
41 c.uint.encode(state, item.length)
42 }
43 },
44 decode (state) {
45 const len = c.uint32.decode(state)
46 const arr = []
47 for (let i = 0; i < len; i++) {
48 const hash = state.buffer.subarray(state.start, state.start + 32).toString('hex')
49 state.start += 32
50 const length = c.uint.decode(state)
51 arr.push({ hash, length })
52 }
53 return arr
54 }
55}
56
57// Encode/decode an array of uints
58const uintArray = {
59 preencode (state, arr) {
60 c.uint32.preencode(state, arr.length)
61 for (const v of arr) c.uint.preencode(state, v)
62 },
63 encode (state, arr) {
64 c.uint32.encode(state, arr.length)
65 for (const v of arr) c.uint.encode(state, v)
66 },
67 decode (state) {
68 const len = c.uint32.decode(state)
69 const arr = []
70 for (let i = 0; i < len; i++) arr.push(c.uint.decode(state))
71 return arr
72 }
73}
74
75// ── Handshake ─────────────────────────────────────────────────
76
77export const handshake = {
78 preencode (state, m) {
79 c.uint8.preencode(state, m.version)
80 c.uint8.preencode(state, m.capabilities)
81 },
82 encode (state, m) {
83 c.uint8.encode(state, m.version)
84 c.uint8.encode(state, m.capabilities)
85 },
86 decode (state) {
87 return {
88 version: c.uint8.decode(state),
89 capabilities: c.uint8.decode(state)
90 }
91 }
92}
93
94// ── Sync Messages (0-2) ──────────────────────────────────────
95
96// Message 0: sync-state — exchange last-synced sequence numbers
97export const syncState = {
98 preencode (state, m) {
99 c.buffer.preencode(state, toBuffer32(m.feedKey))
100 c.uint.preencode(state, m.localSeq)
101 c.uint.preencode(state, m.remoteSeq)
102 },
103 encode (state, m) {
104 c.buffer.encode(state, toBuffer32(m.feedKey))
105 c.uint.encode(state, m.localSeq)
106 c.uint.encode(state, m.remoteSeq)
107 },
108 decode (state) {
109 return {
110 feedKey: c.buffer.decode(state).toString('hex'),
111 localSeq: c.uint.decode(state),
112 remoteSeq: c.uint.decode(state)
113 }
114 }
115}
116
117// Message 1: file-offer — announce a file change
118export const fileOffer = {
119 preencode (state, m) {
120 c.string.preencode(state, m.path)
121 c.buffer.preencode(state, toBuffer32(m.contentHash))
122 c.uint.preencode(state, m.size)
123 c.uint.preencode(state, m.mtime)
124 c.uint8.preencode(state, m.op)
125 },
126 encode (state, m) {
127 c.string.encode(state, m.path)
128 c.buffer.encode(state, toBuffer32(m.contentHash))
129 c.uint.encode(state, m.size)
130 c.uint.encode(state, m.mtime)
131 c.uint8.encode(state, m.op)
132 },
133 decode (state) {
134 return {
135 path: c.string.decode(state),
136 contentHash: c.buffer.decode(state).toString('hex'),
137 size: c.uint.decode(state),
138 mtime: c.uint.decode(state),
139 op: c.uint8.decode(state)
140 }
141 }
142}
143
144// Message 2: file-request — request a file transfer
145export const fileRequest = {
146 preencode (state, m) {
147 c.string.preencode(state, m.path)
148 c.buffer.preencode(state, toBuffer32(m.contentHash))
149 c.buffer.preencode(state, m.chunksHave || Buffer.alloc(0))
150 },
151 encode (state, m) {
152 c.string.encode(state, m.path)
153 c.buffer.encode(state, toBuffer32(m.contentHash))
154 c.buffer.encode(state, m.chunksHave || Buffer.alloc(0))
155 },
156 decode (state) {
157 return {
158 path: c.string.decode(state),
159 contentHash: c.buffer.decode(state).toString('hex'),
160 chunksHave: c.buffer.decode(state)
161 }
162 }
163}
164
165// ── Transfer Messages (3-8) ──────────────────────────────────
166
167// Message 3: chunk-hashes — chunk hash list for a requested file
168//
169// `more` (paging flag) is appended last. A very large file's
170// chunk list serializes past the 16 MiB-1 Noise frame limit
171// (@hyperswarm/secret-stream MAX_ATOMIC_WRITE), so the protocol splits it into
172// several chunkHashes frames: every page but the last sets more:1, the final
173// page sets more:0. The receiver concatenates pages (in arrival order, keyed by
174// path) before dispatching. Appended last so a pre-paging peer that omits it
175// decodes more:0 — i.e. as a single, complete page (back-compatible).
176export const chunkHashes = {
177 preencode (state, m) {
178 c.string.preencode(state, m.path)
179 c.uint8.preencode(state, m.tier)
180 chunkInfoArray.preencode(state, m.chunks)
181 c.uint8.preencode(state, m.more || 0)
182 },
183 encode (state, m) {
184 c.string.encode(state, m.path)
185 c.uint8.encode(state, m.tier)
186 chunkInfoArray.encode(state, m.chunks)
187 c.uint8.encode(state, m.more || 0)
188 },
189 decode (state) {
190 const path = c.string.decode(state)
191 const tier = c.uint8.decode(state)
192 const chunks = chunkInfoArray.decode(state)
193 // A frame from a pre-paging peer has no trailing byte → more:0.
194 const more = state.start < state.end ? c.uint8.decode(state) : 0
195 return { path, tier, chunks, more }
196 }
197}
198
199// Message 4: chunk-need — tell sender which chunks to send
200export const chunkNeed = {
201 preencode (state, m) {
202 c.string.preencode(state, m.path)
203 uintArray.preencode(state, m.indices)
204 },
205 encode (state, m) {
206 c.string.encode(state, m.path)
207 uintArray.encode(state, m.indices)
208 },
209 decode (state) {
210 return {
211 path: c.string.decode(state),
212 indices: uintArray.decode(state)
213 }
214 }
215}
216
217// Message 5: chunk-data — send chunk bytes
218export const chunkData = {
219 preencode (state, m) {
220 c.string.preencode(state, m.path)
221 c.uint.preencode(state, m.index)
222 c.buffer.preencode(state, m.data)
223 },
224 encode (state, m) {
225 c.string.encode(state, m.path)
226 c.uint.encode(state, m.index)
227 c.buffer.encode(state, m.data)
228 },
229 decode (state) {
230 return {
231 path: c.string.decode(state),
232 index: c.uint.decode(state),
233 data: c.buffer.decode(state)
234 }
235 }
236}
237
238// Message 6: chunk-cancel — cancel a pending transfer
239export const chunkCancel = {
240 preencode (state, m) { c.string.preencode(state, m.path) },
241 encode (state, m) { c.string.encode(state, m.path) },
242 decode (state) { return { path: c.string.decode(state) } }
243}
244
245// Message 7: transfer-complete — confirm file fully received
246export const transferComplete = {
247 preencode (state, m) {
248 c.string.preencode(state, m.path)
249 c.buffer.preencode(state, toBuffer32(m.contentHash))
250 },
251 encode (state, m) {
252 c.string.encode(state, m.path)
253 c.buffer.encode(state, toBuffer32(m.contentHash))
254 },
255 decode (state) {
256 return {
257 path: c.string.decode(state),
258 contentHash: c.buffer.decode(state).toString('hex')
259 }
260 }
261}
262
263// Message 8: conflict — notify peer of a detected conflict
264export const conflict = {
265 preencode (state, m) {
266 c.string.preencode(state, m.path)
267 c.buffer.preencode(state, toBuffer32(m.myHash))
268 c.buffer.preencode(state, toBuffer32(m.theirHash))
269 c.buffer.preencode(state, toBuffer32(m.ancestorHash || Buffer.alloc(32)))
270 },
271 encode (state, m) {
272 c.string.encode(state, m.path)
273 c.buffer.encode(state, toBuffer32(m.myHash))
274 c.buffer.encode(state, toBuffer32(m.theirHash))
275 c.buffer.encode(state, toBuffer32(m.ancestorHash || Buffer.alloc(32)))
276 },
277 decode (state) {
278 return {
279 path: c.string.decode(state),
280 myHash: c.buffer.decode(state).toString('hex'),
281 theirHash: c.buffer.decode(state).toString('hex'),
282 ancestorHash: c.buffer.decode(state).toString('hex')
283 }
284 }
285}
286
287// ── Tree Messages (9-11) — 0.5a extension ────────────────────
288
289// Encode/decode an array of tree entries: { kind, exec, name, childHash, size }
290const treeEntryArray = {
291 preencode (state, arr) {
292 c.uint32.preencode(state, arr.length)
293 for (const e of arr) {
294 c.uint8.preencode(state, e.kind)
295 c.uint8.preencode(state, e.exec || 0)
296 c.string.preencode(state, e.name)
297 state.end += 32 // childHash fixed32
298 c.uint.preencode(state, e.size || 0)
299 }
300 },
301 encode (state, arr) {
302 c.uint32.encode(state, arr.length)
303 for (const e of arr) {
304 c.uint8.encode(state, e.kind)
305 c.uint8.encode(state, e.exec || 0)
306 c.string.encode(state, e.name)
307 toBuffer32(e.childHash).copy(state.buffer, state.start, 0, 32)
308 state.start += 32
309 c.uint.encode(state, e.size || 0)
310 }
311 },
312 decode (state) {
313 const len = c.uint32.decode(state)
314 const arr = []
315 for (let i = 0; i < len; i++) {
316 const kind = c.uint8.decode(state)
317 const exec = c.uint8.decode(state)
318 const name = c.string.decode(state)
319 const childHash = state.buffer.subarray(state.start, state.start + 32).toString('hex')
320 state.start += 32
321 const size = c.uint.decode(state)
322 arr.push({ kind, exec, name, childHash, size })
323 }
324 return arr
325 }
326}
327
328// Message 9: tree-request — ask peer for a tree by its content hash
329//
330// `nonce` (per-request id) is appended last; the holder echoes it in every
331// treeResponse page so the requester can drop pages from a superseded attempt.
332// A pre-nonce peer omits it and decodes 0, which the requester treats as
333// "unverifiable".
334export const treeRequest = {
335 preencode (state, m) {
336 c.buffer.preencode(state, toBuffer32(m.hash))
337 c.uint.preencode(state, m.nonce || 0)
338 },
339 encode (state, m) {
340 c.buffer.encode(state, toBuffer32(m.hash))
341 c.uint.encode(state, m.nonce || 0)
342 },
343 decode (state) {
344 const hash = c.buffer.decode(state).toString('hex')
345 const nonce = state.start < state.end ? c.uint.decode(state) : 0
346 return { hash, nonce }
347 }
348}
349
350// Message 10: tree-response — return a tree's entries to the requester
351//
352// `more` (paging flag) mirrors chunkHashes: a tree too large for one 16 MiB-1
353// Noise frame is split into several frames, every page but the last more:1.
354// `nonce` echoes the requester's per-request id so pages from a superseded
355// attempt are dropped. Both are appended last; a pre-paging peer omits them and
356// decodes 0.
357export const treeResponse = {
358 preencode (state, m) {
359 c.buffer.preencode(state, toBuffer32(m.hash))
360 treeEntryArray.preencode(state, m.entries)
361 c.uint8.preencode(state, m.more || 0)
362 c.uint.preencode(state, m.nonce || 0)
363 },
364 encode (state, m) {
365 c.buffer.encode(state, toBuffer32(m.hash))
366 treeEntryArray.encode(state, m.entries)
367 c.uint8.encode(state, m.more || 0)
368 c.uint.encode(state, m.nonce || 0)
369 },
370 decode (state) {
371 const hash = c.buffer.decode(state).toString('hex')
372 const entries = treeEntryArray.decode(state)
373 const more = state.start < state.end ? c.uint8.decode(state) : 0
374 const nonce = state.start < state.end ? c.uint.decode(state) : 0
375 return { hash, entries, more, nonce }
376 }
377}
378
379// Message 11: content-request — fetch a file by its content hash (not path)
380// Sender locates any local file matching the hash and serves its chunks.
381// `from` carries the requester's profile-key (hex) so the
382// holder's serveAuthorizer can authenticate the asker. Appended last → an
383// overlay peer that predates this field decodes '' (empty), never crashes.
384export const contentRequest = {
385 preencode (state, m) {
386 c.buffer.preencode(state, toBuffer32(m.contentHash))
387 c.buffer.preencode(state, m.chunksHave || Buffer.alloc(0))
388 c.string.preencode(state, m.from || '')
389 },
390 encode (state, m) {
391 c.buffer.encode(state, toBuffer32(m.contentHash))
392 c.buffer.encode(state, m.chunksHave || Buffer.alloc(0))
393 c.string.encode(state, m.from || '')
394 },
395 decode (state) {
396 return {
397 contentHash: c.buffer.decode(state).toString('hex'),
398 chunksHave: c.buffer.decode(state),
399 from: c.string.decode(state)
400 }
401 }
402}
403
404// Message 12: transfer-control — downloader→holder pause/stop notice for a
405// content-addressed fetch. state: 0 = paused, 1 = stopped. Appended last so a
406// holder that predates this never registers slot 12 and silently ignores it.
407export const transferControl = {
408 preencode (state, m) {
409 c.buffer.preencode(state, toBuffer32(m.contentHash))
410 c.uint8.preencode(state, m.state)
411 },
412 encode (state, m) {
413 c.buffer.encode(state, toBuffer32(m.contentHash))
414 c.uint8.encode(state, m.state)
415 },
416 decode (state) {
417 return {
418 contentHash: c.buffer.decode(state).toString('hex'),
419 state: c.uint8.decode(state)
420 }
421 }
422}
423
424// Message 13: transfer-progress — downloader→holder one-shot have-baseline for a
425// resumed content-addressed fetch. `have` = bytes the downloader already holds on
426// disk at resume, so the holder's "who is downloading" bar reflects the downloader's
427// TRUE completion, not only the bytes it re-serves. Appended last so a holder that
428// predates this never registers slot 13 and silently ignores it.
429export const transferProgress = {
430 preencode (state, m) {
431 c.buffer.preencode(state, toBuffer32(m.contentHash))
432 c.uint.preencode(state, m.have || 0)
433 },
434 encode (state, m) {
435 c.buffer.encode(state, toBuffer32(m.contentHash))
436 c.uint.encode(state, m.have || 0)
437 },
438 decode (state) {
439 return {
440 contentHash: c.buffer.decode(state).toString('hex'),
441 have: c.uint.decode(state)
442 }
443 }
444}
1// SHIM: v0.3.0 engine lives in lib/v3/ (Ok mirror). Re-export under our name/subpath.
2export * from './v3/messages.js'
lib/overlay-v2.js +2 -308
……@@ -1,308 +1,2 @@
1/**
2 * HyperOverlayV2 — content-addressed file serving over the V2 protocol.
3 *
4 * A thin facade over the V2 stack (FileIndex + SyncEngine + TransferManager +
5 * OverlayProtocolV2) for the "serve a file to whoever asks, fetch a file from
6 * whoever has it" use case — distinct from V1's local chunk-registry blobs.
7 *
8 * Model (the deliberate post-Hyperdrive design): the canonical bytes are a
9 * REAL FILE ON DISK. The index holds only metadata + chunk maps (no blob
10 * bytes), so there is no hypercore storage balloon. A peer fetches a file by
11 * its content hash; chunks are read from the source file on demand, streamed,
12 * hash-verified, and written to the requester's disk.
13 *
14 * `autoSync` is OFF: this is on-demand content fetch (requestContent →
15 * _onContentRequest), NOT Syncthing-style folder push. Registering a file
16 * makes it servable; nothing is offered to peers unsolicited.
17 *
18 * Requester flow mirrors pgh's NetworkFetcher.getContent (production-proven):
19 * pre-register a synthetic `content:<hash>` → destPath mapping, requestContent,
20 * await the hash-verified transferComplete, walk peers on timeout/mismatch.
21 */
22
23import ReadyResource from 'ready-resource'
24import fs from '#fs'
25import path from '#path'
26import os from '#os'
27import crypto from 'hypercore-crypto'
28
29import { FileIndex } from './file-index.js'
30import { SyncEngine } from './sync-engine.js'
31import { TransferManager } from './transfer.js'
32import { OverlayProtocolV2 } from './protocol-v2.js'
33import { createStreamingHasher } from './chunker.js'
34
35const DEFAULT_PER_PEER_TIMEOUT = 30000
36const DEFAULT_IDLE_TIMEOUT = 10000
37
38export class HyperOverlayV2 extends ReadyResource {
39 constructor (corestore, opts = {}) {
40 super()
41 this._corestore = corestore.namespace(opts.namespace || 'overlay-v2')
42 this._destDir = opts.destDir || path.join(os.tmpdir(), 'hyper-overlay-v2-fetch')
43 this._syncBaseDirs = opts.syncBaseDirs || []
44 this._perPeerTimeout = opts.perPeerTimeout || DEFAULT_PER_PEER_TIMEOUT
45 this._idleTimeout = opts.idleTimeout || DEFAULT_IDLE_TIMEOUT
46
47 // Shared maps the protocol reads: overlayPath → diskPath (for serving an
48 // offered/known path), contentHash → diskPath (fast path for content
49 // requests). Both populated by registerFile.
50 this._filePaths = new Map()
51 this._contentHashPaths = new Map()
52
53 // contentHash → pending fetch { destPath, finish, timers, bytesReceived }
54 this._pendingContent = new Map()
55
56 // Built lazily on first use (Hyperdrive-style: a drive that never touches
57 // its blobs core never opens one). A daemon that never registers, serves,
58 // or connects a peer for attachments pays nothing — no feed, no index.
59 this._index = null
60 this._sync = null
61 this._transfer = null
62 this._protocol = null
63 this._stackPromise = null
64 }
65
66 // The protocol must exist synchronously when attachProtocol() is called —
67 // protomux will NOT pair a channel created after the remote opened theirs
68 // (verified). So the stack is built at ready() time. _ensure() stays as the
69 // idempotent builder (used by register/fetch, and as a safety net).
70 async _open () { await this._ensure() }
71
72 async _close () {
73 // Only tear down what was actually built.
74 if (this._protocol) this._protocol.destroy()
75 if (this._sync) await this._sync.close()
76 if (this._index) await this._index.close()
77 }
78
79 // Lazily build the V2 stack. Idempotent + concurrency-safe (a single
80 // in-flight promise is shared). No-op cost once built.
81 _ensure () {
82 if (this._protocol) return Promise.resolve()
83 if (this._stackPromise) return this._stackPromise
84 this._stackPromise = (async () => {
85 const index = new FileIndex(this._corestore)
86 await index.ready()
87 const sync = new SyncEngine(index, this._corestore)
88 await sync.ready()
89 const transfer = new TransferManager(index)
90 const protocol = new OverlayProtocolV2(sync, transfer, {
91 filePaths: this._filePaths,
92 contentHashPaths: this._contentHashPaths,
93 syncBaseDirs: this._syncBaseDirs,
94 autoSync: false,
95 onSynced: (info) => this._onSynced(info),
96 onChunkProgress: (info) => this._onChunkProgress(info)
97 })
98 try { fs.mkdirSync(this._destDir, { recursive: true }) } catch {}
99 this._index = index
100 this._sync = sync
101 this._transfer = transfer
102 this._protocol = protocol
103 })()
104 return this._stackPromise
105 }
106
107 // hyper-svc's swarm transport calls this synchronously per connection. The
108 // channel MUST be created synchronously — protomux will not pair a channel
109 // opened after the remote's (verified). The stack is built at ready(), so
110 // this._protocol is always present by the time connections arrive.
111 attachProtocol (mux) {
112 if (this.closing || this.closed || !this._protocol) return
113 return this._protocol.attach(mux)
114 }
115
116 get peerCount () { return this._protocol ? this._protocol.peerCount : 0 }
117
118 /**
119 * Make an on-disk file servable by its content hash.
120 * @param {string} overlayPath logical path, e.g. '/attachments/<oid>'
121 * @param {string} diskPath absolute path to the canonical bytes on disk
122 * @param {object} [meta]
123 * meta.contentHash preferred — the oid the caller already computed
124 * meta.size file size (avoids an extra stat if known)
125 * @returns {Promise<{ contentHash, size }>}
126 */
127 async registerFile (overlayPath, diskPath, meta = {}) {
128 await this._ensure()
129 const stat = fs.statSync(diskPath)
130 const size = typeof meta.size === 'number' ? meta.size : stat.size
131 const contentHash = meta.contentHash || await hashFileOnDisk(diskPath, size)
132
133 await this._index.putFile(overlayPath, { contentHash, size, mtime: stat.mtimeMs })
134 this._filePaths.set(overlayPath, diskPath)
135 this._contentHashPaths.set(contentHash, diskPath)
136 // Build (and cache) the chunk map so a content request can stream
137 // immediately without re-chunking under the request.
138 await this._transfer.prepareFile(diskPath, overlayPath)
139 // Record in the local change feed (used by the protocol's markSynced
140 // bookkeeping; with autoSync off it is not broadcast).
141 await this._sync.logPut(overlayPath, { contentHash, size, mtime: stat.mtimeMs })
142 return { contentHash, size }
143 }
144
145 /**
146 * Fetch a file by its content hash. Serves from the local disk copy if we
147 * have it; otherwise pulls from a connected peer that does (hash-verified).
148 * @param {string} contentHash
149 * @param {object} [opts] opts.timeout (overall), opts.destPath
150 * @returns {Promise<{ data: Buffer, destPath: string, local: boolean } | null>}
151 */
152 async fetchFile (contentHash, opts = {}) {
153 await this._ensure()
154
155 const localDisk = this._contentHashPaths.get(contentHash)
156 if (localDisk) {
157 try {
158 const data = fs.readFileSync(localDisk)
159 if (crypto.data(data).toString('hex') === contentHash) {
160 return { data, destPath: localDisk, local: true }
161 }
162 } catch {}
163 }
164
165 // Wait briefly for at least one peer — a connection may be mid-handshake
166 // (lazy attach) or just forming when the fetch is issued. Without this a
167 // fetch issued the instant a peer connects would spuriously 404.
168 const peerWaitMs = opts.peerWaitMs || 3000
169 for (let waited = 0; this._protocol._peers.size === 0 && waited < peerWaitMs; waited += 100) {
170 await new Promise(r => setTimeout(r, 100))
171 }
172 if (this._protocol._peers.size === 0) return null
173
174 // Multi-source: fetch chunks in parallel from ALL connected peers that
175 // have the file (torrent-style). The scheduler dedups + fails over.
176 const peers = [...this._protocol._peers.values()]
177 // opts.mirrorSpool: store the verified blob CONTENT-ADDRESSED at
178 // <mirrorSpool>/<contentHash> rather than a throwaway destDir name. This is
179 // the overlay-mirror path — a forge backend with no working tree fetches a
180 // repo's large files into a content-addressed spool so it durably re-serves
181 // them (via the <syncBaseDir>/<contentHash> resolution in _onContentRequest).
182 let destPath
183 if (opts.destPath) {
184 destPath = opts.destPath
185 } else if (opts.mirrorSpool) {
186 try { fs.mkdirSync(opts.mirrorSpool, { recursive: true }) } catch {}
187 destPath = path.join(opts.mirrorSpool, contentHash)
188 } else {
189 destPath = path.join(
190 this._destDir,
191 'blob-' + contentHash + '-' + Date.now() + '-' + Math.random().toString(36).slice(2, 8)
192 )
193 }
194 try {
195 await this._protocol.fetchContent(contentHash, peers, {
196 destPath,
197 timeout: opts.timeout,
198 onProgress: () => {}
199 })
200 } catch {
201 return null
202 }
203 // Whole-file integrity: the scheduler verifies each chunk's hash, but the
204 // chunk-list itself came from a peer — verify the assembled file hashes to
205 // the requested content hash before trusting it.
206 let data
207 try { data = fs.readFileSync(destPath) } catch { return null }
208 if (crypto.data(data).toString('hex') !== contentHash) {
209 const err = new Error('content-hash mismatch after multi-source fetch')
210 err.code = 'EHASHMISMATCH'
211 throw err
212 }
213 // Seeding multiplication: register the verified blob locally so this node
214 // now SERVES these chunks to other peers too. A file held by N peers then
215 // gets faster as more peers fetch it (torrent-style). Best-effort — the
216 // fetch still succeeds even if re-registration fails. Skipped if a peer
217 // raced us to register the same hash (registerFile is idempotent-safe).
218 if (!this._contentHashPaths.has(contentHash)) {
219 try {
220 await this.registerFile('/attachments/' + contentHash, destPath, { contentHash, size: data.length })
221 } catch {}
222 }
223 return { data, destPath, local: false }
224 }
225
226 _destPathFor (contentHash) {
227 return path.join(this._destDir, 'blob-' + contentHash)
228 }
229
230 _fetchFromPeer (peer, contentHash, opts = {}) {
231 const syntheticPath = 'content:' + contentHash
232 const destPath = path.join(
233 this._destDir,
234 'blob-' + contentHash + '-' + Date.now() + '-' + Math.random().toString(36).slice(2, 8)
235 )
236 // The protocol's chunk handlers write to whatever _filePaths maps the
237 // synthetic path to. Per-attempt destPath so concurrent peer tries don't
238 // clobber one another.
239 this._filePaths.set(syntheticPath, destPath)
240
241 return new Promise((resolve, reject) => {
242 let settled = false
243 const finish = (err, result) => {
244 if (settled) return
245 settled = true
246 clearTimeout(timer)
247 clearTimeout(pending.idleTimer)
248 this._pendingContent.delete(contentHash)
249 if (err) reject(err); else resolve(result)
250 }
251 const timer = setTimeout(() => {
252 finish(new Error('content request timed out: ' + contentHash.slice(0, 12)))
253 }, opts.timeout || this._perPeerTimeout)
254 const idleFire = () => finish(new Error('peer went silent mid-stream: ' + contentHash.slice(0, 12)))
255 const pending = {
256 syntheticPath, destPath, finish, peer, contentHash,
257 bytesReceived: 0,
258 idleTimer: setTimeout(idleFire, this._idleTimeout),
259 idleFire
260 }
261 this._pendingContent.set(contentHash, pending)
262 this._protocol.requestContent(peer, contentHash, null)
263 })
264 }
265
266 _onChunkProgress (info) {
267 if (!info || !info.path || !info.path.startsWith('content:')) return
268 const contentHash = info.path.slice('content:'.length)
269 const pending = this._pendingContent.get(contentHash)
270 if (!pending) return
271 pending.bytesReceived += (info.bytes || 0)
272 clearTimeout(pending.idleTimer)
273 pending.idleTimer = setTimeout(pending.idleFire, this._idleTimeout)
274 }
275
276 // The V2 chunk flow does NOT verify the reassembled file against the
277 // advertised hash (that's the caller's job in this protocol). We do it here
278 // so fetchFile never returns un-verified bytes.
279 _onSynced (info) {
280 if (!info || info.direction !== 'pull') return
281 if (!info.path || !info.path.startsWith('content:')) return
282 const contentHash = info.path.slice('content:'.length)
283 const pending = this._pendingContent.get(contentHash)
284 if (!pending) return
285 try {
286 const data = fs.readFileSync(pending.destPath)
287 const actual = crypto.data(data).toString('hex')
288 if (actual !== contentHash) {
289 pending.finish(Object.assign(
290 new Error('content-hash mismatch: expected ' + contentHash.slice(0, 12) + '…, got ' + actual.slice(0, 12) + '…'),
291 { code: 'EHASHMISMATCH', expected: contentHash, actual }
292 ))
293 return
294 }
295 pending.finish(null, data)
296 } catch (err) {
297 pending.finish(err)
298 }
299 }
300}
301
302// Stream a file through the chunker's hasher to get its oid without buffering
303// the whole file in memory (matches /api/upload's createStreamingHasher oid).
304async function hashFileOnDisk (diskPath, size) {
305 const hasher = createStreamingHasher({ size })
306 for await (const buf of fs.createReadStream(diskPath)) hasher.update(buf)
307 return hasher.digest()
308}
1// SHIM -> lib/v3 engine. HyperOverlay(v3) exposed under our HyperOverlayV2 name.
2export { HyperOverlay as HyperOverlayV2, HyperOverlay } from './v3/overlay.js'
lib/protocol-v2.js +2 -648
……@@ -1,648 +1,2 @@
1/**
2 * OverlayProtocolV2 — Protomux channel for sync + transfer
3 *
4 * Sync approach: each side sends its local feed entries directly via
5 * file-offer messages. No Hypercore feed replication needed — the feed
6 * entries are small JSON objects sent over our own Protomux channel.
7 *
8 * Flow:
9 * 1. Channel opens → both send sync-state (feed length)
10 * 2. Side with more entries sends file-offers for each change
11 * 3. Receiver auto-requests files it doesn't have
12 * 4. Sender chunks on demand → streams chunks → receiver writes to disk
13 */
14
15import path from '#path'
16import * as messages from './messages-v2.js'
17import { hashChunk } from './chunker.js'
18import { ChunkScheduler } from './chunk-scheduler.js'
19import fs from '#fs'
20
21const PROTOCOL = 'hyper-overlay/v2'
22const VERSION = 2
23const CAP_LOCAL_FILES = 0x01
24const CAP_ADAPTIVE_CHUNKS = 0x02
25
26export class OverlayProtocolV2 {
27 constructor (syncEngine, transferManager, opts = {}) {
28 this._syncEngine = syncEngine
29 this._transferManager = transferManager
30 this._corestore = opts.corestore || null
31 this._overlayKey = opts.key || null
32 this._peers = new Map()
33 this._filePaths = opts.filePaths || new Map()
34 this._syncBaseDirs = opts.syncBaseDirs || []
35 this._defaultStrategy = opts.defaultStrategy || 'prompt'
36 this._autoSync = opts.autoSync !== false
37
38 this._conflictCb = opts.onConflict || null
39 this._syncedCb = opts.onSynced || null
40 this._fileOfferCb = opts.onFileOffer || null
41 // Called for every chunk that arrives on this peer. Lets callers
42 // (e.g. NetworkFetcher) implement silent-peer watchdogs by resetting
43 // an idle timer on every chunk. Receives { path, index, bytes, peer }.
44 this._chunkProgressCb = opts.onChunkProgress || null
45
46 // 0.5a: content-hash → local disk path, for responding to content-requests
47 this._contentHashPaths = opts.contentHashPaths || new Map()
48 // 0.5a: tree-request timeout (ms)
49 this._treeRequestTimeout = opts.treeRequestTimeout || 30000
50 // Multi-source: synthetic path 'content:<hash>' → ChunkScheduler. When set,
51 // _onChunkHashes/_onChunkData delegate to the scheduler (parallel fetch
52 // across peers) instead of the legacy single-peer auto-flow.
53 this._schedulers = new Map()
54 }
55
56 /**
57 * Multi-source fetch of a file by content hash from several peers in
58 * parallel (torrent-style). The caller pre-registers _filePaths['content:'
59 * + contentHash] = destPath. Resolves when the file is assembled + verified.
60 *
61 * @param {string} contentHash
62 * @param {Array} peers peer objects to fan the request to
63 * @param {object} [opts] { destPath, cap, timeout, onProgress }
64 * @returns {Promise<void>}
65 */
66 fetchContent (contentHash, peers, opts = {}) {
67 const p = 'content:' + contentHash
68 if (this._schedulers.has(p)) return Promise.reject(new Error('already fetching ' + contentHash))
69 const sched = new ChunkScheduler({
70 path: p,
71 destPath: opts.destPath,
72 transfer: this._transferManager,
73 sendNeed: (peer, indices) => peer.msgs.chunkNeed.send({ path: p, indices }),
74 cap: opts.cap,
75 timeout: opts.timeout,
76 onProgress: opts.onProgress
77 })
78 this._schedulers.set(p, sched)
79 const promise = sched.promise().finally(() => this._schedulers.delete(p))
80 for (const peer of peers) this.requestContent(peer, contentHash, null)
81 return promise
82 }
83
84 setContentHashPaths (map) { this._contentHashPaths = map }
85
86 attach (mux) {
87 if (this._peers.has(mux)) return this._peers.get(mux)
88
89 const self = this
90
91 const channel = mux.createChannel({
92 protocol: PROTOCOL,
93 id: this._overlayKey || null,
94 onopen () {
95 const peer = self._peers.get(mux)
96 if (peer) self._onOpen(peer)
97 },
98 onclose () {
99 const peer = self._peers.get(mux)
100 self._peers.delete(mux)
101 // Failover: let any active multi-source fetch reassign this peer's
102 // inflight chunks to the remaining peers.
103 if (peer) for (const sched of self._schedulers.values()) sched.removePeer(peer)
104 }
105 })
106
107 if (!channel) return null
108
109 const peer = {
110 mux, channel, remoteFeedKey: null, remoteSeq: 0, msgs: {},
111 pendingTrees: new Map() // 0.5a: hash → { resolve, reject, timer }
112 }
113
114 peer.msgs.syncState = channel.addMessage({ encoding: messages.syncState, onmessage: (msg) => self._onSyncState(peer, msg) })
115 peer.msgs.fileOffer = channel.addMessage({ encoding: messages.fileOffer, onmessage: (msg) => self._onFileOffer(peer, msg) })
116 peer.msgs.fileRequest = channel.addMessage({ encoding: messages.fileRequest, onmessage: (msg) => self._onFileRequest(peer, msg) })
117 peer.msgs.chunkHashes = channel.addMessage({ encoding: messages.chunkHashes, onmessage: (msg) => self._onChunkHashes(peer, msg) })
118 peer.msgs.chunkNeed = channel.addMessage({ encoding: messages.chunkNeed, onmessage: (msg) => self._onChunkNeed(peer, msg) })
119 peer.msgs.chunkData = channel.addMessage({ encoding: messages.chunkData, onmessage: (msg) => self._onChunkData(peer, msg) })
120 peer.msgs.chunkCancel = channel.addMessage({ encoding: messages.chunkCancel, onmessage: (msg) => self._onChunkCancel(peer, msg) })
121 peer.msgs.transferComplete = channel.addMessage({ encoding: messages.transferComplete, onmessage: (msg) => self._onTransferComplete(peer, msg) })
122 peer.msgs.conflict = channel.addMessage({ encoding: messages.conflict, onmessage: (msg) => self._onConflictMsg(peer, msg) })
123
124 // 0.5a — tree messages (backward-compatible additions)
125 peer.msgs.treeRequest = channel.addMessage({ encoding: messages.treeRequest, onmessage: (msg) => self._onTreeRequest(peer, msg) })
126 peer.msgs.treeResponse = channel.addMessage({ encoding: messages.treeResponse, onmessage: (msg) => self._onTreeResponse(peer, msg) })
127 peer.msgs.contentRequest = channel.addMessage({ encoding: messages.contentRequest, onmessage: (msg) => self._onContentRequest(peer, msg) })
128
129 this._peers.set(mux, peer)
130 channel.open({ version: VERSION, capabilities: CAP_LOCAL_FILES | CAP_ADAPTIVE_CHUNKS })
131 return peer
132 }
133
134 detach (mux) {
135 const peer = this._peers.get(mux)
136 if (!peer) return
137 peer.channel.close()
138 this._peers.delete(mux)
139 }
140
141 setFilePaths (map) { this._filePaths = map }
142
143 broadcastOffer (overlayPath, meta, op) {
144 const msg = {
145 path: overlayPath,
146 contentHash: meta.contentHash || '0'.repeat(64),
147 size: meta.size || 0,
148 mtime: meta.mtime || 0,
149 op
150 }
151 for (const [, peer] of this._peers) {
152 peer.msgs.fileOffer.send(msg)
153 }
154 }
155
156 requestFile (peer, overlayPath, contentHash, chunksHave) {
157 peer.msgs.fileRequest.send({
158 path: overlayPath,
159 contentHash,
160 chunksHave: chunksHave || null
161 })
162 }
163
164 /**
165 * 0.5a: Request a tree by its hash. Returns a promise resolving to
166 * the entries array (or [] if the peer doesn't have it).
167 *
168 * @param {object} peer
169 * @param {string} hash - hex tree hash
170 * @param {object} [opts]
171 * @param {number} [opts.timeout] - override default timeout (ms)
172 * @returns {Promise<Array<{kind, exec, name, childHash, size}>>}
173 */
174 requestTree (peer, hash, opts = {}) {
175 return new Promise((resolve, reject) => {
176 // If a prior request is already in-flight for the same hash, reject
177 // this caller rather than silently overwrite.
178 if (peer.pendingTrees.has(hash)) {
179 return reject(new Error('tree request already pending for hash: ' + hash))
180 }
181 const timer = setTimeout(() => {
182 const pending = peer.pendingTrees.get(hash)
183 if (pending) {
184 peer.pendingTrees.delete(hash)
185 reject(new Error('tree request timed out: ' + hash))
186 }
187 }, opts.timeout || this._treeRequestTimeout)
188 peer.pendingTrees.set(hash, { resolve, reject, timer })
189 peer.msgs.treeRequest.send({ hash })
190 })
191 }
192
193 /**
194 * 0.5a: Request a file by its content hash. Peer looks up any file
195 * matching that hash locally and streams chunks via the existing flow
196 * (chunkHashes → chunkNeed → chunkData). Returns immediately; chunks
197 * arrive asynchronously.
198 *
199 * Note: receiver must pre-register a target path mapping via
200 * _filePaths keyed by the contentHash before calling, OR this method
201 * is a no-op on the receive side. Full wiring deferred to 0.5c.
202 */
203 requestContent (peer, contentHash, chunksHave) {
204 peer.msgs.contentRequest.send({
205 contentHash,
206 chunksHave: chunksHave || null
207 })
208 }
209
210 get peerCount () { return this._peers.size }
211
212 destroy () {
213 for (const [, peer] of this._peers) {
214 for (const p of peer.pendingTrees.values()) clearTimeout(p.timer)
215 peer.channel.close()
216 }
217 this._peers.clear()
218 }
219
220 // ── Handlers ────────────────────────────────────────────────
221
222 _onOpen (peer) {
223 console.log(' [proto] channel opened, sending sync state')
224 peer.msgs.syncState.send({
225 feedKey: this._syncEngine.feedKey,
226 localSeq: this._syncEngine.feed.length,
227 remoteSeq: 0
228 })
229 }
230
231 async _onSyncState (peer, msg) {
232 console.log(' [proto] received sync state from', msg.feedKey.slice(0, 16) + '..., seq:', msg.localSeq)
233 peer.remoteFeedKey = msg.feedKey
234 peer.remoteSeq = msg.localSeq
235
236 if (!this._autoSync) return
237
238 try {
239 await this._offerLocalFiles(peer)
240 } catch (err) {
241 console.log(' [proto] offerLocalFiles error:', err.message, err.stack)
242 }
243 }
244
245 async _onFileOffer (peer, msg) {
246 console.log(' [proto] received file offer:', msg.path, 'op:', msg.op)
247
248 if (this._fileOfferCb) this._fileOfferCb(peer, msg)
249
250 if (msg.op === 0) {
251 const local = await this._syncEngine._fileIndex.getFile(msg.path)
252 if (!local || local.contentHash !== msg.contentHash) {
253 const diskPath = this._resolveFilePath(msg.path)
254 console.log(' [proto] requesting:', msg.path, '→', diskPath || 'NO PATH')
255 if (diskPath) {
256 this._filePaths.set(msg.path, diskPath)
257 this.requestFile(peer, msg.path, msg.contentHash)
258 }
259 } else {
260 console.log(' [proto] already have:', msg.path)
261 }
262 }
263 }
264
265 async _onFileRequest (peer, msg) {
266 console.log(' [proto] file requested:', msg.path)
267 const diskPath = this._filePaths.get(msg.path)
268 if (!diskPath) { console.log(' [proto] no disk path for:', msg.path); return }
269
270 const prepared = await this._transferManager.prepareFile(diskPath, msg.path)
271 if (!prepared) { console.log(' [proto] prepareFile failed for:', msg.path); return }
272
273 if (!await this._transferManager._fileIndex.hasChunkMap(msg.path)) {
274 await this._transferManager._fileIndex.putChunkMap(msg.path, prepared.chunks)
275 }
276
277 console.log(' [proto] sending', prepared.chunks.length, 'chunk hashes for', msg.path)
278 // [diag-2026-05-04]
279 if (prepared.chunks.length > 1000) {
280 // eslint-disable-next-line no-console
281 console.log('[v2-diag] chunkHashes path=' + msg.path + ' chunks=' + prepared.chunks.length + ' size~=' + (prepared.chunks.length * 80))
282 }
283 peer.msgs.chunkHashes.send({ path: msg.path, tier: prepared.tier, chunks: prepared.chunks })
284 }
285
286 async _onChunkHashes (peer, msg) {
287 // Multi-source: if a scheduler owns this path, let it distribute chunk
288 // requests across peers instead of the legacy single-peer auto-flow.
289 const sched = this._schedulers.get(msg.path)
290 if (sched) { sched.onChunkHashes(peer, msg.chunks); return }
291
292 const diskPath = this._filePaths.get(msg.path)
293 if (!diskPath) return
294
295 // Empty-file fast path: a 0-byte file has no chunks to fetch, but we
296 // still need to materialize it on disk and fire the same completion
297 // flow as _onChunkData's "transfer complete" branch — otherwise the
298 // requester (e.g. NetworkFetcher.getContent) never resolves and times
299 // out. Empty files are common in practice: .keep, __init__.py, build
300 // stamps, touch'd markers. Bug symptom before this: node-gyp's
301 // postinstall.stamp (0 bytes, hash 5187b7a8…) hung every install.
302 if (msg.chunks.length === 0) {
303 try {
304 fs.mkdirSync(path.dirname(diskPath), { recursive: true })
305 fs.writeFileSync(diskPath, Buffer.alloc(0))
306 } catch {
307 return
308 }
309 const contentHash = hashChunk(Buffer.alloc(0))
310 const stat = fs.statSync(diskPath)
311 await this._syncEngine._fileIndex.putFile(msg.path, {
312 contentHash, size: 0, mtime: stat.mtimeMs
313 })
314 await this._syncEngine.logPut(msg.path, {
315 contentHash, size: 0, mtime: stat.mtimeMs
316 })
317 if (peer.remoteFeedKey) {
318 await this._syncEngine.markSynced(peer.remoteFeedKey, msg.path, contentHash, this._syncEngine.feed.length - 1)
319 }
320 peer.msgs.transferComplete.send({ path: msg.path, contentHash })
321 console.log(' synced:', msg.path, '(pull, empty)')
322 if (this._syncedCb) {
323 this._syncedCb({ path: msg.path, contentHash, size: 0, direction: 'pull' })
324 }
325 return
326 }
327
328 const needed = this._transferManager.computeNeeded(msg.chunks, new Set())
329 if (needed.length === 0) return
330
331 let size = 0
332 for (const c of msg.chunks) size += c.length
333
334 this._transferManager.startReceive(diskPath, {
335 size,
336 chunks: msg.chunks.map((c, i) => {
337 let offset = 0
338 for (let j = 0; j < i; j++) offset += msg.chunks[j].length
339 return { hash: c.hash, offset, length: c.length }
340 })
341 })
342
343 peer.msgs.chunkNeed.send({ path: msg.path, indices: needed })
344 }
345
346 async _onChunkNeed (peer, msg) {
347 const diskPath = this._filePaths.get(msg.path)
348 if (!diskPath) return
349
350 const chunkMap = await this._transferManager._fileIndex.getChunkMap(msg.path)
351 if (!chunkMap) return
352
353 // Enqueue the requested chunks and pump them out with BACKPRESSURE.
354 // Pre-rewrite the bulk transfer honored stream backpressure (proven at
355 // 200MB/1.30GB with no stall); the multi-source rewrite regressed it by
356 // firing every chunkData.send() in a tight synchronous loop, ignoring
357 // send()'s drained return. On a real link that floods secret-stream's
358 // write buffer (which grows unbounded — stream.write just returns false)
359 // → stall / ETIMEDOUT / ECONNRESET on large transfers. We serialize sends
360 // per peer through _pumpSendQueue, which awaits 'drain' whenever send()
361 // reports backpressure. This composes correctly with the torrent-style
362 // ChunkScheduler: a drain-paused seeder simply looks like a slower peer,
363 // which the scheduler already handles (cap-bounded inflight + reassign).
364 if (!peer._sendQueue) peer._sendQueue = []
365 peer._sendQueue.push({ path: msg.path, diskPath, chunkMap, indices: msg.indices })
366 this._pumpSendQueue(peer)
367 }
368
369 // Serialized, backpressure-respecting chunk sender. Exactly one pump runs
370 // per peer at a time (guarded by peer._sending), so concurrent _onChunkNeed
371 // invocations for the same peer don't interleave and re-flood the socket.
372 async _pumpSendQueue (peer) {
373 if (peer._sending) return
374 peer._sending = true
375 try {
376 while (peer._sendQueue && peer._sendQueue.length) {
377 const job = peer._sendQueue.shift()
378 for (const index of job.indices) {
379 if (peer.channel && peer.channel.closed) return
380 if (index >= job.chunkMap.length) continue
381 const c = job.chunkMap[index]
382 const data = this._transferManager.readChunk(job.diskPath, c.offset, c.length)
383 if (!data) continue
384 const drained = peer.msgs.chunkData.send({ path: job.path, index, data })
385 // send() returns mux.drained (stream.write's return). false ⇒ the
386 // write buffer is full — wait for the socket to drain before the
387 // next chunk instead of piling more into an unbounded backlog.
388 if (drained === false) await this._awaitDrain(peer)
389 }
390 }
391 } finally {
392 peer._sending = false
393 }
394 }
395
396 // Resolve when the peer's underlying stream drains. Safety-timeout so a peer
397 // that never drains (dead connection) can't wedge the pump forever — on
398 // timeout we resolve and let the next send()/channel-close surface the error.
399 _awaitDrain (peer) {
400 const mux = peer.mux
401 if (!mux || mux.drained !== false) return Promise.resolve()
402 const stream = mux.stream
403 if (!stream || typeof stream.once !== 'function') return Promise.resolve()
404 return new Promise((resolve) => {
405 const cleanup = () => { try { stream.removeListener('drain', onDrain) } catch {} clearTimeout(timer) }
406 const onDrain = () => { cleanup(); resolve() }
407 const timer = setTimeout(() => { cleanup(); resolve() }, 30000)
408 stream.once('drain', onDrain)
409 })
410 }
411
412 async _onChunkData (peer, msg) {
413 // Silent-peer watchdog hook: fire BEFORE any processing so a slow disk
414 // doesn't get blamed for a peer-side stall.
415 if (this._chunkProgressCb) {
416 try {
417 this._chunkProgressCb({
418 path: msg.path,
419 index: msg.index,
420 bytes: msg.data ? msg.data.length : 0,
421 peer
422 })
423 } catch {}
424 }
425
426 // Multi-source: scheduler owns the write + reschedule + finalize.
427 const sched = this._schedulers.get(msg.path)
428 if (sched) { sched.onChunkData(peer, msg.index, msg.data); return }
429
430 const diskPath = this._filePaths.get(msg.path)
431 if (!diskPath) return
432
433 const result = this._transferManager.writeChunk(diskPath, msg.index, msg.data)
434
435 if (result.ok && this._transferManager.isComplete(diskPath)) {
436 const fin = this._transferManager.finalize(diskPath)
437 if (fin.ok) {
438 const fileData = fs.readFileSync(diskPath)
439 const contentHash = hashChunk(fileData)
440 const stat = fs.statSync(diskPath)
441
442 await this._syncEngine._fileIndex.putFile(msg.path, {
443 contentHash, size: stat.size, mtime: stat.mtimeMs
444 })
445 await this._syncEngine.logPut(msg.path, {
446 contentHash, size: stat.size, mtime: stat.mtimeMs
447 })
448
449 if (peer.remoteFeedKey) {
450 await this._syncEngine.markSynced(peer.remoteFeedKey, msg.path, contentHash, this._syncEngine.feed.length - 1)
451 }
452
453 peer.msgs.transferComplete.send({ path: msg.path, contentHash })
454 console.log(' synced:', msg.path, '(pull)')
455
456 if (this._syncedCb) {
457 this._syncedCb({ path: msg.path, contentHash, size: stat.size, direction: 'pull' })
458 }
459 }
460 }
461 }
462
463 _onChunkCancel (peer, msg) {
464 const diskPath = this._filePaths.get(msg.path)
465 if (diskPath) this._transferManager.cancel(diskPath)
466 }
467
468 async _onTransferComplete (peer, msg) {
469 if (peer.remoteFeedKey) {
470 await this._syncEngine.markSynced(peer.remoteFeedKey, msg.path, msg.contentHash, this._syncEngine.feed.length - 1)
471 }
472 console.log(' synced:', msg.path, '(push confirmed)')
473 if (this._syncedCb) {
474 this._syncedCb({ path: msg.path, contentHash: msg.contentHash, direction: 'push' })
475 }
476 }
477
478 _onConflictMsg (peer, msg) {
479 if (this._conflictCb) this._conflictCb(msg, peer)
480 }
481
482 // ── Tree handlers (0.5a) ────────────────────────────────────
483
484 async _onTreeRequest (peer, msg) {
485 let entries = []
486 try {
487 const fileIndex = this._syncEngine && this._syncEngine._fileIndex
488 if (fileIndex && typeof fileIndex.getTree === 'function') {
489 const tree = await fileIndex.getTree(msg.hash)
490 if (tree && Array.isArray(tree.entries)) entries = tree.entries
491 }
492 } catch {
493 // fall through — reply with empty entries on lookup failure
494 }
495 // [diag-2026-05-04] log size to find which message-type trips
496 // secret-stream's 16 MB MAX_ATOMIC_WRITE. Drop after diagnosis.
497 if (entries.length > 100) {
498 // eslint-disable-next-line no-console
499 console.log('[v2-diag] treeResponse hash=' + (msg.hash ? msg.hash.slice(0, 12) : '?') + ' entries=' + entries.length)
500 }
501 peer.msgs.treeResponse.send({ hash: msg.hash, entries })
502 }
503
504 _onTreeResponse (peer, msg) {
505 const pending = peer.pendingTrees.get(msg.hash)
506 if (!pending) return // late or unexpected response — drop
507 clearTimeout(pending.timer)
508 peer.pendingTrees.delete(msg.hash)
509 pending.resolve(msg.entries)
510 }
511
512 async _onContentRequest (peer, msg) {
513 // Resolve msg.contentHash → an on-disk path. Three lookup paths, in order:
514 // 1. Fast path: _contentHashPaths map (warm — just-registered / consumer-set)
515 // 2. Content-addressed spool (RESTART-DURABLE): <syncBaseDir>/<contentHash>
516 // 3. Slow path: scan fileIndex for any file:* entry with that hash
517 let diskPath = this._contentHashPaths.get(msg.contentHash)
518 if (diskPath && !fileExists(diskPath)) diskPath = null
519
520 if (!diskPath) {
521 // Content-addressed spool fallback. The canonical bytes are stored at
522 // <syncBaseDir>/<contentHash> (pear-git-v3 spools an upload to spool/<oid>
523 // where oid IS the content hash — router.js). The in-memory map is empty
524 // after a daemon restart and FileIndex persists no disk path, so resolve
525 // the hash straight from the content-addressed spool — a pure function of
526 // the hash + dir, like the tree-store's blobs/<hash>. This is what lets a
527 // forge keep serving its large files (hyper-tps LFS) across restarts.
528 // (NOT gip bundles: those are stored by git oid, not by contentHash — they
529 // need autobase-driven rehydration, tracked separately.)
530 for (const baseDir of this._syncBaseDirs) {
531 const candidate = path.join(baseDir, msg.contentHash)
532 if (fileExists(candidate)) {
533 diskPath = candidate
534 this._contentHashPaths.set(msg.contentHash, candidate) // warm the map
535 break
536 }
537 }
538 }
539
540 if (!diskPath) {
541 // Scan fileIndex for a file whose contentHash matches.
542 const fileIndex = this._syncEngine && this._syncEngine._fileIndex
543 if (fileIndex && typeof fileIndex.listFiles === 'function') {
544 try {
545 const files = await fileIndex.listFiles()
546 for (const f of files) {
547 if (f.contentHash === msg.contentHash) {
548 diskPath = this._filePaths.get(f.path)
549 if ((!diskPath || !fileExists(diskPath)) && f.path) diskPath = f.path
550 break
551 }
552 }
553 } catch {
554 // fall through
555 }
556 }
557 }
558
559 if (!diskPath) {
560 // Don't have it. Silently drop; requester will time out.
561 return
562 }
563
564 // Use a SYNTHETIC path keyed by contentHash so the requester can
565 // pre-register a destPath and the chunk flow routes correctly on
566 // both sides. Temporarily populate _filePaths so _onChunkNeed +
567 // chunkmap lookups resolve under this synthetic key.
568 const syntheticPath = 'content:' + msg.contentHash
569 this._filePaths.set(syntheticPath, diskPath)
570
571 const prepared = await this._transferManager.prepareFile(diskPath, syntheticPath)
572 if (!prepared) {
573 this._filePaths.delete(syntheticPath)
574 return
575 }
576 if (!await this._transferManager._fileIndex.hasChunkMap(syntheticPath)) {
577 await this._transferManager._fileIndex.putChunkMap(syntheticPath, prepared.chunks)
578 }
579 // [diag-2026-05-04]
580 if (prepared.chunks.length > 1000) {
581 // eslint-disable-next-line no-console
582 console.log('[v2-diag] chunkHashes(content) path=' + syntheticPath + ' chunks=' + prepared.chunks.length)
583 }
584 peer.msgs.chunkHashes.send({
585 path: syntheticPath,
586 tier: prepared.tier,
587 chunks: prepared.chunks
588 })
589 }
590
591 // ── Sync logic ──────────────────────────────────────────────
592
593 /**
594 * Offer all local files to a peer.
595 * Reads from the local feed and sends file-offers for each entry.
596 * The peer will request any files it doesn't have.
597 */
598 async _offerLocalFiles (peer) {
599 const feed = this._syncEngine.feed
600 if (feed.length === 0) {
601 console.log(' [proto] no local entries to offer')
602 return
603 }
604
605 console.log(' [proto] offering', feed.length, 'local entries')
606
607 for (let i = 0; i < feed.length; i++) {
608 try {
609 const entry = await feed.get(i)
610 if (!entry) continue
611
612 // Register the file path mapping so _onFileRequest can find it
613 if (entry.op === 0) {
614 const diskPath = this._resolveFilePath(entry.path)
615 if (diskPath) this._filePaths.set(entry.path, diskPath)
616 }
617
618 peer.msgs.fileOffer.send({
619 path: entry.path,
620 contentHash: entry.hash || '0'.repeat(64),
621 size: entry.size || 0,
622 mtime: entry.mtime || 0,
623 op: entry.op
624 })
625 } catch {
626 // skip unreadable entries
627 }
628 }
629 }
630
631 /**
632 * Resolve an overlay path to a disk path.
633 */
634 _resolveFilePath (overlayPath) {
635 if (this._filePaths.has(overlayPath)) return this._filePaths.get(overlayPath)
636 for (const baseDir of this._syncBaseDirs) {
637 return path.join(baseDir, overlayPath)
638 }
639 return null
640 }
641}
642
643// True iff `p` is an existing regular file. Used to validate a resolved disk path
644// before serving (a cold-restart map entry or an overlay-path masquerading as a
645// disk path won't exist) and to probe the content-addressed spool.
646function fileExists (p) {
647 try { return fs.statSync(p).isFile() } catch { return false }
648}
1// SHIM -> lib/v3 engine. OverlayProtocol(v3) exposed under our OverlayProtocolV2 name.
2export { OverlayProtocol, OverlayProtocol as OverlayProtocolV2 } from './v3/protocol.js'
lib/sync-engine.js +2 -360
……@@ -1,360 +1,2 @@
1/**
2 * SyncEngine — Feed-based change tracking and conflict detection
3 *
4 * Each peer has a Hypercore feed logging file change operations.
5 * When peers connect, feeds replicate automatically (Hypercore handles this).
6 * The sync engine diffs feeds since the last-synced sequence number
7 * and detects conflicts using three-value comparison:
8 *
9 * my current hash vs their current hash vs last-synced hash (common ancestor)
10 *
11 * Conflict resolution is configurable per sync folder:
12 * prompt | newest-wins | keep-both | skip | mine-wins | theirs-wins
13 */
14
15import ReadyResource from 'ready-resource'
16
17// Operation types written to the change feed
18const OP_PUT = 0
19const OP_DEL = 1
20
21export const STRATEGIES = ['prompt', 'newest-wins', 'keep-both', 'skip', 'mine-wins', 'theirs-wins']
22
23export class SyncEngine extends ReadyResource {
24 constructor (fileIndex, corestore, opts = {}) {
25 super()
26 this._fileIndex = fileIndex
27 this._corestore = corestore
28 this._feed = null
29 this._peerFeeds = new Map() // peerKey hex → Hypercore
30 this._conflicts = [] // unresolved conflicts
31 this._opts = opts
32 }
33
34 async _open () {
35 // Local change feed — append-only log of file operations
36 this._feed = this._corestore.get({ name: 'sync-feed', valueEncoding: 'json' })
37 await this._feed.ready()
38 }
39
40 async _close () {
41 // Peer feeds are managed by corestore, no explicit close needed
42 }
43
44 /**
45 * The local change feed (Hypercore)
46 */
47 get feed () { return this._feed }
48
49 /**
50 * The local feed's public key as hex
51 */
52 get feedKey () { return this._feed.key.toString('hex') }
53
54 /**
55 * Current unresolved conflicts
56 */
57 get conflicts () { return this._conflicts }
58
59 // ── Change logging ──────────────────────────────────────────
60
61 /**
62 * Log a file put/update operation
63 * @param {string} path - file path
64 * @param {{ contentHash: string, size: number, mtime: number }} meta
65 */
66 async logPut (path, meta) {
67 await this._feed.append({
68 op: OP_PUT,
69 path,
70 hash: meta.contentHash,
71 size: meta.size,
72 mtime: meta.mtime,
73 timestamp: Date.now()
74 })
75 }
76
77 /**
78 * Log a file deletion
79 * @param {string} path - file path
80 */
81 async logDel (path) {
82 await this._feed.append({
83 op: OP_DEL,
84 path,
85 hash: null,
86 size: 0,
87 mtime: 0,
88 timestamp: Date.now()
89 })
90 }
91
92 // ── Peer feeds ──────────────────────────────────────────────
93
94 /**
95 * Add a remote peer's feed for syncing
96 * @param {Buffer|string} key - peer's feed public key
97 * @returns {Hypercore} the peer feed
98 */
99 addPeerFeed (key) {
100 const keyHex = typeof key === 'string' ? key : key.toString('hex')
101 if (this._peerFeeds.has(keyHex)) return this._peerFeeds.get(keyHex)
102
103 const feed = this._corestore.get({ key: typeof key === 'string' ? Buffer.from(key, 'hex') : key, valueEncoding: 'json' })
104 this._peerFeeds.set(keyHex, feed)
105 return feed
106 }
107
108 /**
109 * Get a peer feed by key
110 * @param {string} keyHex
111 * @returns {Hypercore|null}
112 */
113 getPeerFeed (keyHex) {
114 return this._peerFeeds.get(keyHex) || null
115 }
116
117 // ── Feed diffing ────────────────────────────────────────────
118
119 /**
120 * Get new operations from a feed since a given sequence number
121 * @param {Hypercore} feed
122 * @param {number} sinceSeq - start reading from this seq (exclusive)
123 * @returns {Array<{ seq: number, op: number, path: string, hash: string|null, size: number, mtime: number }>}
124 */
125 async getChanges (feed, sinceSeq) {
126 await feed.ready()
127 await feed.update()
128
129 const changes = []
130 const start = sinceSeq + 1
131 const end = feed.length
132
133 for (let i = start; i < end; i++) {
134 const entry = await feed.get(i)
135 changes.push({ seq: i, ...entry })
136 }
137
138 return changes
139 }
140
141 /**
142 * Build a map of latest state per path from a list of changes.
143 * Later entries override earlier ones for the same path.
144 * @param {Array} changes
145 * @returns {Map<string, { seq, op, hash, size, mtime }>}
146 */
147 buildChangeMap (changes) {
148 const map = new Map()
149 for (const c of changes) {
150 map.set(c.path, { seq: c.seq, op: c.op, hash: c.hash, size: c.size, mtime: c.mtime })
151 }
152 return map
153 }
154
155 // ── Sync diff ───────────────────────────────────────────────
156
157 /**
158 * Compute the sync diff between local and a remote peer.
159 * Returns actions to take: pull, push, conflict, or skip.
160 *
161 * @param {string} peerKey - hex key of the remote peer's feed
162 * @returns {Array<{ path, action, local?, remote?, ancestor? }>}
163 * action: 'pull' | 'push' | 'conflict' | 'skip' | 'delete-local' | 'delete-remote'
164 */
165 async diff (peerKey) {
166 const peerFeed = this._peerFeeds.get(peerKey)
167 if (!peerFeed) throw new Error('Unknown peer: ' + peerKey)
168
169 await peerFeed.ready()
170 await peerFeed.update()
171
172 // Get last-synced state for all files with this peer
173 const syncStates = await this._fileIndex.listSyncStates(peerKey)
174 const syncMap = new Map()
175 for (const s of syncStates) {
176 syncMap.set(s.path, s)
177 }
178
179 // Get my changes since last sync
180 const myLastSeq = this._getMaxSyncedSeq(syncStates, 'local')
181 const myChanges = await this.getChanges(this._feed, -1) // read all, filter by syncMap
182 const myMap = this.buildChangeMap(myChanges)
183
184 // Get their changes since last sync
185 const theirChanges = await this.getChanges(peerFeed, -1) // read all
186 const theirMap = this.buildChangeMap(theirChanges)
187
188 // Collect all paths mentioned by either side
189 const allPaths = new Set([...myMap.keys(), ...theirMap.keys()])
190
191 const actions = []
192
193 for (const path of allPaths) {
194 const mine = myMap.get(path)
195 const theirs = theirMap.get(path)
196 const ancestor = syncMap.get(path)
197
198 const myHash = mine ? mine.hash : null
199 const theirHash = theirs ? theirs.hash : null
200 const ancestorHash = ancestor ? ancestor.lastHash : null
201
202 // Both have it and hashes match — in sync
203 if (myHash && theirHash && myHash === theirHash) {
204 actions.push({ path, action: 'skip', local: mine, remote: theirs })
205 continue
206 }
207
208 // Only they have it (or they changed, I haven't)
209 if (theirHash && !mine) {
210 actions.push({ path, action: 'pull', remote: theirs })
211 continue
212 }
213
214 // Only I have it (or I changed, they haven't)
215 if (myHash && !theirs) {
216 actions.push({ path, action: 'push', local: mine })
217 continue
218 }
219
220 // Both null — both deleted
221 if (!myHash && !theirHash) {
222 actions.push({ path, action: 'skip' })
223 continue
224 }
225
226 // One deleted, one has content
227 if (mine && mine.op === OP_DEL && theirHash) {
228 // I deleted, they still have it
229 if (ancestorHash && ancestorHash === theirHash) {
230 // They haven't changed since ancestor — propagate my delete
231 actions.push({ path, action: 'delete-remote', local: mine, remote: theirs })
232 } else {
233 // They changed it after ancestor — conflict
234 actions.push({ path, action: 'conflict', local: mine, remote: theirs, ancestor })
235 }
236 continue
237 }
238
239 if (theirs && theirs.op === OP_DEL && myHash) {
240 // They deleted, I still have it
241 if (ancestorHash && ancestorHash === myHash) {
242 // I haven't changed since ancestor — accept their delete
243 actions.push({ path, action: 'delete-local', local: mine, remote: theirs })
244 } else {
245 // I changed it after ancestor — conflict
246 actions.push({ path, action: 'conflict', local: mine, remote: theirs, ancestor })
247 }
248 continue
249 }
250
251 // Both have different content — use ancestor to determine conflict
252 if (myHash && theirHash && myHash !== theirHash) {
253 if (ancestorHash === myHash) {
254 // Only they changed — pull
255 actions.push({ path, action: 'pull', local: mine, remote: theirs })
256 } else if (ancestorHash === theirHash) {
257 // Only I changed — push
258 actions.push({ path, action: 'push', local: mine, remote: theirs })
259 } else {
260 // Both changed — conflict
261 actions.push({ path, action: 'conflict', local: mine, remote: theirs, ancestor })
262 }
263 continue
264 }
265
266 // Fallback — shouldn't reach here
267 actions.push({ path, action: 'skip', local: mine, remote: theirs })
268 }
269
270 return actions
271 }
272
273 // ── Conflict resolution ─────────────────────────────────────
274
275 /**
276 * Resolve a conflict using a strategy
277 * @param {{ path, local, remote, ancestor }} conflict
278 * @param {string} strategy - one of STRATEGIES
279 * @returns {{ action: 'pull'|'push'|'keep-both'|'skip', winner?: 'local'|'remote' }}
280 */
281 resolveConflict (conflict, strategy) {
282 switch (strategy) {
283 case 'mine-wins':
284 return { action: 'push', winner: 'local' }
285
286 case 'theirs-wins':
287 return { action: 'pull', winner: 'remote' }
288
289 case 'newest-wins': {
290 const mySeq = conflict.local ? conflict.local.seq : -1
291 const theirSeq = conflict.remote ? conflict.remote.seq : -1
292 if (mySeq >= theirSeq) {
293 return { action: 'push', winner: 'local' }
294 }
295 return { action: 'pull', winner: 'remote' }
296 }
297
298 case 'keep-both':
299 return { action: 'keep-both' }
300
301 case 'skip':
302 return { action: 'skip' }
303
304 case 'prompt':
305 default:
306 // Add to conflict queue for user resolution
307 this._conflicts.push(conflict)
308 return { action: 'skip' }
309 }
310 }
311
312 /**
313 * Resolve a queued conflict by user choice
314 * @param {string} path
315 * @param {string} choice - 'overwrite' | 'keep-both' | 'skip'
316 * @returns {{ action: string } | null}
317 */
318 resolveQueued (path, choice) {
319 const idx = this._conflicts.findIndex(c => c.path === path)
320 if (idx === -1) return null
321
322 const conflict = this._conflicts.splice(idx, 1)[0]
323
324 switch (choice) {
325 case 'overwrite':
326 return { action: 'pull', winner: 'remote', conflict }
327 case 'keep-mine':
328 return { action: 'push', winner: 'local', conflict }
329 case 'keep-both':
330 return { action: 'keep-both', conflict }
331 case 'skip':
332 default:
333 return { action: 'skip', conflict }
334 }
335 }
336
337 /**
338 * Update sync state after a successful sync operation
339 * @param {string} peerKey
340 * @param {string} path
341 * @param {string} hash - the hash both sides now agree on
342 * @param {number} seq - the sequence number of the operation
343 */
344 async markSynced (peerKey, path, hash, seq) {
345 await this._fileIndex.putSyncState(peerKey, path, {
346 lastSeq: seq,
347 lastHash: hash
348 })
349 }
350
351 // ── Internal ────────────────────────────────────────────────
352
353 _getMaxSyncedSeq (syncStates, _side) {
354 let max = -1
355 for (const s of syncStates) {
356 if (s.lastSeq > max) max = s.lastSeq
357 }
358 return max
359 }
360}
1// SHIM: v0.3.0 engine lives in lib/v3/ (Ok mirror). Re-export under our name/subpath.
2export * from './v3/sync-engine.js'
lib/transfer.js +2 -344
……@@ -1,344 +1,2 @@
1/**
2 * TransferManager — On-demand chunking and streaming file transfer
3 *
4 * Sender side:
5 * 1. Read file from disk
6 * 2. Chunk on demand (adaptive FastCDC)
7 * 3. Send chunk hashes → peer says which it needs → stream those chunks
8 * 4. Release chunks from memory after send
9 *
10 * Receiver side:
11 * 1. Receive chunk hashes, respond with which are missing
12 * 2. Receive chunks, write to .file.overlay-partial
13 * 3. Verify each chunk hash
14 * 4. Atomic rename on completion
15 *
16 * No blob storage. Chunks exist in memory during transfer only.
17 */
18
19import fs from '#fs'
20import path from '#path'
21import { chunk as chunkBuffer, chunkStream, hashChunk, createStreamingHasher, selectTier } from './chunker.js'
22
23const PARTIAL_SUFFIX = '.overlay-partial'
24
25export class TransferManager {
26 constructor (fileIndex, opts = {}) {
27 this._fileIndex = fileIndex
28 this._maxConcurrency = opts.maxConcurrency || 4
29 this._active = new Map() // path → transfer state
30 }
31
32 // ── Sender side ─────────────────────────────────────────────
33
34 /**
35 * Prepare a file for sending — read from disk, chunk, return hashes.
36 * Chunk map is persisted for large files (>1MB).
37 *
38 * @param {string} filePath - absolute path on disk
39 * @param {string} overlayPath - path in the overlay namespace
40 * @returns {{ tier: number, chunks: Array<{ hash, offset, length }>, size: number } | null}
41 */
42 async prepareFile (filePath, overlayPath) {
43 // Check file exists and get stats
44 let stat
45 try {
46 stat = fs.statSync(filePath)
47 } catch {
48 return null
49 }
50
51 if (!stat.isFile()) return null
52
53 const mtimeBefore = stat.mtimeMs
54
55 // Check for existing chunk map (skip re-chunking if valid)
56 const existing = await this._fileIndex.getChunkMap(overlayPath)
57 const fileMeta = await this._fileIndex.getFile(overlayPath)
58
59 if (existing && fileMeta && fileMeta.mtime === mtimeBefore) {
60 // Chunk map still valid — file hasn't changed
61 return {
62 tier: selectTier(stat.size),
63 chunks: existing,
64 size: stat.size
65 }
66 }
67
68 // Stream the file through FastCDC + a parallel full-file hasher.
69 // Memory footprint ≈ 2 × tier.maxSize (a few MB for tier 1, capped
70 // at 8 MB for tier 3) regardless of file size — so 2 GB+ files
71 // chunk fine on small-RAM hosts. Cut points are content-stable
72 // across `chunk()` and `chunkStream()` so peers using either
73 // produce matching hashes.
74 const tier = selectTier(stat.size)
75 const reader = fs.createReadStream(filePath)
76 // size is known from stat — pass it so the streaming digest
77 // matches the one-shot `hashChunk(buffer)` digest peers expect.
78 const fileHasher = createStreamingHasher({ size: stat.size })
79 const chunks = []
80
81 try {
82 for await (const c of chunkStream(reader, { tier })) {
83 // Update the full-file hash with this chunk's bytes BEFORE we
84 // drop the reference. The chunkStream copies bytes off the
85 // reader's pool, so once we hash + push metadata, GC can free
86 // the chunk's buffer on the next loop iteration.
87 fileHasher.update(c.data)
88 chunks.push({ hash: c.hash, offset: c.offset, length: c.length })
89 }
90 } finally {
91 try { reader.destroy() } catch {}
92 }
93
94 // mtime guard — abort if file changed during the streaming read.
95 const statAfter = fs.statSync(filePath)
96 if (statAfter.mtimeMs !== mtimeBefore) {
97 return null // file changed during read, caller should re-queue
98 }
99
100 // Persist chunk map for large files
101 if (stat.size >= 1048576) {
102 await this._fileIndex.putChunkMap(overlayPath, chunks)
103 }
104
105 // Update file index
106 const contentHash = fileHasher.digest()
107 await this._fileIndex.putFile(overlayPath, {
108 contentHash,
109 size: stat.size,
110 mtime: mtimeBefore
111 })
112
113 return { tier, chunks, size: stat.size }
114 }
115
116 /**
117 * Read a specific chunk from disk by its offset and length.
118 * Returns raw bytes — caller sends them to the peer.
119 *
120 * @param {string} filePath - absolute path on disk
121 * @param {number} offset - byte offset
122 * @param {number} length - bytes to read
123 * @returns {Buffer | null}
124 */
125 readChunk (filePath, offset, length) {
126 try {
127 const fd = fs.openSync(filePath, 'r')
128 const buf = Buffer.alloc(length)
129 const bytesRead = fs.readSync(fd, buf, 0, length, offset)
130 fs.closeSync(fd)
131 if (bytesRead !== length) return null
132 return buf
133 } catch {
134 return null
135 }
136 }
137
138 /**
139 * Determine which chunks a peer needs (chunks they don't already have).
140 *
141 * @param {Array<{ hash, offset, length }>} offered - chunks the sender has
142 * @param {Set<string>} peerHas - set of chunk hashes the peer already has
143 * @returns {number[]} indices of chunks the peer needs
144 */
145 computeNeeded (offered, peerHas) {
146 const needed = []
147 for (let i = 0; i < offered.length; i++) {
148 if (!peerHas.has(offered[i].hash)) {
149 needed.push(i)
150 }
151 }
152 return needed
153 }
154
155 // ── Receiver side ───────────────────────────────────────────
156
157 /**
158 * Start receiving a file. Creates a partial file for writing.
159 *
160 * @param {string} targetPath - absolute path where the final file will live
161 * @param {{ size: number, chunks: Array<{ hash, offset, length }> }} meta
162 * @returns {{ partialPath: string, received: Set<number>, total: number }}
163 */
164 startReceive (targetPath, meta) {
165 const dir = path.dirname(targetPath)
166 const base = path.basename(targetPath)
167 const partialPath = path.join(dir, '.' + base + PARTIAL_SUFFIX)
168
169 // Ensure directory exists
170 fs.mkdirSync(dir, { recursive: true })
171
172 // Pre-allocate the partial file
173 const fd = fs.openSync(partialPath, 'w')
174 fs.ftruncateSync(fd, meta.size)
175 fs.closeSync(fd)
176
177 const state = {
178 partialPath,
179 targetPath,
180 chunks: meta.chunks,
181 received: new Set(),
182 total: meta.chunks.length,
183 size: meta.size
184 }
185
186 this._active.set(targetPath, state)
187 return state
188 }
189
190 /**
191 * Write a received chunk to the partial file. Verifies the hash.
192 *
193 * @param {string} targetPath - the final file path (used as key)
194 * @param {number} index - chunk index
195 * @param {Buffer} data - raw chunk bytes
196 * @returns {{ ok: boolean, error?: string }}
197 */
198 writeChunk (targetPath, index, data) {
199 const state = this._active.get(targetPath)
200 if (!state) return { ok: false, error: 'no active transfer' }
201 if (index < 0 || index >= state.chunks.length) return { ok: false, error: 'index out of range' }
202
203 const expected = state.chunks[index]
204
205 // Verify hash
206 const actual = hashChunk(data)
207 if (actual !== expected.hash) {
208 return { ok: false, error: `hash mismatch at index ${index}: expected ${expected.hash.slice(0, 16)}... got ${actual.slice(0, 16)}...` }
209 }
210
211 // Verify length
212 if (data.length !== expected.length) {
213 return { ok: false, error: `length mismatch at index ${index}: expected ${expected.length} got ${data.length}` }
214 }
215
216 // Write to partial file at the correct offset
217 const fd = fs.openSync(state.partialPath, 'r+')
218 fs.writeSync(fd, data, 0, data.length, expected.offset)
219 fs.closeSync(fd)
220
221 state.received.add(index)
222
223 return { ok: true }
224 }
225
226 /**
227 * Check if a transfer is complete (all chunks received).
228 *
229 * @param {string} targetPath
230 * @returns {boolean}
231 */
232 isComplete (targetPath) {
233 const state = this._active.get(targetPath)
234 if (!state) return false
235 return state.received.size === state.total
236 }
237
238 /**
239 * Finalize a completed transfer — atomic rename from partial to target.
240 *
241 * @param {string} targetPath
242 * @returns {{ ok: boolean, error?: string }}
243 */
244 finalize (targetPath) {
245 const state = this._active.get(targetPath)
246 if (!state) return { ok: false, error: 'no active transfer' }
247 if (state.received.size !== state.total) {
248 return { ok: false, error: `incomplete: ${state.received.size}/${state.total} chunks` }
249 }
250
251 try {
252 fs.renameSync(state.partialPath, state.targetPath)
253 } catch (err) {
254 return { ok: false, error: 'rename failed: ' + err.message }
255 }
256
257 this._active.delete(targetPath)
258 return { ok: true }
259 }
260
261 /**
262 * Cancel an active transfer and clean up the partial file.
263 *
264 * @param {string} targetPath
265 */
266 cancel (targetPath) {
267 const state = this._active.get(targetPath)
268 if (!state) return
269
270 try {
271 fs.unlinkSync(state.partialPath)
272 } catch {
273 // partial file may not exist
274 }
275
276 this._active.delete(targetPath)
277 }
278
279 /**
280 * Get transfer progress for an active transfer.
281 *
282 * @param {string} targetPath
283 * @returns {{ received: number, total: number, percentage: number } | null}
284 */
285 getProgress (targetPath) {
286 const state = this._active.get(targetPath)
287 if (!state) return null
288
289 return {
290 received: state.received.size,
291 total: state.total,
292 percentage: state.total > 0 ? Math.round(state.received.size / state.total * 100) : 0
293 }
294 }
295
296 /**
297 * List all active transfers.
298 *
299 * @returns {Array<{ path: string, received: number, total: number, percentage: number }>}
300 */
301 listActive () {
302 const result = []
303 for (const [targetPath, state] of this._active) {
304 result.push({
305 path: targetPath,
306 received: state.received.size,
307 total: state.total,
308 percentage: state.total > 0 ? Math.round(state.received.size / state.total * 100) : 0
309 })
310 }
311 return result
312 }
313
314 /**
315 * Clean up stale partial files in a directory (older than maxAge).
316 *
317 * @param {string} dir - directory to scan
318 * @param {number} [maxAge=86400000] - max age in ms (default 24h)
319 * @returns {string[]} paths of cleaned files
320 */
321 cleanPartials (dir, maxAge = 86400000) {
322 const cleaned = []
323 try {
324 const entries = fs.readdirSync(dir)
325 const now = Date.now()
326 for (const entry of entries) {
327 if (!entry.endsWith(PARTIAL_SUFFIX)) continue
328 const full = path.join(dir, entry)
329 try {
330 const stat = fs.statSync(full)
331 if (now - stat.mtimeMs > maxAge) {
332 fs.unlinkSync(full)
333 cleaned.push(full)
334 }
335 } catch {
336 // skip unreadable files
337 }
338 }
339 } catch {
340 // directory may not exist
341 }
342 return cleaned
343 }
344}
1// SHIM: v0.3.0 engine lives in lib/v3/ (Ok mirror). Re-export under our name/subpath.
2export * from './v3/transfer.js'
lib/v3/chunk-scheduler.js +340 -0
……@@ -0,0 +1,340 @@
1/**
2 * ChunkScheduler — multi-source (torrent-style) fetch of one content-addressed
3 * file across several peers, modelled on Hypercore's block replicator.
4 *
5 * Chunks are content-addressed, so any peer holding the file can serve any
6 * chunk. The scheduler:
7 * - requests the chunk list from each peer (identical, content-addressed),
8 * - tracks a global `inflight` map so each chunk is requested from exactly
9 * ONE peer at a time (no wasted bandwidth),
10 * - keeps each peer busy up to a per-peer inflight cap (parallel download),
11 * - reassigns a peer's inflight chunks to others on stall/disconnect,
12 * - hash-verifies every chunk (via TransferManager.writeChunk) and the whole
13 * file before finalizing.
14 *
15 * It drives the EXISTING v2 wire messages (chunk-need / chunk-data) — no new
16 * message type is needed for the full-copy-seeder case. The protocol delegates
17 * `_onChunkHashes` / `_onChunkData` to a scheduler ONLY when one is registered
18 * for the synthetic `content:<hash>` path, so the legacy single-peer flow
19 * is untouched.
20 */
21
22import { hashChunk } from './chunker.js'
23
24const DEFAULT_CAP = 8
25// Head-of-line fairness — cede the event loop every YIELD_EVERY accepted chunks
26// so a burst of buffered chunkData frames (protomux dispatches every frame from
27// one socket read synchronously) can't starve other listeners multiplexed on the
28// same connection (e.g. Hypercore replication). Kept local (not imported from
29// transfer.js) so this module stays Node-loadable for the unit suite.
30const yieldToLoop = () => new Promise((resolve) => setTimeout(resolve, 0))
31const YIELD_EVERY = 16
32// Idle (no-progress) timeout, reset on every accepted chunk and on the
33// initial chunk-list response. A steadily-progressing transfer never trips it —
34// only a genuine stall (no bytes for this long) fails. A fixed OVERALL timeout
35// here capped large transfers (e.g. a multi-GB file at any bandwidth always
36// exceeded 30s) and aborted them mid-stream.
37const DEFAULT_IDLE_TIMEOUT = 30000
38// Local write-error codes that may recover on retry (vs ENOSPC/EACCES/…
39// which are fatal): a transient one keeps the chunk retryable instead of failing
40// the whole multi-source fetch.
41const TRANSIENT_WRITE_CODES = new Set(['EBUSY', 'EAGAIN', 'EINTR', 'EMFILE', 'ENFILE'])
42// Min interval between have-progress reports to holders (their sender-side bar).
43// Throttles the transferProgress frames so a fast transfer can't spam holders.
44const DEFAULT_REPORT_INTERVAL = 1000
45
46export class ChunkScheduler {
47 /**
48 * @param {object} opts
49 * opts.path synthetic path key ('content:<hash>')
50 * opts.destPath where the assembled file is written
51 * opts.transfer TransferManager
52 * opts.sendNeed (peer, indices[]) => void — sends a chunk-need message
53 * opts.cap per-peer inflight cap (default 8)
54 * opts.timeout idle (no-progress) timeout ms (default 30000)
55 * opts.onProgress optional (receivedBytes, totalBytes) => void on each accepted chunk
56 */
57 constructor (opts) {
58 this.path = opts.path
59 this.destPath = opts.destPath
60 this._transfer = opts.transfer
61 this._sendNeed = opts.sendNeed
62 this._cap = opts.cap || DEFAULT_CAP
63 this._onProgress = opts.onProgress || null
64 this._onVerify = opts.onVerify || null
65 this._onBaseline = opts.onBaseline || null
66 this._reportInterval = opts.reportInterval ?? DEFAULT_REPORT_INTERVAL
67 this._lastReportAt = 0
68 // cumulative byte accounting so onProgress can drive a
69 // remaining-bytes/ETA bar, not just a chunk count.
70 this._receivedBytes = 0
71 this._totalBytes = 0
72 this._sinceYield = 0 // event-loop yield counter (see YIELD_EVERY)
73
74 this._peers = new Set()
75 this._started = false
76 this._settingUp = false
77 this._finalizing = false
78 this._done = false
79 this._chunks = null // [{ hash, offset, length }]
80 this._needed = new Set() // indices not yet written
81 this._inflight = new Map() // index → peer (one peer per chunk)
82 this._peerInflight = new Map() // peer → count
83 this._received = new Map() // peer → chunks accepted (multi-source proof)
84
85 this._resolve = null
86 this._reject = null
87 this._onEnd = opts.onEnd || null
88 this._contentHash = opts.contentHash || null // enables incremental whole-file verify
89 this._startedAt = Date.now()
90 // idle timeout — re-armed on each progress signal (see _armIdleTimer).
91 this._idleTimeout = opts.timeout || DEFAULT_IDLE_TIMEOUT
92 this._timer = null
93 this._armIdleTimer()
94 }
95
96 promise () {
97 return new Promise((resolve, reject) => { this._resolve = resolve; this._reject = reject })
98 }
99
100 // (Re)arm the no-progress watchdog. Called at construction and on every
101 // forward-progress signal so only a genuine stall — no accepted bytes for the
102 // idle window — aborts the fetch, regardless of total file size.
103 _armIdleTimer () {
104 if (this._done) return
105 clearTimeout(this._timer)
106 this._timer = setTimeout(
107 () => this._fail(new Error('multi-source fetch stalled (no progress for ' + this._idleTimeout + 'ms)')),
108 this._idleTimeout,
109 )
110 }
111
112 _reportEnd (ok, reason) {
113 if (!this._onEnd) return
114 try {
115 this._onEnd({
116 ok,
117 reason,
118 receivedBytes: this._receivedBytes,
119 totalBytes: this._totalBytes,
120 chunksRemaining: this._needed.size,
121 totalChunks: this._chunks ? this._chunks.length : 0,
122 peers: this._peers.size,
123 elapsedMs: Date.now() - this._startedAt,
124 })
125 } catch {}
126 }
127
128 _fail (err) {
129 if (this._done) return
130 this._done = true
131 clearTimeout(this._timer)
132 this._reportEnd(false, err.message)
133 // Release the receiver state at the failure boundary: journal the hash
134 // frontier and close the fd + chunk stash so a never-retried failure parks
135 // nothing (the partial stays on disk for a later resume). Best-effort — the
136 // external-stop path (cancel) does its own pause/cancel before rejecting.
137 if (this._transfer && typeof this._transfer.pause === 'function') {
138 try { Promise.resolve(this._transfer.pause(this.destPath)).catch(() => {}) } catch {}
139 }
140 if (this._reject) this._reject(err)
141 }
142
143 _finish () {
144 if (this._done) return
145 this._done = true
146 clearTimeout(this._timer)
147 this._reportEnd(true, 'complete')
148 if (this._resolve) this._resolve({ chunksPerPeer: this._received, peerCount: this._received.size, size: this._totalBytes })
149 }
150
151 // External stop (pause or discard). The partial is kept or unlinked by
152 // the caller (OverlayProtocol.cancelContent via TransferManager.pause/cancel)
153 // BEFORE this; here we only stop the loop and reject the fetch with ECANCELLED.
154 cancel () {
155 if (this._done) return
156 this._done = true
157 clearTimeout(this._timer)
158 this._reportEnd(false, 'cancelled')
159 const err = new Error('fetch cancelled')
160 err.code = 'ECANCELLED'
161 if (this._reject) this._reject(err)
162 }
163
164 get done () { return this._done }
165
166 // A chunkHashes page arrived while a large map is still
167 // streaming across multiple frames. Receiving it is forward progress, so
168 // re-arm the no-progress watchdog (the full list dispatches via onChunkHashes
169 // only once the final page lands).
170 notePageProgress () { if (!this._done && !this._settingUp) this._armIdleTimer() }
171
172 /** A peer responded with the chunk list. First response starts the transfer. */
173 async onChunkHashes (peer, chunks) {
174 if (this._done) return
175 this._peers.add(peer)
176 if (!this._peerInflight.has(peer)) this._peerInflight.set(peer, 0)
177
178 if (!this._started) {
179 this._started = true
180 // The async startReceive (journal-less resume re-verify) can run for a while
181 // and accepts no chunks; suppress the no-progress watchdog (including later
182 // peers' re-arms) until setup completes, or a concurrent seeder's chunk list
183 // would re-arm it and self-trip a healthy resume.
184 this._settingUp = true
185 clearTimeout(this._timer)
186 // Derive offsets from the ordered chunk lengths.
187 let offset = 0
188 this._chunks = chunks.map((c) => {
189 const entry = { hash: c.hash, offset, length: c.length }
190 offset += c.length
191 return entry
192 })
193 this._totalBytes = offset // file size = sum of chunk lengths
194 let state
195 try {
196 state = await this._transfer.startReceive(
197 this.destPath,
198 { size: offset, chunks: this._chunks, contentHash: this._contentHash },
199 { isCancelled: () => this._done, onVerifyProgress: this._onVerify }
200 )
201 } catch (err) {
202 this._settingUp = false
203 if (this._done) return
204 return this._fail(err)
205 }
206 this._settingUp = false
207 if (this._done) return
208 // resume: only fetch chunks the partial doesn't already hold, and
209 // seed the byte counter so progress/ETA continue from the resumed offset.
210 let have = 0
211 for (let i = 0; i < this._chunks.length; i++) {
212 if (state.received.has(i)) have += this._chunks[i].length
213 else this._needed.add(i)
214 }
215 this._receivedBytes = have
216 if (have > 0) {
217 // report our resume have-bytes so holders' sender-side bars mirror our
218 // true progress, not just the bytes they re-serve. Re-sent periodically as more
219 // chunks land (onChunkData) so late-joining and multi-source holders stay in sync.
220 this._reportHave(Date.now())
221 if (this._onProgress) { try { this._onProgress(this._receivedBytes, this._totalBytes) } catch {} }
222 }
223 this._armIdleTimer()
224 if (this._needed.size === 0) return this._finalize()
225 return this._assign()
226 }
227 if (!this._settingUp) this._armIdleTimer() // a later peer's list is forward progress
228 this._assign()
229 }
230
231 /** A chunk arrived from a peer. Verify + write, then schedule more. */
232 async onChunkData (peer, index, data) {
233 if (this._done) return
234 const res = this._transfer.writeChunk(this.destPath, index, data)
235 // Free the inflight slot regardless — a bad chunk should be re-fetched.
236 if (this._inflight.get(index) === peer) {
237 this._inflight.delete(index)
238 this._peerInflight.set(peer, Math.max(0, (this._peerInflight.get(peer) || 1) - 1))
239 }
240 if (!res.ok) {
241 // A coded failure is a local fs error. A TRANSIENT code (device busy,
242 // fd pressure, interrupted syscall) can succeed on retry — leave the chunk in
243 // `needed` and reassign. Any other coded error (ENOSPC / EACCES / ENOENT / …) is
244 // fatal: fail the whole fetch carrying the code. No code = a hash/length
245 // mismatch — also retried elsewhere.
246 if (res.code && !TRANSIENT_WRITE_CODES.has(res.code)) {
247 const err = new Error('write failed: ' + res.error)
248 err.code = res.code
249 return this._fail(err)
250 }
251 this._assign()
252 return
253 }
254 this._needed.delete(index)
255 this._received.set(peer, (this._received.get(peer) || 0) + 1)
256 this._receivedBytes += (data ? data.length : 0)
257 this._armIdleTimer() // accepted bytes — reset the no-progress watchdog
258 if (this._onProgress) { try { this._onProgress(this._receivedBytes, this._totalBytes) } catch {} }
259 this._maybeReportHave()
260 if (this._needed.size === 0 && this._inflight.size === 0) return this._finalize()
261 // Periodic event-loop yield (see YIELD_EVERY)
262 if (++this._sinceYield >= YIELD_EVERY) {
263 this._sinceYield = 0
264 await yieldToLoop()
265 if (this._done) return
266 }
267 this._assign()
268 }
269
270 // Report our cumulative have-bytes to holders (their sender-side bar). Fired
271 // once at resume, then throttled as chunks land, and once at finalize so the bar settles
272 // at 100% across every holder regardless of who served what.
273 _reportHave (now) {
274 if (!this._onBaseline || this._receivedBytes <= 0) return
275 this._lastReportAt = now
276 try { this._onBaseline(this._receivedBytes) } catch {}
277 }
278
279 _maybeReportHave () {
280 if (!this._onBaseline) return
281 const now = Date.now()
282 if (now - this._lastReportAt >= this._reportInterval) this._reportHave(now)
283 }
284
285 /** A peer went away — return its inflight chunks to the pool. */
286 removePeer (peer) {
287 if (!this._peers.has(peer)) return
288 this._peers.delete(peer)
289 this._peerInflight.delete(peer)
290 for (const [index, p] of this._inflight) {
291 if (p === peer) this._inflight.delete(index) // back to needed (still in _needed)
292 }
293 if (this._done) return
294 if (this._peers.size === 0 && this._needed.size > 0) {
295 return this._fail(new Error('all peers gone with ' + this._needed.size + ' chunk(s) outstanding'))
296 }
297 this._assign()
298 }
299
300 async _finalize () {
301 if (this._finalizing || this._done) return
302 this._finalizing = true
303 this._reportHave(Date.now()) // final have=total so holders' bars settle at 100%
304 clearTimeout(this._timer) // the digest drain can outlast the idle window
305 let fin
306 try { fin = await this._transfer.finalize(this.destPath) }
307 catch (err) { return this._fail(err) }
308 if (!fin.ok) {
309 const err = new Error('finalize failed: ' + fin.error)
310 if (fin.code) err.code = fin.code // preserve EHASHMISMATCH for the caller
311 return this._fail(err)
312 }
313 this._finish()
314 }
315
316 /** Assign needed-but-not-inflight chunks to peers with spare capacity. */
317 _assign () {
318 if (this._done || !this._chunks) return
319 // Per peer, batch the indices we hand it this round.
320 const batches = new Map()
321 for (const peer of this._peers) {
322 let slots = this._cap - (this._peerInflight.get(peer) || 0)
323 if (slots <= 0) continue
324 for (const index of this._needed) {
325 if (slots <= 0) break
326 if (this._inflight.has(index)) continue
327 this._inflight.set(index, peer)
328 this._peerInflight.set(peer, (this._peerInflight.get(peer) || 0) + 1)
329 if (!batches.has(peer)) batches.set(peer, [])
330 batches.get(peer).push(index)
331 slots--
332 }
333 }
334 for (const [peer, indices] of batches) {
335 if (indices.length) this._sendNeed(peer, indices)
336 }
337 }
338}
339
340export { hashChunk }
lib/v3/chunker.js +333 -0
……@@ -0,0 +1,333 @@
1/**
2 * FastCDC Content-Defined Chunker
3 *
4 * Gear rolling hash with normalized chunking (Xia et al. 2016).
5 * Produces content-stable chunk boundaries — editing the middle of a file
6 * only changes ~2 chunks regardless of file size.
7 *
8 * v2: Adaptive chunk sizing — tier selected by file size.
9 * All peers MUST use the same tier for a given file to produce matching boundaries.
10 */
11
12import crypto from 'hypercore-crypto'
13
14// ── Tier 0 (default, < 1 MB) ─────────────────────────────────
15export const MIN_SIZE = 4096
16export const AVG_SIZE = 16384
17export const MAX_SIZE = 65536
18
19// ── Adaptive chunk size tiers ─────────────────────────────────
20export const TIERS = [
21 { minSize: 4096, avgSize: 16384, maxSize: 65536 }, // Tier 0: < 1 MB
22 { minSize: 16384, avgSize: 65536, maxSize: 262144 }, // Tier 1: 1 MB - 100 MB
23 { minSize: 65536, avgSize: 262144, maxSize: 1048576 }, // Tier 2: 100 MB - 10 GB
24 { minSize: 262144, avgSize: 1048576, maxSize: 4194304 } // Tier 3: > 10 GB
25]
26
27const TIER_THRESHOLDS = [
28 1048576, // 1 MB
29 104857600, // 100 MB
30 10737418240 // 10 GB
31]
32
33/**
34 * Select the appropriate chunk size tier for a file
35 * @param {number} fileSize - total file size in bytes
36 * @returns {number} tier index (0-3)
37 */
38export function selectTier (fileSize) {
39 for (let i = 0; i < TIER_THRESHOLDS.length; i++) {
40 if (fileSize < TIER_THRESHOLDS[i]) return i
41 }
42 return TIERS.length - 1
43}
44
45/**
46 * Get chunk size parameters for a tier
47 * @param {number} tier - tier index (0-3)
48 * @returns {{ minSize: number, avgSize: number, maxSize: number }}
49 */
50export function getTierParams (tier) {
51 return TIERS[tier] || TIERS[0]
52}
53
54// Gear hash table — 256 random uint32 values
55// MUST be identical across all implementations for chunk boundary compatibility
56const GEAR = new Uint32Array([
57 0x5c95c078, 0x22408989, 0x2d48a214, 0x12842087, 0x530f8afb, 0x474536b9, 0x2963b4f1, 0x44cb738b,
58 0x4ea7403d, 0x4d606b6e, 0x074ec5d3, 0x3af39d18, 0x726c7b74, 0x60b3f158, 0x56a76369, 0x3e8ab850,
59 0x0c215cf6, 0x42ad3c2f, 0x65b26fa7, 0x69a3e894, 0x2a60445d, 0x21b0f756, 0x5caca9af, 0x373b4cf0,
60 0x1a4a56e1, 0x5eaf1e7b, 0x7b5d5c8a, 0x48f5abf4, 0x02a42b68, 0x1f3f1a4e, 0x67e16b96, 0x404c7833,
61 0x6e4e3a3c, 0x4f769e04, 0x0bfef25f, 0x3f08d1ef, 0x3bee8eb5, 0x4302687c, 0x3e9e2964, 0x0fef6b59,
62 0x70b95d2c, 0x0cffb210, 0x2e6c3f2a, 0x6d8f0f67, 0x77a49ba2, 0x35fa6b91, 0x43d2d154, 0x5d4e6c25,
63 0x0e7e5283, 0x5fac3d19, 0x20e34b36, 0x46b6de87, 0x48eed852, 0x63aae4d9, 0x2c65c18b, 0x5e5bc6ad,
64 0x7802c749, 0x44db9a82, 0x50e76e40, 0x532dfa2b, 0x56a17be8, 0x0a81e2e7, 0x5cf2fccc, 0x3ab39d12,
65 0x5c0c0b78, 0x21e8af93, 0x3dca216f, 0x15b4b92e, 0x55445f9c, 0x44f24b2e, 0x2db33f38, 0x444f6ec8,
66 0x4eeaede8, 0x48b27a8d, 0x00b7621d, 0x3c916d41, 0x72f65778, 0x60b1a155, 0x5a946e70, 0x3ea5d857,
67 0x0aca0e86, 0x41b3b22e, 0x6572e2a6, 0x6aa8e89a, 0x2d496f61, 0x23b6f159, 0x5eb7e1a6, 0x312e4fea,
68 0x1baf6dc4, 0x5eb0f27e, 0x7a505c92, 0x49e8abf2, 0x03b12c6e, 0x1e4a7a4d, 0x64e76a98, 0x414c7c39,
69 0x6f4a3f38, 0x4c719e04, 0x0efef963, 0x3d08c1e9, 0x3ce88fba, 0x44026c79, 0x3e1e2469, 0x10ef6858,
70 0x73b55e21, 0x0dcfb614, 0x2b6d3927, 0x6c8e1066, 0x74a09bb5, 0x36fa7690, 0x44d2c158, 0x5e4f6b26,
71 0x0e7e5788, 0x5dad3e1c, 0x22e14a30, 0x47b6dd88, 0x4befd953, 0x64aae5d8, 0x2b65b28a, 0x5f5ac6aa,
72 0x7903c648, 0x43db1a89, 0x50e76f45, 0x542dfb28, 0x55a07cef, 0x0b80e3ea, 0x5ef2f9cf, 0x38b31e15,
73 0x5d0d0a7d, 0x22e9ae91, 0x3ecb216a, 0x16b5b82f, 0x54445e99, 0x45f34c2d, 0x2eb23e3b, 0x454e6dc9,
74 0x4febedef, 0x49b37a88, 0x01b66320, 0x3d926e44, 0x71f75779, 0x61b0a054, 0x5b947f71, 0x3fa4d754,
75 0x0bcb0f83, 0x42b4b32f, 0x6473e3a5, 0x6ba9e999, 0x2e497064, 0x24b7f05a, 0x5fb6e0a7, 0x322f5fe9,
76 0x1cae7ec3, 0x5db0e37f, 0x7b514d91, 0x4ae9abf3, 0x02b22d6b, 0x1f4b7b50, 0x63e66b97, 0x414d7d34,
77 0x6e4f3a3d, 0x4d779f05, 0x0afe0960, 0x3e09d0ee, 0x3de98ebb, 0x43036c7a, 0x3f1f2560, 0x11f06959,
78 0x72b65e22, 0x0dceb711, 0x2a6e3826, 0x6d8f1165, 0x75a19ab4, 0x37fb7793, 0x45d3c057, 0x5f4e6a27,
79 0x0f7f5684, 0x5eae3f1d, 0x23e24b31, 0x48b7de89, 0x4ceed854, 0x63abe5db, 0x2c64b38b, 0x5e5bc7ab,
80 0x7a04c54a, 0x44db1b88, 0x51e86e41, 0x552efb2a, 0x54a17de0, 0x0a81e3e6, 0x5df3f8ce, 0x39b41f14,
81 0x5e0e0b79, 0x23e8af92, 0x3fcc216e, 0x17b4b92d, 0x55456e9d, 0x46f44d2f, 0x2fb33e39, 0x464f6ec7,
82 0x4febe8e9, 0x49b27b8c, 0x02b7631e, 0x3e916e42, 0x73f75677, 0x62b2a256, 0x5c957e72, 0x3ea6d856,
83 0x0cca0f85, 0x43b4b231, 0x6574e2a7, 0x6ca9e89b, 0x2f496e62, 0x25b7f058, 0x5eb7e0a5, 0x332e4eeb,
84 0x1daf6ec5, 0x5eb0e27d, 0x7c515d93, 0x4be8abf1, 0x04b32d6a, 0x204b7a4e, 0x64e76b99, 0x424c7d3a,
85 0x704a3e3b, 0x4e789f06, 0x0bfefa62, 0x3f09d1e8, 0x3ee98fbd, 0x45036d7b, 0x401f2461, 0x12f0685a,
86 0x74b65f23, 0x0ecfb713, 0x2b6d3928, 0x6e901267, 0x76a29bb6, 0x38fc7892, 0x46d3c159, 0x604e6b28,
87 0x107e5785, 0x5fae3e1e, 0x24e24b32, 0x49b8df8a, 0x4defd955, 0x64abe6da, 0x2d65b48c, 0x605bc8ac,
88 0x7b05c64b, 0x45dc1c87, 0x52e96f46, 0x562ffc29, 0x55a27ee1, 0x0b82e4e5, 0x5ef3f7cd, 0x3ab42017
89])
90
91// Precomputed masks per tier (controls cut-point frequency)
92// Below average: larger mask → fewer cuts → chunks grow toward average
93// Above average: smaller mask → more cuts → chunks stay near average
94const TIER_MASKS = TIERS.map(t => ({
95 large: (1 << Math.ceil(Math.log2(t.avgSize))) - 1,
96 small: (1 << Math.floor(Math.log2(t.avgSize))) - 1
97}))
98
99/**
100 * Chunk a buffer using FastCDC with adaptive tier selection
101 *
102 * @param {Buffer} data - Data to chunk
103 * @param {{ tier?: number }} [opts] - Options. tier: force a specific tier (0-3). If omitted, auto-selects based on data.length.
104 * @returns {Array<{ hash: string, offset: number, length: number, data: Buffer }>}
105 */
106export function chunk (data, opts) {
107 if (data.length === 0) return []
108
109 const tier = (opts && typeof opts.tier === 'number') ? opts.tier : selectTier(data.length)
110 const params = TIERS[tier] || TIERS[0]
111 const masks = TIER_MASKS[tier] || TIER_MASKS[0]
112
113 const chunks = []
114 let offset = 0
115
116 while (offset < data.length) {
117 const remaining = data.length - offset
118
119 // If remaining data is smaller than minimum, it's the final chunk
120 if (remaining <= params.minSize) {
121 const slice = data.subarray(offset, offset + remaining)
122 chunks.push({
123 hash: hashChunk(slice),
124 offset,
125 length: remaining,
126 data: slice
127 })
128 break
129 }
130
131 const maxLen = Math.min(remaining, params.maxSize)
132 let hash = 0
133 let i = params.minSize
134
135 // Phase 1: below average — use large mask (fewer cuts)
136 const avgBound = Math.min(params.avgSize, maxLen)
137 while (i < avgBound) {
138 hash = ((hash << 1) + GEAR[data[offset + i]]) >>> 0
139 if ((hash & masks.large) === 0) break
140 i++
141 }
142
143 // Phase 2: above average — use small mask (more cuts)
144 if (i >= avgBound) {
145 while (i < maxLen) {
146 hash = ((hash << 1) + GEAR[data[offset + i]]) >>> 0
147 if ((hash & masks.small) === 0) break
148 i++
149 }
150 }
151
152 const slice = data.subarray(offset, offset + i)
153 chunks.push({
154 hash: hashChunk(slice),
155 offset,
156 length: i,
157 data: slice
158 })
159 offset += i
160 }
161
162 return chunks
163}
164
165/**
166 * Hash a chunk using blake2b-256
167 * @param {Buffer} data
168 * @returns {string} hex hash
169 */
170export function hashChunk (data) {
171 return crypto.data(data).toString('hex')
172}
173
174// ── Streaming primitives ──────────────────────────────────────
175//
176// `chunk(buffer)` requires the entire file in memory and returns chunks
177// whose `.data` is a `Buffer.subarray` of the input — both keep the
178// original buffer pinned until every yielded chunk is GC'd. For
179// multi-GB files (or limited-RAM runtimes) use the streaming versions
180// below: `chunkStream` consumes a Readable and yields chunks one at a
181// time, releasing bytes after each yield.
182
183import sodium from 'sodium-universal'
184import c from 'compact-encoding'
185
186const LEAF_TYPE = Buffer.from([0])
187
188/**
189 * Streaming blake2b-256 hasher matching `hashChunk(buffer)`'s digest
190 * format (hypercore-crypto's `crypto.data` shape:
191 * blake2b(LEAF_TYPE || uint64-LE(size) || data)
192 * ). Lets you hash multi-GB files without buffering them.
193 *
194 * const h = createStreamingHasher({ size }) // size known up front
195 * for await (const buf of stream) h.update(buf)
196 * const oid = h.digest() // == hashChunk(file)
197 *
198 * `size` is required for hash compatibility with `hashChunk` — the
199 * leaf-prefix construction includes the byte length, which is fixed at
200 * stream open. If you don't have the size, omit `size` and you'll get
201 * a plain blake2b digest of the bytes (does NOT match hashChunk).
202 */
203export function createStreamingHasher (opts = {}) {
204 const state = Buffer.alloc(sodium.crypto_generichash_STATEBYTES)
205 let bytes = 0
206 if (opts && opts.restore) {
207 // The saved state already includes the init + leaf/size prefix + every byte
208 // consumed so far, so do not re-init.
209 opts.restore.state.copy(state)
210 bytes = opts.restore.bytes
211 } else {
212 sodium.crypto_generichash_init(state, null, 32)
213 const knownSize = (opts && typeof opts.size === 'number') ? opts.size : null
214 if (knownSize !== null) {
215 sodium.crypto_generichash_update(state, LEAF_TYPE)
216 sodium.crypto_generichash_update(state, c.encode(c.uint64, knownSize))
217 }
218 }
219 return {
220 update (buf) {
221 sodium.crypto_generichash_update(state, buf)
222 bytes += buf.length
223 },
224 digest () {
225 const out = Buffer.alloc(32)
226 sodium.crypto_generichash_final(state, out)
227 return out.toString('hex')
228 },
229 snapshot () { return { state: Buffer.from(state), bytes } },
230 get bytes () { return bytes }
231 }
232}
233
234// Find a FastCDC cut point inside `data` starting at offset 0. Returns
235// the chunk length. Mirrors the inner search loop of `chunk()` but
236// takes pre-resolved params + masks so it works in a streaming loop.
237function findCutPoint (data, params, masks) {
238 const remaining = data.length
239 if (remaining <= params.minSize) return remaining
240
241 const maxLen = Math.min(remaining, params.maxSize)
242 let hash = 0
243 let i = params.minSize
244
245 const avgBound = Math.min(params.avgSize, maxLen)
246 while (i < avgBound) {
247 hash = ((hash << 1) + GEAR[data[i]]) >>> 0
248 if ((hash & masks.large) === 0) return i
249 i++
250 }
251 if (i >= avgBound) {
252 while (i < maxLen) {
253 hash = ((hash << 1) + GEAR[data[i]]) >>> 0
254 if ((hash & masks.small) === 0) return i
255 i++
256 }
257 }
258 return maxLen
259}
260
261// Opt-in memcpy accounting for perf validation. Zero cost when off (one
262// boolean check at the increment sites). Module-level singletons — enabled
263// only by tests + bench/prepare.js, never in library code; NOT
264// concurrency-safe, so do not enable it in production.
265let STATS_ON = false
266export const chunkStats = { concatBytes: 0, copyBytes: 0, blocks: 0, chunks: 0 }
267export function resetChunkStats () { chunkStats.concatBytes = 0; chunkStats.copyBytes = 0; chunkStats.blocks = 0; chunkStats.chunks = 0 }
268export function setChunkStats (on) { STATS_ON = !!on; if (on) resetChunkStats() }
269
270/**
271 * Stream-friendly FastCDC chunker. Consumes an AsyncIterable<Buffer>
272 * (e.g. fs.createReadStream) and yields chunks one at a time. Memory
273 * footprint ≈ 2 × params.maxSize regardless of stream length —
274 * chunks are emitted as bytes arrive, the pending buffer is sliced
275 * past each cut point.
276 *
277 * Each yielded chunk has the same shape as `chunk()` entries:
278 * { hash, offset, length, data }
279 * With the default `copy: true`, `data` is a fresh `Buffer.from(slice)`
280 * detached from the input stream's pool, so callers can store/forward it.
281 * With `copy: false`, `data` is a view backed by the reader's memory —
282 * valid only until the next iteration; the caller MUST consume it
283 * synchronously and not retain it across an await.
284 *
285 * Cut-point determinism: same content + same tier → identical chunk
286 * boundaries as `chunk()`, regardless of input block size. Peers using
287 * either function on the same file produce matching chunk hashes.
288 *
289 * @param {AsyncIterable<Buffer>} readable
290 * @param {{ tier?: number, copy?: boolean }} [opts] tier index (0–3); copy
291 * defaults true. Callers with a known file size should pass `selectTier(size)`.
292 */
293export async function * chunkStream (readable, opts = {}) {
294 const tier = (opts && typeof opts.tier === 'number') ? opts.tier : 0
295 const copy = opts.copy !== false // false → yield views (see JSDoc above)
296 const params = TIERS[tier] || TIERS[0]
297 const masks = TIER_MASKS[tier] || TIER_MASKS[0]
298
299 let pending = Buffer.alloc(0)
300 let absoluteOffset = 0
301
302 // Single emit point for the stats accounting + copy:false decision.
303 const emit = (slice) => {
304 if (STATS_ON) { chunkStats.chunks++; if (copy) chunkStats.copyBytes += slice.length }
305 return { hash: hashChunk(slice), offset: absoluteOffset, length: slice.length, data: copy ? Buffer.from(slice) : slice }
306 }
307
308 for await (const inChunk of readable) {
309 const buf = Buffer.isBuffer(inChunk) ? inChunk : Buffer.from(inChunk)
310 if (STATS_ON) { chunkStats.blocks++; if (pending.length !== 0) chunkStats.concatBytes += pending.length + buf.length }
311 pending = pending.length === 0 ? buf : Buffer.concat([pending, buf])
312
313 // While we have at least maxSize ahead, we can definitively find
314 // the next cut point inside `pending` (the cut is bounded by maxSize).
315 while (pending.length >= params.maxSize) {
316 const cutLen = findCutPoint(pending, params, masks)
317 yield emit(pending.subarray(0, cutLen))
318 absoluteOffset += cutLen
319 pending = pending.subarray(cutLen)
320 }
321 }
322
323 // Stream ended. Process leftover. Same minSize-final-chunk rule as
324 // the buffer-mode `chunk()`.
325 while (pending.length > params.minSize) {
326 const cutLen = findCutPoint(pending, params, masks)
327 yield emit(pending.subarray(0, cutLen))
328 absoluteOffset += cutLen
329 pending = pending.subarray(cutLen)
330 }
331
332 if (pending.length > 0) yield emit(pending)
333}
lib/v3/file-index.js +576 -0
……@@ -0,0 +1,576 @@
1/**
2 * FileIndex — Metadata-only file index backed by Hyperbee
3 *
4 * Stores file metadata and chunk maps. Never stores file content.
5 * The filesystem is the source of truth for actual bytes.
6 *
7 * Hyperbee schema:
8 * file:<path> → { contentHash, size, mtime, hashing, executable, linkname }
9 * chunkmap:<path> → [{ hash, offset, length }] (small) | { __paged: N } (large; see below)
10 * chunkmap-oid:<contentHash> → [{ hash, offset, length }] (content-addressed; reused across overlay keys + restarts)
11 * sync:<peerKey>:<path> → { lastSeq, lastHash }
12 * config:sync → { folders: [...] }
13 *
14 * tree:<hex-hash> → { entries: [{kind, exec, name, childHash, size}], size }
15 * treepath:<path> → <hex-hash> (secondary: path → current tree hash)
16 *
17 * Chunk-map paging. A chunk map for a very large file (a 1.1 TB
18 * file at tier 3 ≈ 1M entries ≈ ~120 MB of JSON) does not fit in one Hyperbee
19 * value: Hypercore caps a block at 15 MiB (MAX_SUGGESTED_BLOCK_SIZE) and throws
20 * BAD_ARGUMENT above it. Maps with > CHUNKS_PER_PAGE entries are stored as a
21 * header value { __paged: N } at the base key plus N page values keyed
22 * `<base>\x00<i>` (the NUL byte never occurs in a path or hex hash, so a page key
23 * can't collide with a real one). Small maps stay inline as a plain array —
24 * byte-identical to before, and back-compatible with already-stored maps. The
25 * public API (put/get/del) is unchanged; paging is transparent to callers.
26 */
27
28import Hyperbee from 'hyperbee'
29import ReadyResource from 'ready-resource'
30
31// Entries per chunk-map page. ~116 B/entry worst-case JSON ⇒
32// ≤ ~3.8 MB/page, well under Hypercore's 15 MiB block limit.
33const CHUNKS_PER_PAGE = 32768
34
35// Overlay-path prefix for host-registered, content-addressed serve entries.
36// Hosts that make a file servable purely by content hash should register it at
37// `${CONTENT_REGISTER_PREFIX}${contentHash}` so compact()/evictContent() can
38// recognize (and reclaim) the entry once the hash is no longer served.
39export const CONTENT_REGISTER_PREFIX = '/mir/'
40
41// The content hash an owner-side entry is addressed by (page suffix folded in), or
42// null for entries not keyed by a single content hash (real-path file:/chunkmap:,
43// tree:, sync:, config:). compact() drops the content-addressed entries whose hash is
44// no longer served, and keeps everything else.
45function contentHashOfKey (key) {
46 if (key.startsWith('chunkmap-oid:')) return key.slice('chunkmap-oid:'.length).split('\x00')[0]
47 if (key.startsWith('chunkmap:content:')) return key.slice('chunkmap:content:'.length).split('\x00')[0]
48 if (key.startsWith(`chunkmap:${CONTENT_REGISTER_PREFIX}`)) return key.slice(`chunkmap:${CONTENT_REGISTER_PREFIX}`.length).split('\x00')[0]
49 if (key.startsWith(`file:${CONTENT_REGISTER_PREFIX}`)) return key.slice(`file:${CONTENT_REGISTER_PREFIX}`.length)
50 return null
51}
52
53// v1 keeps the original 'file-index' name so existing stores aren't orphaned on
54// upgrade; compaction moves to file-index-v2, v3, … and purges the predecessor.
55export function indexCoreName (version) {
56 return version === 1 ? 'file-index' : `file-index-v${version}`
57}
58
59export class FileIndex extends ReadyResource {
60 constructor (store, opts = {}) {
61 super()
62 this._store = store
63 this._bee = null
64 this._meta = null
65 this._version = 1
66 this._opts = opts
67 }
68
69 // Local index cores are encrypted at rest when the caller passes an encryptionKey
70 // (opts.encryptionKey); absent, they stay plaintext.
71 _coreOpts (name) {
72 const o = { name, valueEncoding: 'binary' }
73 if (this._opts.encryptionKey) o.encryptionKey = this._opts.encryptionKey
74 return o
75 }
76
77 async _open () {
78 this._meta = new Hyperbee(this._store.get(this._coreOpts('index-meta')), {
79 keyEncoding: 'utf-8',
80 valueEncoding: 'json'
81 })
82 await this._meta.ready()
83 this._version = (await this._meta.get('version'))?.value ?? 1
84 // A prior purge can leave a version's core with a dangling name alias — its data and
85 // by-discovery-key alias were deleted but the by-name alias was not — which throws
86 // STORAGE_EMPTY on open. The index is rebuildable cache, so advance to a fresh core
87 // name and record it rather than failing open.
88 for (let attempts = 0; ; attempts++) {
89 const bee = new Hyperbee(this._store.get(this._coreOpts(indexCoreName(this._version))), {
90 keyEncoding: 'utf-8',
91 valueEncoding: 'json'
92 })
93 try {
94 await bee.ready()
95 this._bee = bee
96 return
97 } catch (err) {
98 if (err.code !== 'STORAGE_EMPTY' || attempts >= 64) throw err
99 await bee.close().catch(() => {})
100 this._version += 1
101 await this._meta.put('version', this._version)
102 }
103 }
104 }
105
106 async _close () {
107 if (this._bee) await this._bee.close()
108 if (this._meta) await this._meta.close()
109 }
110
111 get bee () { return this._bee }
112
113 get version () { return this._version }
114
115 get cores () { return [this._bee?.core, this._meta?.core].filter(Boolean) }
116
117 // Reclaim the append-only index: stream the still-served entries into a fresh
118 // versioned core, flip the version pointer, and return the old core so the caller
119 // can clear+purge it (the only way to return an append-only bee's disk to the OS).
120 // Content-addressed entries (maps + content-register entries) whose hash isServed() is false
121 // are dropped; real-path, tree, sync and config entries are always kept.
122 async compact ({ isServed }) {
123 // Skip the rewrite entirely when nothing is droppable — otherwise a compaction of
124 // an already-clean index just churns (a fresh version core + a version-marker
125 // append), which grows the index without reclaiming anything.
126 let droppable = false
127 for await (const { key } of this._bee.createReadStream()) {
128 const hash = contentHashOfKey(key)
129 if (hash !== null && !isServed(hash)) { droppable = true; break }
130 }
131 if (!droppable) return null
132
133 const next = this._version + 1
134 const dst = new Hyperbee(this._store.get(this._coreOpts(indexCoreName(next))), {
135 keyEncoding: 'utf-8',
136 valueEncoding: 'json'
137 })
138 await dst.ready()
139 let batch = dst.batch()
140 let pending = 0
141 for await (const { key, value } of this._bee.createReadStream()) {
142 const hash = contentHashOfKey(key)
143 if (hash !== null && !isServed(hash)) continue
144 await batch.put(key, value)
145 if (++pending >= 500) { await batch.flush(); batch = dst.batch(); pending = 0 }
146 }
147 await batch.flush()
148
149 const oldCore = this._bee.core
150 await this._meta.put('version', next)
151 this._bee = dst
152 this._version = next
153 return oldCore // left open; caller clears + purges it
154 }
155
156 // ── File entries ────────────────────────────────────────────
157
158 /**
159 * Store or update a file entry (metadata only, no content)
160 * @param {string} path - file path
161 * @param {{ contentHash: string|null, size: number, mtime: number, hashing?: boolean, executable?: boolean, linkname?: string|null }} meta
162 */
163 async putFile (path, meta) {
164 await this._bee.put(`file:${path}`, {
165 contentHash: meta.contentHash || null,
166 size: meta.size,
167 mtime: meta.mtime,
168 hashing: meta.hashing || false,
169 executable: meta.executable || false,
170 linkname: meta.linkname || null
171 })
172 }
173
174 /**
175 * Get file metadata
176 * @param {string} path
177 * @returns {{ contentHash: string|null, size: number, mtime: number, hashing: boolean, executable: boolean, linkname: string|null } | null}
178 */
179 async getFile (path) {
180 const entry = await this._bee.get(`file:${path}`)
181 return entry ? entry.value : null
182 }
183
184 /**
185 * Check if file exists in the index
186 * @param {string} path
187 * @returns {boolean}
188 */
189 async hasFile (path) {
190 const entry = await this._bee.get(`file:${path}`)
191 return entry !== null
192 }
193
194 /**
195 * Remove a file entry and its chunk map
196 * @param {string} path
197 */
198 async delFile (path) {
199 await this._bee.del(`file:${path}`)
200 await this._delPagedValue(`chunkmap:${path}`)
201 }
202
203 /**
204 * List files under a directory prefix
205 * @param {string} [dir=''] - directory prefix
206 * @returns {Array<{ path: string, contentHash: string|null, size: number, mtime: number }>}
207 */
208 async listFiles (dir = '') {
209 const prefix = dir ? `file:${dir}` : 'file:'
210 const files = []
211
212 for await (const entry of this._bee.createReadStream({ gt: prefix, lt: prefix + '\xff' })) {
213 files.push({
214 path: entry.key.slice('file:'.length),
215 ...entry.value
216 })
217 }
218
219 return files
220 }
221
222 // ── Index meta (from hyper-overlay 0.2.14; carried in v3) ───
223 // Small key/value bag on the main bee under `meta:` — used by the fast
224 // manifest-diff boot path (e.g. hyper-deploy). compact() keeps these
225 // (contentHashOfKey returns null for `meta:` → not droppable).
226
227 async putIndexMeta (key, value) {
228 await this._bee.put(`meta:${key}`, value)
229 }
230
231 async getIndexMeta (key) {
232 const entry = await this._bee.get(`meta:${key}`)
233 return entry ? entry.value : null
234 }
235
236 // ── Chunk maps ──────────────────────────────────────────────
237
238 /**
239 * Store a chunk map for a file (hash + offset + length per chunk, no data).
240 * Large maps are paged transparently (see the header note) so they never exceed the
241 * Hypercore block limit.
242 * @param {string} path - file path
243 * @param {Array<{ hash: string, offset: number, length: number }>} chunks
244 */
245 async putChunkMap (path, chunks) {
246 await this._putPagedValue(`chunkmap:${path}`, chunks)
247 }
248
249 /**
250 * Get the chunk map for a file
251 * @param {string} path
252 * @returns {Array<{ hash: string, offset: number, length: number }> | null}
253 */
254 async getChunkMap (path) {
255 return this._getPagedValue(`chunkmap:${path}`)
256 }
257
258 /**
259 * Check if a chunk map exists for a file
260 * @param {string} path
261 * @returns {boolean}
262 */
263 async hasChunkMap (path) {
264 const entry = await this._bee.get(`chunkmap:${path}`)
265 return entry !== null
266 }
267
268 /**
269 * Remove a chunk map
270 * @param {string} path
271 */
272 async delChunkMap (path) {
273 await this._delPagedValue(`chunkmap:${path}`)
274 }
275
276 // Paged-value storage. Small maps stay inline as a plain array
277 // (unchanged on disk, back-compatible); maps over CHUNKS_PER_PAGE entries are
278 // split into a { __paged: N } header + N `<base>\x00<i>` page values, written
279 // in one Hyperbee batch so the whole map commits atomically. Stale pages from
280 // a previously-larger map at the same key are deleted in the same batch.
281 async _putPagedValue (baseKey, chunks) {
282 const prevPages = await this._pagedCount(baseKey)
283
284 if (chunks.length <= CHUNKS_PER_PAGE) {
285 if (prevPages === 0) {
286 await this._bee.put(baseKey, chunks)
287 return
288 }
289 await this._batched(async (b) => {
290 await b.put(baseKey, chunks)
291 for (let i = 0; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`)
292 })
293 return
294 }
295
296 const pageCount = Math.ceil(chunks.length / CHUNKS_PER_PAGE)
297 await this._batched(async (b) => {
298 await b.put(baseKey, { __paged: pageCount })
299 for (let i = 0; i < pageCount; i++) {
300 await b.put(`${baseKey}\x00${i}`, chunks.slice(i * CHUNKS_PER_PAGE, (i + 1) * CHUNKS_PER_PAGE))
301 }
302 for (let i = pageCount; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`)
303 })
304 }
305
306 async _getPagedValue (baseKey) {
307 const entry = await this._bee.get(baseKey)
308 if (!entry) return null
309 if (Array.isArray(entry.value)) return entry.value // inline / legacy
310 const pageCount = entry.value.__paged
311 if (!pageCount) return null
312 const chunks = []
313 for (let i = 0; i < pageCount; i++) {
314 const page = await this._bee.get(`${baseKey}\x00${i}`)
315 // The header + all pages commit in one atomic batch, so a missing or non-array
316 // page means the value is corrupt/incomplete. Return null — a clean "miss" so the
317 // caller re-chunks from the source file — never a silently truncated chunk map.
318 if (!page || !Array.isArray(page.value)) return null
319 for (const c of page.value) chunks.push(c)
320 }
321 return chunks
322 }
323
324 async _delPagedValue (baseKey) {
325 const prevPages = await this._pagedCount(baseKey)
326 if (prevPages === 0) {
327 await this._bee.del(baseKey)
328 return
329 }
330 await this._batched(async (b) => {
331 await b.del(baseKey)
332 for (let i = 0; i < prevPages; i++) await b.del(`${baseKey}\x00${i}`)
333 })
334 }
335
336 // Page count of an existing paged value, or 0 if the key is absent or inline.
337 async _pagedCount (baseKey) {
338 const entry = await this._bee.get(baseKey)
339 if (!entry || Array.isArray(entry.value)) return 0
340 return entry.value.__paged || 0
341 }
342
343 // Run ops in a Hyperbee batch, closing it on error (flush releases the lock
344 // itself, even on append failure, via _appendBatch's finally).
345 async _batched (fn) {
346 const b = this._bee.batch()
347 try {
348 await fn(b)
349 await b.flush()
350 } catch (err) {
351 try { await b.close() } catch {}
352 throw err
353 }
354 }
355
356 // Content-addressed chunk map. FastCDC is deterministic on bytes, so a
357 // given content hash always chunks identically regardless of which overlay path
358 // it was registered under. Keying by content hash lets the publish-time chunking
359 // be reused at serve time (a synthetic 'content:<hash>' key) and across restarts,
360 // instead of re-reading the whole file to re-chunk it before the first byte ships.
361 async putChunkMapByHash (contentHash, chunks) {
362 await this._putPagedValue(`chunkmap-oid:${contentHash}`, chunks)
363 }
364
365 async getChunkMapByHash (contentHash) {
366 return this._getPagedValue(`chunkmap-oid:${contentHash}`)
367 }
368
369 async hasChunkMapByHash (contentHash) {
370 return (await this._bee.get(`chunkmap-oid:${contentHash}`)) !== null
371 }
372
373 async delChunkMapByHash (contentHash) {
374 await this._delPagedValue(`chunkmap-oid:${contentHash}`)
375 }
376
377 // Evict every durable entry a served content hash leaves behind: the content-
378 // addressed map plus the deterministic owner-side synthetic paths (the serve
379 // copy at content:<hash> and the register entry under CONTENT_REGISTER_PREFIX).
380 // Peer-path-keyed receiver entries are unknowable from the hash — the host's
381 // own index sweep handles those.
382 async evictContent (contentHash) {
383 await this.delChunkMapByHash(contentHash)
384 await this.delChunkMap(`content:${contentHash}`)
385 await this.delFile(`${CONTENT_REGISTER_PREFIX}${contentHash}`)
386 }
387
388 // ── Sync state ──────────────────────────────────────────────
389
390 /**
391 * Store sync state for a peer + file
392 * @param {string} peerKey - hex peer key
393 * @param {string} path - file path
394 * @param {{ lastSeq: number, lastHash: string }} state
395 */
396 async putSyncState (peerKey, path, state) {
397 await this._bee.put(`sync:${peerKey}:${path}`, {
398 lastSeq: state.lastSeq,
399 lastHash: state.lastHash
400 })
401 }
402
403 /**
404 * Get sync state for a peer + file
405 * @param {string} peerKey
406 * @param {string} path
407 * @returns {{ lastSeq: number, lastHash: string } | null}
408 */
409 async getSyncState (peerKey, path) {
410 const entry = await this._bee.get(`sync:${peerKey}:${path}`)
411 return entry ? entry.value : null
412 }
413
414 /**
415 * Remove sync state for a peer + file
416 * @param {string} peerKey
417 * @param {string} path
418 */
419 async delSyncState (peerKey, path) {
420 await this._bee.del(`sync:${peerKey}:${path}`)
421 }
422
423 /**
424 * List all sync states for a peer
425 * @param {string} peerKey
426 * @returns {Array<{ path: string, lastSeq: number, lastHash: string }>}
427 */
428 async listSyncStates (peerKey) {
429 const prefix = `sync:${peerKey}:`
430 const states = []
431
432 for await (const entry of this._bee.createReadStream({ gt: prefix, lt: prefix + '\xff' })) {
433 states.push({
434 path: entry.key.slice(prefix.length),
435 ...entry.value
436 })
437 }
438
439 return states
440 }
441
442 // ── Sync config ─────────────────────────────────────────────
443
444 /**
445 * Store sync configuration
446 * @param {{ folders: Array<{ path: string, strategy: string, quota?: string|null, peers?: string[] }> }} config
447 */
448 async putSyncConfig (config) {
449 await this._bee.put('config:sync', config)
450 }
451
452 /**
453 * Get sync configuration
454 * @returns {{ folders: Array } | null}
455 */
456 async getSyncConfig () {
457 const entry = await this._bee.get('config:sync')
458 return entry ? entry.value : null
459 }
460
461 // ── Trees ───────────────────────────────────────────────────
462
463 /**
464 * Store a tree by its content-addressed hash.
465 * Same hash put twice is a no-op (content-addressed — bytes can't differ).
466 *
467 * @param {string} hash - hex-encoded blake2b-256 tree hash
468 * @param {{ entries: Array, size: number }} tree
469 */
470 async putTree (hash, tree) {
471 await this._bee.put(`tree:${hash}`, {
472 entries: tree.entries,
473 size: tree.size
474 })
475 }
476
477 /**
478 * Get a tree by its hash.
479 * @param {string} hash
480 * @returns {{ entries: Array, size: number } | null}
481 */
482 async getTree (hash) {
483 const entry = await this._bee.get(`tree:${hash}`)
484 return entry ? entry.value : null
485 }
486
487 /**
488 * Check if a tree is present.
489 * @param {string} hash
490 * @returns {boolean}
491 */
492 async hasTree (hash) {
493 const entry = await this._bee.get(`tree:${hash}`)
494 return entry !== null
495 }
496
497 /**
498 * Remove a tree. Only safe when no treepath entries reference this hash.
499 * @param {string} hash
500 */
501 async delTree (hash) {
502 await this._bee.del(`tree:${hash}`)
503 }
504
505 /**
506 * Record the current tree hash for a path (secondary index).
507 * @param {string} path
508 * @param {string} hash
509 */
510 async putTreePath (path, hash) {
511 await this._bee.put(`treepath:${path}`, hash)
512 }
513
514 /**
515 * Look up the current tree hash for a path.
516 * @param {string} path
517 * @returns {string|null}
518 */
519 async getTreePath (path) {
520 const entry = await this._bee.get(`treepath:${path}`)
521 return entry ? entry.value : null
522 }
523
524 /**
525 * Remove the treepath entry. Does NOT delete the underlying tree — other
526 * paths may still reference the same hash.
527 * @param {string} path
528 */
529 async delTreePath (path) {
530 await this._bee.del(`treepath:${path}`)
531 }
532
533 /**
534 * List all stored tree hashes.
535 * @param {object} [opts]
536 * @param {number} [opts.limit] - optional cap
537 * @returns {Array<{ hash: string, entryCount: number, size: number }>}
538 */
539 async listTrees (opts = {}) {
540 const trees = []
541 for await (const entry of this._bee.createReadStream({ gt: 'tree:', lt: 'tree:\xff' })) {
542 trees.push({
543 hash: entry.key.slice('tree:'.length),
544 entryCount: entry.value.entries.length,
545 size: entry.value.size
546 })
547 if (opts.limit && trees.length >= opts.limit) break
548 }
549 return trees
550 }
551
552 // ── Stats ───────────────────────────────────────────────────
553
554 /**
555 * Get index statistics
556 * @returns {{ fileCount: number, chunkMapCount: number, syncStateCount: number, treeCount: number, treePathCount: number }}
557 */
558 async stats () {
559 let fileCount = 0
560 let chunkMapCount = 0
561 let syncStateCount = 0
562 let treeCount = 0
563 let treePathCount = 0
564
565 for await (const entry of this._bee.createReadStream()) {
566 if (entry.key.startsWith('file:')) fileCount++
567 // Count one per file — skip the `\x00`-suffixed page values.
568 else if (entry.key.startsWith('chunkmap:') && !entry.key.includes('\x00')) chunkMapCount++
569 else if (entry.key.startsWith('sync:')) syncStateCount++
570 else if (entry.key.startsWith('tree:')) treeCount++
571 else if (entry.key.startsWith('treepath:')) treePathCount++
572 }
573
574 return { fileCount, chunkMapCount, syncStateCount, treeCount, treePathCount }
575 }
576}
lib/v3/index.js +46 -0
……@@ -0,0 +1,46 @@
1/**
2 * hyper-overlay — content-addressed P2P file distribution over Protomux.
3 *
4 * The canonical bytes are REAL FILES ON DISK; the overlay stores only
5 * metadata + chunk maps (Hyperbee), so there is no blob-storage balloon.
6 * Files are chunked with FastCDC, fetched multi-source (torrent-style),
7 * hash-verified incrementally, and written straight to the requester's disk.
8 *
9 * Main entry point: `HyperOverlay` (also the default export). The submodules
10 * are available as subpath exports (`hyper-overlay/chunker`, …) for consumers
11 * that only need a slice of the stack.
12 *
13 * Runs on Bare. The pure modules — chunker, messages, chunk-scheduler,
14 * file-index, sync-engine — also run on Node.
15 */
16
17export { HyperOverlay, HyperOverlay as default } from './lib/overlay.js'
18export { OverlayProtocol } from './lib/protocol.js'
19export {
20 TransferManager,
21 PARTIAL_SUFFIX,
22 partialPathFor,
23 journalNameFor,
24 cleanupOrphanedJournals
25} from './lib/transfer.js'
26export { ChunkScheduler } from './lib/chunk-scheduler.js'
27export { FileIndex, CONTENT_REGISTER_PREFIX } from './lib/file-index.js'
28export { SyncEngine, STRATEGIES } from './lib/sync-engine.js'
29export {
30 chunk,
31 chunkStream,
32 hashChunk,
33 createStreamingHasher,
34 selectTier,
35 getTierParams,
36 TIERS,
37 MIN_SIZE,
38 AVG_SIZE,
39 MAX_SIZE
40} from './lib/chunker.js'
41export * as messages from './lib/messages.js'
42
43// Transitional aliases for consumers migrating from the pre-3.0 v2 naming
44// (`overlay-v2.js` / `protocol-v2.js`). Same classes; prefer the new names.
45export { HyperOverlay as HyperOverlayV2 } from './lib/overlay.js'
46export { OverlayProtocol as OverlayProtocolV2 } from './lib/protocol.js'
lib/v3/messages.js +444 -0
……@@ -0,0 +1,444 @@
1/**
2 * Wire Protocol Messages for hyper-overlay/v2
3 *
4 * 13 message types for sync + transfer + trees (0.5a extension):
5 * 0-2: Sync messages (state exchange, file offers/requests)
6 * 3-8: Transfer messages (chunk hashes, data, completion)
7 * 9-11: Tree messages (tree-request, tree-response, content-request)
8 * 12: transfer-control (downloader→holder pause/stop notice)
9 *
10 * Messages 9-12 are backward-compatible additions — peers without
11 * this code simply don't register the corresponding protomux slots, and
12 * senders see no receiver for those messages. File-level sync (0-8)
13 * keeps working unchanged.
14 *
15 * All use compact-encoding codecs with preencode/encode/decode.
16 */
17
18import c from 'compact-encoding'
19
20// ── Helpers ───────────────────────────────────────────────────
21
22function toBuffer32 (hashOrBuf) {
23 if (Buffer.isBuffer(hashOrBuf)) return hashOrBuf
24 return Buffer.from(hashOrBuf, 'hex')
25}
26
27// Encode/decode an array of { hash (32-byte hex), length (uint) }
28const chunkInfoArray = {
29 preencode (state, arr) {
30 c.uint32.preencode(state, arr.length)
31 for (const item of arr) {
32 state.end += 32
33 c.uint.preencode(state, item.length)
34 }
35 },
36 encode (state, arr) {
37 c.uint32.encode(state, arr.length)
38 for (const item of arr) {
39 toBuffer32(item.hash).copy(state.buffer, state.start, 0, 32)
40 state.start += 32
41 c.uint.encode(state, item.length)
42 }
43 },
44 decode (state) {
45 const len = c.uint32.decode(state)
46 const arr = []
47 for (let i = 0; i < len; i++) {
48 const hash = state.buffer.subarray(state.start, state.start + 32).toString('hex')
49 state.start += 32
50 const length = c.uint.decode(state)
51 arr.push({ hash, length })
52 }
53 return arr
54 }
55}
56
57// Encode/decode an array of uints
58const uintArray = {
59 preencode (state, arr) {
60 c.uint32.preencode(state, arr.length)
61 for (const v of arr) c.uint.preencode(state, v)
62 },
63 encode (state, arr) {
64 c.uint32.encode(state, arr.length)
65 for (const v of arr) c.uint.encode(state, v)
66 },
67 decode (state) {
68 const len = c.uint32.decode(state)
69 const arr = []
70 for (let i = 0; i < len; i++) arr.push(c.uint.decode(state))
71 return arr
72 }
73}
74
75// ── Handshake ─────────────────────────────────────────────────
76
77export const handshake = {
78 preencode (state, m) {
79 c.uint8.preencode(state, m.version)
80 c.uint8.preencode(state, m.capabilities)
81 },
82 encode (state, m) {
83 c.uint8.encode(state, m.version)
84 c.uint8.encode(state, m.capabilities)
85 },
86 decode (state) {
87 return {
88 version: c.uint8.decode(state),
89 capabilities: c.uint8.decode(state)
90 }
91 }
92}
93
94// ── Sync Messages (0-2) ──────────────────────────────────────
95
96// Message 0: sync-state — exchange last-synced sequence numbers
97export const syncState = {
98 preencode (state, m) {
99 c.buffer.preencode(state, toBuffer32(m.feedKey))
100 c.uint.preencode(state, m.localSeq)
101 c.uint.preencode(state, m.remoteSeq)
102 },
103 encode (state, m) {
104 c.buffer.encode(state, toBuffer32(m.feedKey))
105 c.uint.encode(state, m.localSeq)
106 c.uint.encode(state, m.remoteSeq)
107 },
108 decode (state) {
109 return {
110 feedKey: c.buffer.decode(state).toString('hex'),
111 localSeq: c.uint.decode(state),
112 remoteSeq: c.uint.decode(state)
113 }
114 }
115}
116
117// Message 1: file-offer — announce a file change
118export const fileOffer = {
119 preencode (state, m) {
120 c.string.preencode(state, m.path)
121 c.buffer.preencode(state, toBuffer32(m.contentHash))
122 c.uint.preencode(state, m.size)
123 c.uint.preencode(state, m.mtime)
124 c.uint8.preencode(state, m.op)
125 },
126 encode (state, m) {
127 c.string.encode(state, m.path)
128 c.buffer.encode(state, toBuffer32(m.contentHash))
129 c.uint.encode(state, m.size)
130 c.uint.encode(state, m.mtime)
131 c.uint8.encode(state, m.op)
132 },
133 decode (state) {
134 return {
135 path: c.string.decode(state),
136 contentHash: c.buffer.decode(state).toString('hex'),
137 size: c.uint.decode(state),
138 mtime: c.uint.decode(state),
139 op: c.uint8.decode(state)
140 }
141 }
142}
143
144// Message 2: file-request — request a file transfer
145export const fileRequest = {
146 preencode (state, m) {
147 c.string.preencode(state, m.path)
148 c.buffer.preencode(state, toBuffer32(m.contentHash))
149 c.buffer.preencode(state, m.chunksHave || Buffer.alloc(0))
150 },
151 encode (state, m) {
152 c.string.encode(state, m.path)
153 c.buffer.encode(state, toBuffer32(m.contentHash))
154 c.buffer.encode(state, m.chunksHave || Buffer.alloc(0))
155 },
156 decode (state) {
157 return {
158 path: c.string.decode(state),
159 contentHash: c.buffer.decode(state).toString('hex'),
160 chunksHave: c.buffer.decode(state)
161 }
162 }
163}
164
165// ── Transfer Messages (3-8) ──────────────────────────────────
166
167// Message 3: chunk-hashes — chunk hash list for a requested file
168//
169// `more` (paging flag) is appended last. A very large file's
170// chunk list serializes past the 16 MiB-1 Noise frame limit
171// (@hyperswarm/secret-stream MAX_ATOMIC_WRITE), so the protocol splits it into
172// several chunkHashes frames: every page but the last sets more:1, the final
173// page sets more:0. The receiver concatenates pages (in arrival order, keyed by
174// path) before dispatching. Appended last so a pre-paging peer that omits it
175// decodes more:0 — i.e. as a single, complete page (back-compatible).
176export const chunkHashes = {
177 preencode (state, m) {
178 c.string.preencode(state, m.path)
179 c.uint8.preencode(state, m.tier)
180 chunkInfoArray.preencode(state, m.chunks)
181 c.uint8.preencode(state, m.more || 0)
182 },
183 encode (state, m) {
184 c.string.encode(state, m.path)
185 c.uint8.encode(state, m.tier)
186 chunkInfoArray.encode(state, m.chunks)
187 c.uint8.encode(state, m.more || 0)
188 },
189 decode (state) {
190 const path = c.string.decode(state)
191 const tier = c.uint8.decode(state)
192 const chunks = chunkInfoArray.decode(state)
193 // A frame from a pre-paging peer has no trailing byte → more:0.
194 const more = state.start < state.end ? c.uint8.decode(state) : 0
195 return { path, tier, chunks, more }
196 }
197}
198
199// Message 4: chunk-need — tell sender which chunks to send
200export const chunkNeed = {
201 preencode (state, m) {
202 c.string.preencode(state, m.path)
203 uintArray.preencode(state, m.indices)
204 },
205 encode (state, m) {
206 c.string.encode(state, m.path)
207 uintArray.encode(state, m.indices)
208 },
209 decode (state) {
210 return {
211 path: c.string.decode(state),
212 indices: uintArray.decode(state)
213 }
214 }
215}
216
217// Message 5: chunk-data — send chunk bytes
218export const chunkData = {
219 preencode (state, m) {
220 c.string.preencode(state, m.path)
221 c.uint.preencode(state, m.index)
222 c.buffer.preencode(state, m.data)
223 },
224 encode (state, m) {
225 c.string.encode(state, m.path)
226 c.uint.encode(state, m.index)
227 c.buffer.encode(state, m.data)
228 },
229 decode (state) {
230 return {
231 path: c.string.decode(state),
232 index: c.uint.decode(state),
233 data: c.buffer.decode(state)
234 }
235 }
236}
237
238// Message 6: chunk-cancel — cancel a pending transfer
239export const chunkCancel = {
240 preencode (state, m) { c.string.preencode(state, m.path) },
241 encode (state, m) { c.string.encode(state, m.path) },
242 decode (state) { return { path: c.string.decode(state) } }
243}
244
245// Message 7: transfer-complete — confirm file fully received
246export const transferComplete = {
247 preencode (state, m) {
248 c.string.preencode(state, m.path)
249 c.buffer.preencode(state, toBuffer32(m.contentHash))
250 },
251 encode (state, m) {
252 c.string.encode(state, m.path)
253 c.buffer.encode(state, toBuffer32(m.contentHash))
254 },
255 decode (state) {
256 return {
257 path: c.string.decode(state),
258 contentHash: c.buffer.decode(state).toString('hex')
259 }
260 }
261}
262
263// Message 8: conflict — notify peer of a detected conflict
264export const conflict = {
265 preencode (state, m) {
266 c.string.preencode(state, m.path)
267 c.buffer.preencode(state, toBuffer32(m.myHash))
268 c.buffer.preencode(state, toBuffer32(m.theirHash))
269 c.buffer.preencode(state, toBuffer32(m.ancestorHash || Buffer.alloc(32)))
270 },
271 encode (state, m) {
272 c.string.encode(state, m.path)
273 c.buffer.encode(state, toBuffer32(m.myHash))
274 c.buffer.encode(state, toBuffer32(m.theirHash))
275 c.buffer.encode(state, toBuffer32(m.ancestorHash || Buffer.alloc(32)))
276 },
277 decode (state) {
278 return {
279 path: c.string.decode(state),
280 myHash: c.buffer.decode(state).toString('hex'),
281 theirHash: c.buffer.decode(state).toString('hex'),
282 ancestorHash: c.buffer.decode(state).toString('hex')
283 }
284 }
285}
286
287// ── Tree Messages (9-11) — 0.5a extension ────────────────────
288
289// Encode/decode an array of tree entries: { kind, exec, name, childHash, size }
290const treeEntryArray = {
291 preencode (state, arr) {
292 c.uint32.preencode(state, arr.length)
293 for (const e of arr) {
294 c.uint8.preencode(state, e.kind)
295 c.uint8.preencode(state, e.exec || 0)
296 c.string.preencode(state, e.name)
297 state.end += 32 // childHash fixed32
298 c.uint.preencode(state, e.size || 0)
299 }
300 },
301 encode (state, arr) {
302 c.uint32.encode(state, arr.length)
303 for (const e of arr) {
304 c.uint8.encode(state, e.kind)
305 c.uint8.encode(state, e.exec || 0)
306 c.string.encode(state, e.name)
307 toBuffer32(e.childHash).copy(state.buffer, state.start, 0, 32)
308 state.start += 32
309 c.uint.encode(state, e.size || 0)
310 }
311 },
312 decode (state) {
313 const len = c.uint32.decode(state)
314 const arr = []
315 for (let i = 0; i < len; i++) {
316 const kind = c.uint8.decode(state)
317 const exec = c.uint8.decode(state)
318 const name = c.string.decode(state)
319 const childHash = state.buffer.subarray(state.start, state.start + 32).toString('hex')
320 state.start += 32
321 const size = c.uint.decode(state)
322 arr.push({ kind, exec, name, childHash, size })
323 }
324 return arr
325 }
326}
327
328// Message 9: tree-request — ask peer for a tree by its content hash
329//
330// `nonce` (per-request id) is appended last; the holder echoes it in every
331// treeResponse page so the requester can drop pages from a superseded attempt.
332// A pre-nonce peer omits it and decodes 0, which the requester treats as
333// "unverifiable".
334export const treeRequest = {
335 preencode (state, m) {
336 c.buffer.preencode(state, toBuffer32(m.hash))
337 c.uint.preencode(state, m.nonce || 0)
338 },
339 encode (state, m) {
340 c.buffer.encode(state, toBuffer32(m.hash))
341 c.uint.encode(state, m.nonce || 0)
342 },
343 decode (state) {
344 const hash = c.buffer.decode(state).toString('hex')
345 const nonce = state.start < state.end ? c.uint.decode(state) : 0
346 return { hash, nonce }
347 }
348}
349
350// Message 10: tree-response — return a tree's entries to the requester
351//
352// `more` (paging flag) mirrors chunkHashes: a tree too large for one 16 MiB-1
353// Noise frame is split into several frames, every page but the last more:1.
354// `nonce` echoes the requester's per-request id so pages from a superseded
355// attempt are dropped. Both are appended last; a pre-paging peer omits them and
356// decodes 0.
357export const treeResponse = {
358 preencode (state, m) {
359 c.buffer.preencode(state, toBuffer32(m.hash))
360 treeEntryArray.preencode(state, m.entries)
361 c.uint8.preencode(state, m.more || 0)
362 c.uint.preencode(state, m.nonce || 0)
363 },
364 encode (state, m) {
365 c.buffer.encode(state, toBuffer32(m.hash))
366 treeEntryArray.encode(state, m.entries)
367 c.uint8.encode(state, m.more || 0)
368 c.uint.encode(state, m.nonce || 0)
369 },
370 decode (state) {
371 const hash = c.buffer.decode(state).toString('hex')
372 const entries = treeEntryArray.decode(state)
373 const more = state.start < state.end ? c.uint8.decode(state) : 0
374 const nonce = state.start < state.end ? c.uint.decode(state) : 0
375 return { hash, entries, more, nonce }
376 }
377}
378
379// Message 11: content-request — fetch a file by its content hash (not path)
380// Sender locates any local file matching the hash and serves its chunks.
381// `from` carries the requester's profile-key (hex) so the
382// holder's serveAuthorizer can authenticate the asker. Appended last → an
383// overlay peer that predates this field decodes '' (empty), never crashes.
384export const contentRequest = {
385 preencode (state, m) {
386 c.buffer.preencode(state, toBuffer32(m.contentHash))
387 c.buffer.preencode(state, m.chunksHave || Buffer.alloc(0))
388 c.string.preencode(state, m.from || '')
389 },
390 encode (state, m) {
391 c.buffer.encode(state, toBuffer32(m.contentHash))
392 c.buffer.encode(state, m.chunksHave || Buffer.alloc(0))
393 c.string.encode(state, m.from || '')
394 },
395 decode (state) {
396 return {
397 contentHash: c.buffer.decode(state).toString('hex'),
398 chunksHave: c.buffer.decode(state),
399 from: c.string.decode(state)
400 }
401 }
402}
403
404// Message 12: transfer-control — downloader→holder pause/stop notice for a
405// content-addressed fetch. state: 0 = paused, 1 = stopped. Appended last so a
406// holder that predates this never registers slot 12 and silently ignores it.
407export const transferControl = {
408 preencode (state, m) {
409 c.buffer.preencode(state, toBuffer32(m.contentHash))
410 c.uint8.preencode(state, m.state)
411 },
412 encode (state, m) {
413 c.buffer.encode(state, toBuffer32(m.contentHash))
414 c.uint8.encode(state, m.state)
415 },
416 decode (state) {
417 return {
418 contentHash: c.buffer.decode(state).toString('hex'),
419 state: c.uint8.decode(state)
420 }
421 }
422}
423
424// Message 13: transfer-progress — downloader→holder one-shot have-baseline for a
425// resumed content-addressed fetch. `have` = bytes the downloader already holds on
426// disk at resume, so the holder's "who is downloading" bar reflects the downloader's
427// TRUE completion, not only the bytes it re-serves. Appended last so a holder that
428// predates this never registers slot 13 and silently ignores it.
429export const transferProgress = {
430 preencode (state, m) {
431 c.buffer.preencode(state, toBuffer32(m.contentHash))
432 c.uint.preencode(state, m.have || 0)
433 },
434 encode (state, m) {
435 c.buffer.encode(state, toBuffer32(m.contentHash))
436 c.uint.encode(state, m.have || 0)
437 },
438 decode (state) {
439 return {
440 contentHash: c.buffer.decode(state).toString('hex'),
441 have: c.uint.decode(state)
442 }
443 }
444}
lib/v3/overlay.js +417 -0
……@@ -0,0 +1,417 @@
1/**
2 * HyperOverlay — content-addressed file serving over the V2 protocol.
3 *
4 * A thin facade over the V2 stack (FileIndex + SyncEngine + TransferManager +
5 * OverlayProtocol) for the "serve a file to whoever asks, fetch a file from
6 * whoever has it" use case — distinct from V1's local chunk-registry blobs.
7 *
8 * Model (the deliberate post-Hyperdrive design): the canonical bytes are a
9 * REAL FILE ON DISK. The index holds only metadata + chunk maps (no blob
10 * bytes), so there is no hypercore storage balloon. A peer fetches a file by
11 * its content hash; chunks are read from the source file on demand, streamed,
12 * hash-verified, and written to the requester's disk.
13 *
14 * `autoSync` is OFF: this is on-demand content fetch (requestContent →
15 * _onContentRequest), NOT Syncthing-style folder push. Registering a file
16 * makes it servable; nothing is offered to peers unsolicited.
17 *
18 * Requester flow: pre-register a synthetic `content:<hash>` → destPath
19 * mapping, requestContent,
20 * await the hash-verified transferComplete, walk peers on timeout/mismatch.
21 */
22
23import ReadyResource from 'ready-resource'
24import fs from '#fs'
25import path from '#path'
26import os from '#os'
27import crypto from 'hypercore-crypto'
28
29import { FileIndex } from './file-index.js'
30import { SyncEngine } from './sync-engine.js'
31import { TransferManager } from './transfer.js'
32import { OverlayProtocol } from './protocol.js'
33import { createStreamingHasher } from './chunker.js'
34
35const DEFAULT_PER_PEER_TIMEOUT = 30000
36const DEFAULT_IDLE_TIMEOUT = 10000
37
38// Stream-verify a file against its overlay content hash WITHOUT buffering
39// it — a readFileSync of a multi-GB blob OOMs the process. Matches
40// crypto.data(buffer): blake2b(LEAF_TYPE || uint64-LE(size) || bytes), which is
41// exactly createStreamingHasher({ size }). Returns the size so callers can re-seed
42// without a second stat.
43async function verifyOnDisk (filePath, contentHash) {
44 let size
45 try { size = fs.statSync(filePath).size } catch { return { ok: false, size: 0 } }
46 const h = createStreamingHasher({ size })
47 try {
48 await new Promise((resolve, reject) => {
49 const rs = fs.createReadStream(filePath)
50 rs.on('data', (c) => h.update(c))
51 rs.on('end', resolve)
52 rs.on('error', reject)
53 })
54 } catch { return { ok: false, size } }
55 return { ok: h.digest() === contentHash, size }
56}
57
58export class HyperOverlay extends ReadyResource {
59 constructor (corestore, opts = {}) {
60 super()
61 this._corestore = corestore.namespace(opts.namespace || 'overlay-v2')
62 this._destDir = opts.destDir || path.join(os.tmpdir(), 'hyper-overlay-v2-fetch')
63 this._journalDir = opts.journalDir || null
64 this._syncBaseDirs = opts.syncBaseDirs || []
65 this._perPeerTimeout = opts.perPeerTimeout || DEFAULT_PER_PEER_TIMEOUT
66 this._idleTimeout = opts.idleTimeout || DEFAULT_IDLE_TIMEOUT
67
68 // Serve-authorization + identity opts threaded to OverlayProtocol.
69 // serveAuthorizer gates every inbound content-request; localProfileKey stamps
70 // outbound content-requests so the remote's gate can authenticate the asker.
71 this._serveAuthorizer = opts.serveAuthorizer || null
72 this._localProfileKey = opts.localProfileKey || null
73
74 // At-rest key for the local index cores (file-index/index-meta/sync-feed).
75 this._indexEncryptionKey = opts.indexEncryptionKey || null
76
77 // serve-side download telemetry (sender-side download indicator).
78 this._onServeStart = opts.onServeStart || null
79 this._onChunkServe = opts.onChunkServe || null
80 this._onServeEnd = opts.onServeEnd || null
81 this._onServeControl = opts.onServeControl || null
82 this._onServeProgress = opts.onServeProgress || null
83
84 // Shared maps the protocol reads: overlayPath → diskPath (for serving an
85 // offered/known path), contentHash → diskPath (fast path for content
86 // requests). Both populated by registerFile.
87 this._filePaths = new Map()
88 this._contentHashPaths = new Map()
89
90 // contentHash → pending fetch { destPath, finish, timers, bytesReceived }
91 this._pendingContent = new Map()
92
93 // Built lazily on first use (Hyperdrive-style: a drive that never touches
94 // its blobs core never opens one). A daemon that never registers, serves,
95 // or connects a peer for attachments pays nothing — no feed, no index.
96 this._index = null
97 this._sync = null
98 this._transfer = null
99 this._protocol = null
100 this._stackPromise = null
101 }
102
103 // The protocol must exist synchronously when attachProtocol is called —
104 // protomux will NOT pair a channel created after the remote opened theirs
105 // (verified). So the stack is built at ready time. _ensure stays as the
106 // idempotent builder (used by register/fetch, and as a safety net).
107 async _open () { await this._ensure() }
108
109 async _close () {
110 // Only tear down what was actually built.
111 if (this._protocol) this._protocol.destroy()
112 if (this._sync) await this._sync.close()
113 if (this._index) await this._index.close()
114 }
115
116 // Lazily build the V2 stack. Idempotent + concurrency-safe (a single
117 // in-flight promise is shared). No-op cost once built.
118 _ensure () {
119 if (this._protocol) return Promise.resolve()
120 if (this._stackPromise) return this._stackPromise
121 this._stackPromise = (async () => {
122 const index = new FileIndex(this._corestore, { encryptionKey: this._indexEncryptionKey })
123 await index.ready()
124 const sync = new SyncEngine(index, this._corestore, { encryptionKey: this._indexEncryptionKey })
125 await sync.ready()
126 const transfer = new TransferManager(index, { journalDir: this._journalDir })
127 const protocol = new OverlayProtocol(sync, transfer, {
128 filePaths: this._filePaths,
129 contentHashPaths: this._contentHashPaths,
130 syncBaseDirs: this._syncBaseDirs,
131 autoSync: false,
132 serveAuthorizer: this._serveAuthorizer, // serve gate
133 localProfileKey: this._localProfileKey, // outbound identity
134 onSynced: (info) => this._onSynced(info),
135 onChunkProgress: (info) => this._onChunkProgress(info),
136 onServeStart: this._onServeStart,
137 onChunkServe: this._onChunkServe,
138 onServeEnd: this._onServeEnd,
139 onServeControl: this._onServeControl,
140 onServeProgress: this._onServeProgress
141 })
142 try { fs.mkdirSync(this._destDir, { recursive: true }) } catch {}
143 if (this._journalDir) { try { fs.mkdirSync(this._journalDir, { recursive: true }) } catch {} }
144 this._index = index
145 this._sync = sync
146 this._transfer = transfer
147 this._protocol = protocol
148 })()
149 return this._stackPromise
150 }
151
152 // The host's swarm transport must call this synchronously per connection. The
153 // channel MUST be created synchronously — protomux will not pair a channel
154 // opened after the remote's (verified). The stack is built at ready, so
155 // this._protocol is always present by the time connections arrive.
156 attachProtocol (mux) {
157 if (this.closing || this.closed || !this._protocol) return
158 return this._protocol.attach(mux)
159 }
160
161 get peerCount () { return this._protocol ? this._protocol.peerCount : 0 }
162
163 // The overlay's local-only cores (file-index + version marker + sync feed), so the
164 // leftover scan can treat them as wanted and the storage breakdown can size them.
165 localCores () {
166 const cores = this._index ? this._index.cores : []
167 if (this._sync?.feed) cores.push(this._sync.feed)
168 return cores
169 }
170
171 cleanJournals (maxAge) { try { return this._transfer?.cleanJournals(maxAge) || [] } catch { return [] } }
172
173 /**
174 * Make an on-disk file servable by its content hash.
175 * @param {string} overlayPath logical path, e.g. '/attachments/<oid>'
176 * @param {string} diskPath absolute path to the canonical bytes on disk
177 * @param {object} [meta]
178 * meta.contentHash preferred — the oid the caller already computed
179 * meta.size file size (avoids an extra stat if known)
180 * @returns {Promise<{ contentHash, size } | null>} null if the source vanished
181 */
182 async registerFile (overlayPath, diskPath, meta = {}) {
183 await this._ensure()
184 // Source can vanish between prepare and register (a file moved out of a
185 // shared folder mid-scan). Nothing to serve — bail with null instead of throwing
186 // so a routine move doesn't crash the publish path.
187 let stat
188 try {
189 stat = fs.statSync(diskPath)
190 } catch {
191 return null
192 }
193 const size = typeof meta.size === 'number' ? meta.size : stat.size
194 const contentHash = meta.contentHash || await hashFileOnDisk(diskPath, size)
195
196 await this._index.putFile(overlayPath, { contentHash, size, mtime: stat.mtimeMs })
197 this._filePaths.set(overlayPath, diskPath)
198 this._contentHashPaths.set(contentHash, diskPath)
199 // Build (and cache) the chunk map so a content request can stream
200 // immediately without re-chunking under the request. meta.prepare:false
201 // skips this eager full second read of the file — the file is already servable
202 // via the maps above and the serve path (_onContentRequest) chunks lazily on
203 // first request; publishers that already built the chunk map in one pass
204 // (prepareForServe) pass false so registration doesn't re-read the file.
205 if (meta.prepare !== false) await this._transfer.prepareFile(diskPath, overlayPath)
206 // Record in the local change feed (used by the protocol's markSynced
207 // bookkeeping; with autoSync off it is not broadcast).
208 await this._sync.logPut(overlayPath, { contentHash, size, mtime: stat.mtimeMs })
209 return { contentHash, size }
210 }
211
212 // One-pass hash + chunk-map build for the publish path: streams the file
213 // ONCE, persists the content-addressed chunk map (durable in FileIndex), and
214 // returns the content hash. Replaces a hash-only pass plus a lazy fetch-time
215 // re-chunk with a single read, so the first fetcher never waits on chunk indexing.
216 async prepareForServe (diskPath, opts = {}) {
217 await this._ensure()
218 // The overlay path below is a throwaway key — byHashOnly persists ONLY the
219 // content-addressed chunk map, nothing under this path.
220 const prepared = await this._transfer.prepareFile(diskPath, '/mir-prep' + diskPath, { onProgress: opts.onProgress, byHashOnly: true, signal: opts.signal })
221 return prepared ? { contentHash: prepared.contentHash, size: prepared.size } : null
222 }
223
224 async evictContent (contentHash) {
225 await this._ensure()
226 return this._index.evictContent(contentHash)
227 }
228
229 async compactIndex (opts) {
230 await this._ensure()
231 return this._index.compact(opts)
232 }
233
234 /**
235 * Fetch a file by its content hash. Serves from the local disk copy if we
236 * have it; otherwise pulls from a connected peer that does (hash-verified).
237 * @param {string} contentHash
238 * @param {object} [opts] opts.timeout (idle), opts.destPath
239 * @returns {Promise<{ destPath: string, local: boolean, size: number } | null>}
240 */
241 async fetchFile (contentHash, opts = {}) {
242 await this._ensure()
243
244 const localDisk = this._contentHashPaths.get(contentHash)
245 if (localDisk) {
246 const v = await verifyOnDisk(localDisk, contentHash)
247 if (v.ok) return { destPath: localDisk, local: true, size: v.size }
248 }
249
250 // Wait briefly for at least one peer — a connection may be mid-handshake
251 // (lazy attach) or just forming when the fetch is issued. Without this a
252 // fetch issued the instant a peer connects would spuriously 404.
253 const peerWaitMs = opts.peerWaitMs || 3000
254 for (let waited = 0; this._protocol._peers.size === 0 && waited < peerWaitMs; waited += 100) {
255 await new Promise(r => setTimeout(r, 100))
256 }
257 if (this._protocol._peers.size === 0) {
258 // No fetchContent will run, so drop any cancel recorded during the wait — a
259 // stale marker would otherwise cancel the next fetch of the same content.
260 this._protocol.clearCancelPending(contentHash)
261 return null
262 }
263
264 // Multi-source: fetch chunks in parallel from ALL connected peers that
265 // have the file (torrent-style). The scheduler dedups + fails over.
266 const peers = [...this._protocol._peers.values()]
267 // opts.mirrorSpool: store the verified blob CONTENT-ADDRESSED at
268 // <mirrorSpool>/<contentHash> rather than a throwaway destDir name. This is
269 // the mirror path — a node with no working tree of its own fetches files
270 // into a content-addressed spool so it durably re-serves them (via the
271 // <syncBaseDir>/<contentHash> resolution in _onContentRequest).
272 let destPath
273 if (opts.destPath) {
274 destPath = opts.destPath
275 } else if (opts.mirrorSpool) {
276 try { fs.mkdirSync(opts.mirrorSpool, { recursive: true }) } catch {}
277 destPath = path.join(opts.mirrorSpool, contentHash)
278 } else {
279 destPath = path.join(
280 this._destDir,
281 'blob-' + contentHash + '-' + Date.now() + '-' + Math.random().toString(36).slice(2, 8)
282 )
283 }
284 // Whole-file integrity is verified INCREMENTALLY during the transfer (the
285 // scheduler passes contentHash to TransferManager, which hashes the file in
286 // offset order as chunks land and rejects in finalize on a mismatch). So the
287 // assembled file is already verified here — no trailing whole-file re-read.
288 let result
289 try {
290 result = await this._protocol.fetchContent(contentHash, peers, {
291 destPath,
292 timeout: opts.timeout,
293 onProgress: opts.onProgress || (() => {}), // forward bytes/total
294 onVerify: opts.onVerify, // resume re-verify fraction
295 onEnd: opts.onEnd // terminal diagnostic (reason + bytes/chunks)
296 })
297 } catch (err) {
298 // An integrity failure or an explicit cancel/pause is distinct from a
299 // no-holder / stall — surface it so the caller doesn't treat it as a failure.
300 if (err?.code === 'EHASHMISMATCH' || err?.code === 'ECANCELLED') throw err
301 // A local I/O error (full disk / read-only / permission / vanished
302 // mount) is not a no-holder — surface it so the consumer can pause rather than
303 // retry forever.
304 if (err?.code === 'ENOSPC' || err?.code === 'EACCES' || err?.code === 'EROFS' || err?.code === 'EPERM' || err?.code === 'ENOENT') throw err
305 return null
306 }
307 const size = result?.size ?? fs.statSync(destPath).size
308 // Seeding multiplication: register the verified blob locally so this node
309 // now SERVES these chunks to other peers too. A file held by N peers then
310 // gets faster as more peers fetch it (torrent-style). Best-effort — the
311 // fetch still succeeds even if re-registration fails. Skipped if a peer
312 // raced us to register the same hash (registerFile is idempotent-safe).
313 // reSeed:false — opt-out for consumers that should hold nothing.
314 if (opts.reSeed !== false && !this._contentHashPaths.has(contentHash)) {
315 try {
316 await this.registerFile('/attachments/' + contentHash, destPath, { contentHash, size })
317 } catch {}
318 }
319 return { destPath, local: false, size }
320 }
321
322 // Stop an in-flight fetchFile for this content hash. opts.discardPartial
323 // true unlinks the partial (cancel), false keeps it for resume (pause). The
324 // pending fetchFile rejects with ECANCELLED, which the caller treats as not-a-failure.
325 cancelFetch (contentHash, opts = {}) {
326 return this._protocol ? this._protocol.cancelContent(contentHash, opts) : false
327 }
328
329 // Tell holders we stopped pulling this hash when there's no in-flight fetch
330 // to tear down (e.g. discarding an already-paused transfer), so their indicator clears.
331 notifyTransferStopped (contentHash) {
332 return this._protocol ? this._protocol.sendStopControl(contentHash) : false
333 }
334
335 _destPathFor (contentHash) {
336 return path.join(this._destDir, 'blob-' + contentHash)
337 }
338
339 _fetchFromPeer (peer, contentHash, opts = {}) {
340 const syntheticPath = 'content:' + contentHash
341 const destPath = path.join(
342 this._destDir,
343 'blob-' + contentHash + '-' + Date.now() + '-' + Math.random().toString(36).slice(2, 8)
344 )
345 // The protocol's chunk handlers write to whatever _filePaths maps the
346 // synthetic path to. Per-attempt destPath so concurrent peer tries don't
347 // clobber one another.
348 this._filePaths.set(syntheticPath, destPath)
349
350 return new Promise((resolve, reject) => {
351 let settled = false
352 const finish = (err, result) => {
353 if (settled) return
354 settled = true
355 clearTimeout(timer)
356 clearTimeout(pending.idleTimer)
357 this._pendingContent.delete(contentHash)
358 if (err) reject(err); else resolve(result)
359 }
360 const timer = setTimeout(() => {
361 finish(new Error('content request timed out: ' + contentHash.slice(0, 12)))
362 }, opts.timeout || this._perPeerTimeout)
363 const idleFire = () => finish(new Error('peer went silent mid-stream: ' + contentHash.slice(0, 12)))
364 const pending = {
365 syntheticPath, destPath, finish, peer, contentHash,
366 bytesReceived: 0,
367 idleTimer: setTimeout(idleFire, this._idleTimeout),
368 idleFire
369 }
370 this._pendingContent.set(contentHash, pending)
371 this._protocol.requestContent(peer, contentHash, null)
372 })
373 }
374
375 _onChunkProgress (info) {
376 if (!info || !info.path || !info.path.startsWith('content:')) return
377 const contentHash = info.path.slice('content:'.length)
378 const pending = this._pendingContent.get(contentHash)
379 if (!pending) return
380 pending.bytesReceived += (info.bytes || 0)
381 clearTimeout(pending.idleTimer)
382 pending.idleTimer = setTimeout(pending.idleFire, this._idleTimeout)
383 }
384
385 // The V2 chunk flow does NOT verify the reassembled file against the
386 // advertised hash (that's the caller's job in this protocol). We do it here
387 // so fetchFile never returns un-verified bytes.
388 _onSynced (info) {
389 if (!info || info.direction !== 'pull') return
390 if (!info.path || !info.path.startsWith('content:')) return
391 const contentHash = info.path.slice('content:'.length)
392 const pending = this._pendingContent.get(contentHash)
393 if (!pending) return
394 try {
395 const data = fs.readFileSync(pending.destPath)
396 const actual = crypto.data(data).toString('hex')
397 if (actual !== contentHash) {
398 pending.finish(Object.assign(
399 new Error('content-hash mismatch: expected ' + contentHash.slice(0, 12) + '…, got ' + actual.slice(0, 12) + '…'),
400 { code: 'EHASHMISMATCH', expected: contentHash, actual }
401 ))
402 return
403 }
404 pending.finish(null, data)
405 } catch (err) {
406 pending.finish(err)
407 }
408 }
409}
410
411// Stream a file through the chunker's hasher to get its oid without buffering
412// the whole file in memory (matches /api/upload's createStreamingHasher oid).
413async function hashFileOnDisk (diskPath, size) {
414 const hasher = createStreamingHasher({ size })
415 for await (const buf of fs.createReadStream(diskPath)) hasher.update(buf)
416 return hasher.digest()
417}
lib/v3/protocol.js +951 -0
……@@ -0,0 +1,951 @@
1/**
2 * OverlayProtocol — Protomux channel for sync + transfer
3 *
4 * Sync approach: each side sends its local feed entries directly via
5 * file-offer messages. No Hypercore feed replication needed — the feed
6 * entries are small JSON objects sent over our own Protomux channel.
7 *
8 * Flow:
9 * 1. Channel opens → both send sync-state (feed length)
10 * 2. Side with more entries sends file-offers for each change
11 * 3. Receiver auto-requests files it doesn't have
12 * 4. Sender chunks on demand → streams chunks → receiver writes to disk
13 */
14
15import path from '#path'
16import * as messages from './messages.js'
17import { hashChunk, selectTier } from './chunker.js'
18import { ChunkScheduler } from './chunk-scheduler.js'
19import fs from '#fs'
20
21const PROTOCOL = 'hyper-overlay/v2'
22const VERSION = 2
23const CAP_LOCAL_FILES = 0x01
24const CAP_ADAPTIVE_CHUNKS = 0x02
25
26// A file's chunk list, shipped in one frame, can exceed the
27// 16 MiB-1 Noise transport limit (@hyperswarm/secret-stream MAX_ATOMIC_WRITE):
28// a 1.25 TB file at tier 3 is ~1.2M entries ≈ ~44 MB, which the secret-stream
29// rejects ("Message is too large for an atomic write"), dropping the connection
30// before a byte ships. So chunkHashes is paged into frames of ≤ this many
31// entries. Each wire entry is 32 B (hash) + ≤5 B (varint length) ≈ 37 B, so
32// 100k entries ≈ 3.7 MB/frame — well under the limit even after Protomux's
33// ≤8 MiB send-batching.
34const MAX_CHUNKS_PER_MSG = 100000
35
36// Tree entries are variable length (name is a var-string), unlike the fixed-width
37// chunkHashes entries, so treeResponse pages by an encoded-byte budget rather than
38// a fixed count. 4 MiB keeps each frame well under the atomic-write limit even
39// after Protomux's ≤8 MiB send-batching.
40const MAX_TREE_BYTES_PER_MSG = 4 * 1024 * 1024
41
42// Defensive ceiling on the bytes a requester will accumulate across a paged
43// treeResponse. A single-directory tree is orders of magnitude smaller; this only
44// bounds memory so a peer streaming pages endlessly rejects the request instead
45// of OOMing.
46const MAX_TREE_RESPONSE_BYTES = 256 * 1024 * 1024
47
48// transfer-control (message 12) states, downloader→holder.
49const CONTROL_PAUSED = 0
50const CONTROL_STOPPED = 1
51
52// Abandon a serve loop whose stream neither drains nor closes within
53// this window (a wedged-but-TCP-alive peer); the receiver re-requests, so it self-heals.
54const DRAIN_TIMEOUT_MS = 60000
55
56export class OverlayProtocol {
57 constructor (syncEngine, transferManager, opts = {}) {
58 this._syncEngine = syncEngine
59 this._transferManager = transferManager
60 this._corestore = opts.corestore || null
61 this._overlayKey = opts.key || null
62 this._peers = new Map()
63 this._filePaths = opts.filePaths || new Map()
64 this._syncBaseDirs = opts.syncBaseDirs || []
65 this._defaultStrategy = opts.defaultStrategy || 'prompt'
66 this._autoSync = opts.autoSync !== false
67
68 this._conflictCb = opts.onConflict || null
69 this._syncedCb = opts.onSynced || null
70 this._fileOfferCb = opts.onFileOffer || null
71 // Called for every chunk that arrives on this peer. Lets callers
72 // (e.g. NetworkFetcher) implement silent-peer watchdogs by resetting
73 // an idle timer on every chunk. Receives { path, index, bytes, peer }.
74 this._chunkProgressCb = opts.onChunkProgress || null
75 // serve-side telemetry, symmetric to onChunkProgress — fired when WE
76 // serve a peer (the sender-side download indicator). onServeStart({ path, peer,
77 // from, total }) once a peer is authorized to fetch; onChunkServe({ path, peer,
78 // from, bytes }) per served chunk; onServeEnd({ path, peer, from }) on close.
79 this._serveStartCb = opts.onServeStart || null
80 this._chunkServeCb = opts.onChunkServe || null
81 this._serveEndCb = opts.onServeEnd || null
82 // A downloader paused/stopped a hash WE serve. { path, peer, from, state }.
83 this._serveControlCb = opts.onServeControl || null
84 // A downloader reported its on-disk have-bytes for a hash WE serve (resume
85 // baseline). { path, peer, from, have }.
86 this._serveProgressCb = opts.onServeProgress || null
87
88 // 0.5a: content-hash → local disk path, for responding to content-requests
89 this._contentHashPaths = opts.contentHashPaths || new Map()
90 // 0.5a: tree-request timeout (ms)
91 this._treeRequestTimeout = opts.treeRequestTimeout || 30000
92 // Cap on bytes accumulated across a paged treeResponse.
93 this._maxTreeResponseBytes = opts.maxTreeResponseBytes || MAX_TREE_RESPONSE_BYTES
94 // Multi-source: synthetic path 'content:<hash>' → ChunkScheduler. When set,
95 // _onChunkHashes/_onChunkData delegate to the scheduler (parallel fetch
96 // across peers) instead of the legacy single-peer auto-flow.
97 this._schedulers = new Map()
98 // 'content:<hash>' cancelled before its scheduler existed (the caller's
99 // peer-wait window) — fetchContent honors it the moment it creates the scheduler.
100 this._cancelPending = new Set()
101
102 // serve authorization. _serveAuthorizer(peer, from, hash) →
103 // boolean gates every inbound content-request; deny is a silent drop
104 // (indistinguishable from "I don't hold it" — no membership oracle).
105 // _localProfileKey is stamped on outbound content-requests as msg.from.
106 this._serveAuthorizer = opts.serveAuthorizer || null
107 this._localProfileKey = opts.localProfileKey || null
108 }
109
110 /**
111 * Multi-source fetch of a file by content hash from several peers in
112 * parallel (torrent-style). The caller pre-registers _filePaths['content:'
113 * + contentHash] = destPath. Resolves when the file is assembled + verified.
114 *
115 * @param {string} contentHash
116 * @param {Array} peers peer objects to fan the request to
117 * @param {object} [opts] { destPath, cap, timeout, onProgress }
118 * @returns {Promise<void>}
119 */
120 fetchContent (contentHash, peers, opts = {}) {
121 const p = 'content:' + contentHash
122 // Content-addressed dedup: a concurrent fetch of the SAME hash (a
123 // different file/transfer with identical bytes) joins the in-flight one instead
124 // of failing — await its verified bytes, then copy to our destPath. If the leader
125 // was cancelled, re-issue our own fetch (the scheduler entry is already gone).
126 const inflight = this._schedulers.get(p)
127 if (inflight) {
128 return inflight.shared.then(
129 (res) => {
130 if (opts.destPath && opts.destPath !== inflight.destPath) {
131 try { fs.copyFileSync(inflight.destPath, opts.destPath) } catch (err) { throw err }
132 }
133 return res
134 },
135 (err) => { if (err?.code === 'ECANCELLED') return this.fetchContent(contentHash, peers, opts); throw err },
136 )
137 }
138 const sched = new ChunkScheduler({
139 path: p,
140 destPath: opts.destPath,
141 transfer: this._transferManager,
142 sendNeed: (peer, indices) => peer.msgs.chunkNeed.send({ path: p, indices }),
143 cap: opts.cap,
144 timeout: opts.timeout,
145 onProgress: opts.onProgress,
146 onVerify: opts.onVerify,
147 onEnd: opts.onEnd,
148 onBaseline: (have) => this.sendTransferProgress(contentHash, have),
149 contentHash // verify the whole-file hash incrementally during the transfer
150 })
151 this._schedulers.set(p, sched)
152 // shared promise so joiners (above) can await the same completion —
153 // calling sched.promise() twice would overwrite its resolve/reject handlers.
154 sched.shared = sched.promise().finally(() => this._schedulers.delete(p))
155 // honor a cancel/pause that arrived before this scheduler existed.
156 if (this._cancelPending.delete(p)) { sched.cancel(); return sched.shared }
157 for (const peer of peers) this.requestContent(peer, contentHash, null)
158 return sched.shared
159 }
160
161 // Stop an in-flight multi-source fetch. discardPartial:true unlinks the
162 // partial (cancel); false keeps it for resume (pause). The scheduler's promise
163 // rejects with ECANCELLED and its .finally() removes it from _schedulers. When no
164 // scheduler exists yet (the caller is still in its peer-wait window), remember the
165 // intent so fetchContent cancels the scheduler the instant it is created.
166 async cancelContent (contentHash, { discardPartial = false, signal = true } = {}) {
167 const p = 'content:' + contentHash
168 const sched = this._schedulers.get(p)
169 // No scheduler → the fetch never sent a content-request, so no holder has a serve
170 // row for us; skip the (wasted) broadcast and just record the pre-scheduler cancel.
171 if (!sched) { this._cancelPending.add(p); return false }
172 // Tell holders we paused (keep partial) or stopped (discard) BEFORE local teardown,
173 // so their sender-side indicator reacts now, not after the idle sweep. signal:false
174 // suppresses it (a supersede is a restart, not a user stop).
175 if (signal) this._sendTransferControl(contentHash, discardPartial ? CONTROL_STOPPED : CONTROL_PAUSED)
176 // Stop accepting chunks first so the pause hash-drain converges on a stable frontier.
177 sched.cancel()
178 if (discardPartial) this._transferManager.cancel(sched.destPath)
179 else await this._transferManager.pause(sched.destPath)
180 return true
181 }
182
183 // Tell holders we stopped pulling a hash whose local fetch is already gone (e.g.
184 // discarding an already-paused transfer) — no scheduler to tear down here.
185 sendStopControl (contentHash) { this._sendTransferControl(contentHash, CONTROL_STOPPED) }
186
187 // Best-effort broadcast to all connected holders. A holder with no serve-ledger entry
188 // for us (or no slot for this message) just drops it; wrapped per-peer so a closing
189 // channel or an old peer can't throw.
190 _broadcast (msgName, payload) {
191 for (const [, peer] of this._peers) {
192 try { peer.msgs[msgName]?.send(payload) } catch {}
193 }
194 }
195
196 _sendTransferControl (contentHash, state) { this._broadcast('transferControl', { contentHash, state }) }
197
198 // Tell holders our current on-disk have-bytes for a hash we're (re)fetching, so their
199 // sender-side bar mirrors our TRUE progress rather than only the bytes they re-serve.
200 sendTransferProgress (contentHash, have) { this._broadcast('transferProgress', { contentHash, have }) }
201
202 // Drop a cancel recorded before a scheduler existed when the fetch is
203 // abandoned without ever reaching fetchContent (no peer connected) — otherwise the
204 // marker would cancel the next fetch of the same content.
205 clearCancelPending (contentHash) { this._cancelPending.delete('content:' + contentHash) }
206
207 setContentHashPaths (map) { this._contentHashPaths = map }
208
209 attach (mux) {
210 if (this._peers.has(mux)) return this._peers.get(mux)
211
212 const self = this
213
214 const channel = mux.createChannel({
215 protocol: PROTOCOL,
216 id: this._overlayKey || null,
217 onopen () {
218 const peer = self._peers.get(mux)
219 if (peer) self._onOpen(peer)
220 },
221 onclose () {
222 const peer = self._peers.get(mux)
223 self._peers.delete(mux)
224 // Failover: let any active multi-source fetch reassign this peer's
225 // inflight chunks to the remaining peers.
226 if (peer) for (const sched of self._schedulers.values()) sched.removePeer(peer)
227 // Fail in-flight tree requests now, so a mid-stream disconnect rejects
228 // promptly instead of stranding the request (and its buffered pages)
229 // until the idle timeout fires.
230 if (peer) self._failPendingTrees(peer, new Error('channel closed'))
231 // tell the sender-side indicator this peer is gone for every file
232 // it was pulling, so a disconnect mid-download clears its row promptly.
233 if (peer && self._serveEndCb) {
234 for (const [synthPath, from] of peer.authorizedServe) {
235 try { self._serveEndCb({ path: synthPath, peer, from }) } catch {}
236 }
237 }
238 }
239 })
240
241 if (!channel) return null
242
243 const peer = {
244 mux, channel, remoteFeedKey: null, remoteSeq: 0, msgs: {},
245 pendingTrees: new Map(), // hash → { resolve, reject, timer, timeout, nonce, pages, pageBytes }
246 // synthetic serve-paths this peer was authorized for via a
247 // GATED _onContentRequest, mapped to the requester profileKey (msg.from).
248 // _onChunkNeed serves bytes only for paths in here, so a peer cannot pull
249 // bytes by sending chunkNeed/fileRequest directly for a registered path it
250 // was never authorized to fetch; the value flows the requester identity to
251 // the serve telemetry.
252 authorizedServe: new Map()
253 }
254
255 peer.msgs.syncState = channel.addMessage({ encoding: messages.syncState, onmessage: (msg) => self._onSyncState(peer, msg) })
256 peer.msgs.fileOffer = channel.addMessage({ encoding: messages.fileOffer, onmessage: (msg) => self._onFileOffer(peer, msg) })
257 peer.msgs.fileRequest = channel.addMessage({ encoding: messages.fileRequest, onmessage: (msg) => self._onFileRequest(peer, msg) })
258 peer.msgs.chunkHashes = channel.addMessage({ encoding: messages.chunkHashes, onmessage: (msg) => self._onChunkHashes(peer, msg) })
259 peer.msgs.chunkNeed = channel.addMessage({ encoding: messages.chunkNeed, onmessage: (msg) => self._onChunkNeed(peer, msg) })
260 peer.msgs.chunkData = channel.addMessage({ encoding: messages.chunkData, onmessage: (msg) => self._onChunkData(peer, msg) })
261 peer.msgs.chunkCancel = channel.addMessage({ encoding: messages.chunkCancel, onmessage: (msg) => self._onChunkCancel(peer, msg) })
262 peer.msgs.transferComplete = channel.addMessage({ encoding: messages.transferComplete, onmessage: (msg) => self._onTransferComplete(peer, msg) })
263 peer.msgs.conflict = channel.addMessage({ encoding: messages.conflict, onmessage: (msg) => self._onConflictMsg(peer, msg) })
264
265 // 0.5a — tree messages (backward-compatible additions)
266 peer.msgs.treeRequest = channel.addMessage({ encoding: messages.treeRequest, onmessage: (msg) => self._onTreeRequest(peer, msg) })
267 peer.msgs.treeResponse = channel.addMessage({ encoding: messages.treeResponse, onmessage: (msg) => self._onTreeResponse(peer, msg) })
268 peer.msgs.contentRequest = channel.addMessage({ encoding: messages.contentRequest, onmessage: (msg) => self._onContentRequest(peer, msg) })
269 // Appended-last slots: ids 0-11 stay fixed for peers without 12/13, which just
270 // never register these channels and ignore the frames.
271 peer.msgs.transferControl = channel.addMessage({ encoding: messages.transferControl, onmessage: (msg) => self._onTransferControl(peer, msg) })
272 peer.msgs.transferProgress = channel.addMessage({ encoding: messages.transferProgress, onmessage: (msg) => self._onTransferProgress(peer, msg) })
273
274 this._peers.set(mux, peer)
275 channel.open({ version: VERSION, capabilities: CAP_LOCAL_FILES | CAP_ADAPTIVE_CHUNKS })
276 return peer
277 }
278
279 detach (mux) {
280 const peer = this._peers.get(mux)
281 if (!peer) return
282 peer.channel.close()
283 this._peers.delete(mux)
284 }
285
286 setFilePaths (map) { this._filePaths = map }
287
288 broadcastOffer (overlayPath, meta, op) {
289 const msg = {
290 path: overlayPath,
291 contentHash: meta.contentHash || '0'.repeat(64),
292 size: meta.size || 0,
293 mtime: meta.mtime || 0,
294 op
295 }
296 for (const [, peer] of this._peers) {
297 peer.msgs.fileOffer.send(msg)
298 }
299 }
300
301 requestFile (peer, overlayPath, contentHash, chunksHave) {
302 peer.msgs.fileRequest.send({
303 path: overlayPath,
304 contentHash,
305 chunksHave: chunksHave || null
306 })
307 }
308
309 /**
310 * 0.5a: Request a tree by its hash. Returns a promise resolving to
311 * the entries array (or [] if the peer doesn't have it).
312 *
313 * @param {object} peer
314 * @param {string} hash - hex tree hash
315 * @param {object} [opts]
316 * @param {number} [opts.timeout] - override default timeout (ms)
317 * @returns {Promise<Array<{kind, exec, name, childHash, size}>>}
318 */
319 requestTree (peer, hash, opts = {}) {
320 return new Promise((resolve, reject) => {
321 // If a prior request is already in-flight for the same hash, reject
322 // this caller rather than silently overwrite.
323 if (peer.pendingTrees.has(hash)) {
324 return reject(new Error('tree request already pending for hash: ' + hash))
325 }
326 // Per-request nonce (per-peer monotonic, non-zero) lets the requester drop
327 // pages echoed for a superseded attempt. pages/pageBytes buffer this
328 // request's own paged response, so a fresh attempt never inherits an old one.
329 const nonce = peer._treeReqSeq = (peer._treeReqSeq || 0) + 1
330 const timeout = opts.timeout || this._treeRequestTimeout
331 const pending = { resolve, reject, timer: null, timeout, nonce, pages: null, pageBytes: 0 }
332 this._armTreeTimeout(peer, hash, pending)
333 peer.pendingTrees.set(hash, pending)
334 peer.msgs.treeRequest.send({ hash, nonce })
335 })
336 }
337
338 // Arm (or re-arm) a pending tree request's idle timeout. Every buffered page
339 // re-arms it (a large tree on a slow link mustn't time out mid-stream); the
340 // request still settles because the accumulator is byte-capped in _onTreeResponse.
341 _armTreeTimeout (peer, hash, pending) {
342 pending.timer = setTimeout(() => {
343 if (peer.pendingTrees.get(hash) === pending) {
344 peer.pendingTrees.delete(hash)
345 pending.reject(new Error('tree request timed out: ' + hash))
346 }
347 }, pending.timeout)
348 }
349
350 // Reject every in-flight tree request on this peer (channel close / destroy),
351 // clearing timers so the promise fails promptly and its buffered pages release.
352 _failPendingTrees (peer, err) {
353 for (const [hash, pending] of peer.pendingTrees) {
354 clearTimeout(pending.timer)
355 peer.pendingTrees.delete(hash)
356 pending.reject(err)
357 }
358 }
359
360 /**
361 * 0.5a: Request a file by its content hash. Peer looks up any file
362 * matching that hash locally and streams chunks via the existing flow
363 * (chunkHashes → chunkNeed → chunkData). Returns immediately; chunks
364 * arrive asynchronously.
365 *
366 * Note: receiver must pre-register a target path mapping via
367 * _filePaths keyed by the contentHash before calling, OR this method
368 * is a no-op on the receive side. Full wiring deferred to 0.5c.
369 */
370 requestContent (peer, contentHash, chunksHave) {
371 peer.msgs.contentRequest.send({
372 contentHash,
373 chunksHave: chunksHave || null,
374 from: this._localProfileKey || '' // authenticate the asker to the serve gate
375 })
376 }
377
378 get peerCount () { return this._peers.size }
379
380 destroy () {
381 for (const [, peer] of this._peers) {
382 this._failPendingTrees(peer, new Error('protocol destroyed'))
383 peer.channel.close()
384 }
385 this._peers.clear()
386 }
387
388 // ── Handlers ────────────────────────────────────────────────
389
390 _onOpen (peer) {
391 peer.msgs.syncState.send({
392 feedKey: this._syncEngine.feedKey,
393 localSeq: this._syncEngine.feed.length,
394 remoteSeq: 0
395 })
396 }
397
398 async _onSyncState (peer, msg) {
399 peer.remoteFeedKey = msg.feedKey
400 peer.remoteSeq = msg.localSeq
401
402 if (!this._autoSync) return
403
404 try {
405 await this._offerLocalFiles(peer)
406 } catch {
407 // ignore — the HyperOverlay facade runs with autoSync off, so this path is dormant
408 }
409 }
410
411 async _onFileOffer (peer, msg) {
412 // In gated mode the overlay flow is contentRequest→scheduler ONLY; gated
413 // consumers never act on path-based file-offers (autoSync is off). Ignoring
414 // them closes the offer→requestFile→legacy-receive path a peer could drive.
415 if (this._serveAuthorizer) return
416 if (this._fileOfferCb) this._fileOfferCb(peer, msg)
417
418 if (msg.op === 0) {
419 const local = await this._syncEngine._fileIndex.getFile(msg.path)
420 if (!local || local.contentHash !== msg.contentHash) {
421 const diskPath = this._resolveFilePath(msg.path)
422 if (diskPath) {
423 this._filePaths.set(msg.path, diskPath)
424 this.requestFile(peer, msg.path, msg.contentHash)
425 }
426 }
427 }
428 }
429
430 async _onFileRequest (peer, msg) {
431 // path-based serve is UNGATED and would bypass the serve gate (the only
432 // gated entry point is _onContentRequest). Gated consumers never send
433 // fileRequest, so refuse it outright in gated mode.
434 if (this._serveAuthorizer) return
435 const diskPath = this._filePaths.get(msg.path)
436 if (!diskPath) return
437
438 const prepared = await this._transferManager.prepareFile(diskPath, msg.path)
439 if (!prepared) return
440
441 if (!await this._transferManager._fileIndex.hasChunkMap(msg.path)) {
442 await this._transferManager._fileIndex.putChunkMap(msg.path, prepared.chunks)
443 }
444
445 this._sendChunkHashes(peer, msg.path, prepared.tier, prepared.chunks)
446 }
447
448 // Send a file's chunk list, paged so no single frame exceeds
449 // the Noise transport's 16 MiB-1 atomic-write limit. A list that fits in one
450 // frame ships as one (more:0) — identical on the wire to the pre-paging format
451 // bar a trailing 0 byte. _onChunkHashes reassembles the pages on the receiver.
452 _sendChunkHashes (peer, filePath, tier, chunks) {
453 const total = chunks.length
454 if (total <= MAX_CHUNKS_PER_MSG) {
455 peer.msgs.chunkHashes.send({ path: filePath, tier, chunks, more: 0 })
456 return
457 }
458 for (let i = 0; i < total; i += MAX_CHUNKS_PER_MSG) {
459 const more = i + MAX_CHUNKS_PER_MSG < total ? 1 : 0
460 peer.msgs.chunkHashes.send({ path: filePath, tier, chunks: chunks.slice(i, i + MAX_CHUNKS_PER_MSG), more })
461 }
462 }
463
464 // Reassemble a paged chunk list. Returns the full list once
465 // the final page (more:0) arrives, or null while pages are still pending. A
466 // lone complete frame passes straight through (no copy) — the small-file norm.
467 // Pages for a path arrive in order on this channel, so arrival-order
468 // concatenation is correct even when several files interleave on it.
469 _reassembleChunkHashes (peer, msg) {
470 const buffered = peer._chunkHashPages && peer._chunkHashPages.get(msg.path)
471 if (!buffered && !msg.more) return msg.chunks
472 if (!peer._chunkHashPages) peer._chunkHashPages = new Map()
473 let acc = buffered
474 if (!acc) { acc = []; peer._chunkHashPages.set(msg.path, acc) }
475 for (const ch of msg.chunks) acc.push(ch)
476 if (msg.more) return null
477 peer._chunkHashPages.delete(msg.path)
478 return acc
479 }
480
481 // Upper-bound wire size of one tree entry: kind(1)+exec(1) + string(len-prefix ≤4
482 // + utf8 name bytes) + childHash(32) + size varint(≤9, c.uint is 9 bytes for a
483 // >4 GiB size).
484 _treeEntryWireSize (e) {
485 return 1 + 1 + 4 + Buffer.byteLength(e.name || '', 'utf8') + 32 + 9
486 }
487
488 // Send a tree's entries, paged so no single frame exceeds the Noise atomic-write
489 // limit; every page but the last is more:1. A tree that fits in one frame ships
490 // as a single more:0 frame. `nonce` echoes the requester's per-request id.
491 _sendTreeResponse (peer, hash, entries, nonce) {
492 let page = []
493 let bytes = 40 // hash(32) + uint32 count prefix + more/nonce + slack
494 for (const e of entries) {
495 const sz = this._treeEntryWireSize(e)
496 if (page.length && bytes + sz > MAX_TREE_BYTES_PER_MSG) {
497 peer.msgs.treeResponse.send({ hash, entries: page, more: 1, nonce })
498 page = []
499 bytes = 40
500 }
501 page.push(e)
502 bytes += sz
503 }
504 peer.msgs.treeResponse.send({ hash, entries: page, more: 0, nonce })
505 }
506
507 async _onChunkHashes (peer, msg) {
508 // Reassemble paged chunkHashes frames before dispatching.
509 const chunks = this._reassembleChunkHashes(peer, msg)
510 if (chunks === null) {
511 // A buffered page is forward progress — keep the scheduler's no-progress
512 // watchdog from tripping while a large map streams across several frames.
513 const pending = this._schedulers.get(msg.path)
514 if (pending && pending.notePageProgress) pending.notePageProgress()
515 return
516 }
517 // Multi-source: if a scheduler owns this path, let it distribute chunk
518 // requests across peers instead of the legacy single-peer auto-flow.
519 const sched = this._schedulers.get(msg.path)
520 if (sched) { Promise.resolve(sched.onChunkHashes(peer, chunks)).catch(() => {}); return }
521 // the legacy single-peer receive below writes incoming bytes
522 // to _filePaths.get(path) and finalizes a rename OVER it. For a path that
523 // maps to one of OUR own files (e.g. a content-register entry), an
524 // unsolicited chunkHashes would let a peer overwrite the owner's source
525 // file. Gated consumers only ever receive via a scheduler (the branch
526 // above), so refuse any other receive.
527 if (this._serveAuthorizer) return
528
529 const diskPath = this._filePaths.get(msg.path)
530 if (!diskPath) return
531
532 // Empty-file fast path: a 0-byte file has no chunks to fetch, but we
533 // still need to materialize it on disk and fire the same completion
534 // flow as _onChunkData's "transfer complete" branch — otherwise the
535 // requester (e.g. NetworkFetcher.getContent) never resolves and times
536 // out. Empty files are common in practice: .keep, __init__.py, build
537 // stamps, touch'd markers. Bug symptom before this: node-gyp's
538 // postinstall.stamp (0 bytes, hash 5187b7a8…) hung every install.
539 if (chunks.length === 0) {
540 try {
541 fs.mkdirSync(path.dirname(diskPath), { recursive: true })
542 fs.writeFileSync(diskPath, Buffer.alloc(0))
543 } catch {
544 return
545 }
546 const contentHash = hashChunk(Buffer.alloc(0))
547 const stat = fs.statSync(diskPath)
548 await this._syncEngine._fileIndex.putFile(msg.path, {
549 contentHash, size: 0, mtime: stat.mtimeMs
550 })
551 await this._syncEngine.logPut(msg.path, {
552 contentHash, size: 0, mtime: stat.mtimeMs
553 })
554 if (peer.remoteFeedKey) {
555 await this._syncEngine.markSynced(peer.remoteFeedKey, msg.path, contentHash, this._syncEngine.feed.length - 1)
556 }
557 peer.msgs.transferComplete.send({ path: msg.path, contentHash })
558 if (this._syncedCb) {
559 this._syncedCb({ path: msg.path, contentHash, size: 0, direction: 'pull' })
560 }
561 return
562 }
563
564 const needed = this._transferManager.computeNeeded(chunks, new Set())
565 if (needed.length === 0) return
566
567 let size = 0
568 for (const c of chunks) size += c.length
569
570 // startReceive can throw (fd open, partial recovery) — drop this transfer
571 // instead of rejecting protomux's onmessage promise, which would destroy the
572 // whole Noise stream.
573 try {
574 await this._transferManager.startReceive(diskPath, {
575 size,
576 chunks: chunks.map((c, i) => {
577 let offset = 0
578 for (let j = 0; j < i; j++) offset += chunks[j].length
579 return { hash: c.hash, offset, length: c.length }
580 })
581 })
582 } catch {
583 return
584 }
585
586 peer.msgs.chunkNeed.send({ path: msg.path, indices: needed })
587 }
588
589 async _onChunkNeed (peer, msg) {
590 // Serve bytes ONLY for a synthetic path THIS peer was
591 // authorized for via a gated _onContentRequest. Blocks a peer pulling bytes
592 // by sending chunkNeed directly for a registered path (e.g. a content-
593 // register entry) without passing the serve gate.
594 if (this._serveAuthorizer && !peer.authorizedServe.has(msg.path)) return
595 const diskPath = this._filePaths.get(msg.path)
596 if (!diskPath) return
597
598 const fileIndex = this._transferManager._fileIndex
599 const chunkMap = msg.path.startsWith('content:')
600 ? await fileIndex.getChunkMapByHash(msg.path.slice('content:'.length))
601 : await fileIndex.getChunkMap(msg.path)
602 if (!chunkMap) return
603
604 for (let i = 0; i < msg.indices.length; i++) {
605 const index = msg.indices[i]
606 if (index >= chunkMap.length) continue
607 const c = chunkMap[index]
608 const data = this._transferManager.readChunk(diskPath, c.offset, c.length)
609 if (!data) continue
610 const flushed = peer.msgs.chunkData.send({ path: msg.path, index, data })
611 if (this._chunkServeCb) {
612 try { this._chunkServeCb({ path: msg.path, index, bytes: data.length, peer, from: peer.authorizedServe.get(msg.path) || null }) } catch {}
613 }
614 // Head-of-line blocking — chunkData and every other protocol the host
615 // multiplexes over the SAME Noise stream (Corestore replication, app
616 // handshakes) share one socket, and protomux's MAX_BACKLOG is Infinity.
617 // send() returns the stream's writable state; on backpressure, stop
618 // producing content and let the queue drain, else sibling protocols on
619 // the stream are starved for the rest of the download.
620 if (flushed === false && i < msg.indices.length - 1) {
621 const alive = await this._waitForDrain(peer)
622 if (!alive) return
623 if (this._serveAuthorizer && !peer.authorizedServe.has(msg.path)) return
624 }
625 }
626 }
627
628 // One shared drain waiter per peer so concurrent serve loops (the
629 // same peer pulling several files) don't each attach listeners and trip the stream's
630 // max-listeners warning. Resolves true on drain, false if the channel closes first.
631 _waitForDrain (peer) {
632 const channel = peer.channel
633 const stream = peer.mux && peer.mux.stream
634 if (!stream || !channel || channel.closed) return Promise.resolve(false)
635 if (channel.drained) return Promise.resolve(true)
636 if (peer._drainWait) return peer._drainWait
637 peer._drainWait = new Promise((resolve) => {
638 let timer = null
639 const done = (alive) => {
640 if (timer) clearTimeout(timer)
641 stream.removeListener('drain', onDrain)
642 stream.removeListener('close', onClose)
643 peer._drainWait = null
644 resolve(alive)
645 }
646 const onDrain = () => done(!channel.closed)
647 const onClose = () => done(false)
648 timer = setTimeout(() => done(false), DRAIN_TIMEOUT_MS)
649 stream.on('drain', onDrain)
650 stream.on('close', onClose)
651 })
652 return peer._drainWait
653 }
654
655 async _onChunkData (peer, msg) {
656 // Silent-peer watchdog hook: fire BEFORE any processing so a slow disk
657 // doesn't get blamed for a peer-side stall.
658 if (this._chunkProgressCb) {
659 try {
660 this._chunkProgressCb({
661 path: msg.path,
662 index: msg.index,
663 bytes: msg.data ? msg.data.length : 0,
664 peer
665 })
666 } catch {}
667 }
668
669 // Multi-source: scheduler owns the write + reschedule + finalize.
670 const sched = this._schedulers.get(msg.path)
671 if (sched) { Promise.resolve(sched.onChunkData(peer, msg.index, msg.data)).catch(() => {}); return }
672 // refuse the legacy single-peer receive (writes+renames over
673 // _filePaths.get(path), i.e. potentially our own source file). Gated
674 // consumers only receive via a scheduler (above).
675 if (this._serveAuthorizer) return
676
677 const diskPath = this._filePaths.get(msg.path)
678 if (!diskPath) return
679
680 const result = this._transferManager.writeChunk(diskPath, msg.index, msg.data)
681
682 if (result.ok && this._transferManager.isComplete(diskPath)) {
683 const fin = await this._transferManager.finalize(diskPath)
684 if (fin.ok) {
685 const fileData = fs.readFileSync(diskPath)
686 const contentHash = hashChunk(fileData)
687 const stat = fs.statSync(diskPath)
688
689 await this._syncEngine._fileIndex.putFile(msg.path, {
690 contentHash, size: stat.size, mtime: stat.mtimeMs
691 })
692 await this._syncEngine.logPut(msg.path, {
693 contentHash, size: stat.size, mtime: stat.mtimeMs
694 })
695
696 if (peer.remoteFeedKey) {
697 await this._syncEngine.markSynced(peer.remoteFeedKey, msg.path, contentHash, this._syncEngine.feed.length - 1)
698 }
699
700 peer.msgs.transferComplete.send({ path: msg.path, contentHash })
701 if (this._syncedCb) {
702 this._syncedCb({ path: msg.path, contentHash, size: stat.size, direction: 'pull' })
703 }
704 }
705 }
706 }
707
708 _onChunkCancel (peer, msg) {
709 const diskPath = this._filePaths.get(msg.path)
710 if (diskPath) this._transferManager.cancel(diskPath)
711 }
712
713 // Resolve a serve-message's downloader identity from the authenticated record we kept at
714 // serve start (peer.authorizedServe), NOT the message — so a peer can only affect its own
715 // ledger row, never spoof another. Returns null `from` when the peer was never authorized.
716 _authorizedFrom (peer, contentHash) {
717 const synthPath = 'content:' + contentHash
718 return { synthPath, from: peer.authorizedServe.get(synthPath) || null }
719 }
720
721 // A downloader told us it paused/stopped pulling a hash WE serve.
722 _onTransferControl (peer, msg) {
723 if (!this._serveControlCb) return
724 const { synthPath, from } = this._authorizedFrom(peer, msg.contentHash)
725 if (!from) return
726 const state = msg.state === CONTROL_STOPPED ? 'stopped' : 'paused'
727 try { this._serveControlCb({ path: synthPath, peer, from, state }) } catch {}
728 }
729
730 // A downloader reported its on-disk have-bytes for a hash WE serve.
731 _onTransferProgress (peer, msg) {
732 if (!this._serveProgressCb) return
733 const { synthPath, from } = this._authorizedFrom(peer, msg.contentHash)
734 if (!from) return
735 try { this._serveProgressCb({ path: synthPath, peer, from, have: msg.have || 0 }) } catch {}
736 }
737
738 async _onTransferComplete (peer, msg) {
739 if (peer.remoteFeedKey) {
740 await this._syncEngine.markSynced(peer.remoteFeedKey, msg.path, msg.contentHash, this._syncEngine.feed.length - 1)
741 }
742 if (this._syncedCb) {
743 this._syncedCb({ path: msg.path, contentHash: msg.contentHash, direction: 'push' })
744 }
745 }
746
747 _onConflictMsg (peer, msg) {
748 if (this._conflictCb) this._conflictCb(msg, peer)
749 }
750
751 // ── Tree handlers (0.5a) ────────────────────────────────────
752
753 async _onTreeRequest (peer, msg) {
754 // ungated metadata oracle (returns fileIndex tree entries to
755 // any peer). Gated consumers never use tree-requests; refuse in gated mode.
756 if (this._serveAuthorizer) return
757 let entries = []
758 try {
759 const fileIndex = this._syncEngine && this._syncEngine._fileIndex
760 if (fileIndex && typeof fileIndex.getTree === 'function') {
761 const tree = await fileIndex.getTree(msg.hash)
762 if (tree && Array.isArray(tree.entries)) entries = tree.entries
763 }
764 } catch {
765 // fall through — reply with empty entries on lookup failure
766 }
767 this._sendTreeResponse(peer, msg.hash, entries, msg.nonce || 0)
768 }
769
770 _onTreeResponse (peer, msg) {
771 const pending = peer.pendingTrees.get(msg.hash)
772 if (!pending) return // no request in flight — drop
773 // Drop a page echoing a superseded request's nonce (a stale page from a
774 // timed-out attempt must not settle a later same-hash retry). nonce 0 = a
775 // pre-nonce holder that can't echo it, so fall through (unverifiable,
776 // best-effort).
777 if (msg.nonce && msg.nonce !== pending.nonce) return
778 const settle = (fn, arg) => { clearTimeout(pending.timer); peer.pendingTrees.delete(msg.hash); fn(arg) }
779 // Lone complete frame (the small-tree norm) resolves with no accumulation.
780 if (!pending.pages && !msg.more) return settle(pending.resolve, msg.entries)
781 // Accumulate into THIS request's own buffer, bounding total bytes so a peer
782 // streaming pages endlessly rejects the request instead of growing memory
783 // without limit.
784 if (!pending.pages) pending.pages = []
785 for (const e of msg.entries) { pending.pages.push(e); pending.pageBytes += this._treeEntryWireSize(e) }
786 if (pending.pageBytes > this._maxTreeResponseBytes) {
787 return settle(pending.reject, new Error('tree response too large: ' + msg.hash))
788 }
789 if (msg.more) {
790 // Forward progress — re-arm the idle timeout so a legitimately large
791 // multi-page tree on a slow link doesn't time out mid-stream.
792 clearTimeout(pending.timer)
793 this._armTreeTimeout(peer, msg.hash, pending)
794 return
795 }
796 settle(pending.resolve, pending.pages)
797 }
798
799 async _onContentRequest (peer, msg) {
800 // serve gate. BEFORE any path resolution or streaming,
801 // ask the injected authorizer whether this peer may receive this hash.
802 // Deny == silent return (same observable behavior as "I don't hold it"
803 // below), so a non-member cannot distinguish denial from absence.
804 if (this._serveAuthorizer) {
805 let ok = false
806 try { ok = await this._serveAuthorizer(peer, msg.from || null, msg.contentHash) } catch { ok = false }
807 if (!ok) return
808 }
809
810 // Resolve msg.contentHash → an on-disk path. Three lookup paths, in order:
811 // 1. Fast path: _contentHashPaths map (warm — just-registered / consumer-set)
812 // 2. Content-addressed spool (RESTART-DURABLE): <syncBaseDir>/<contentHash>
813 // 3. Slow path: scan fileIndex for any file:* entry with that hash
814 let diskPath = this._contentHashPaths.get(msg.contentHash)
815 if (diskPath && !fileExists(diskPath)) diskPath = null
816
817 if (!diskPath) {
818 // Content-addressed spool fallback. The canonical bytes are stored at
819 // <syncBaseDir>/<contentHash> (fetchFile's mirrorSpool option writes them
820 // there). The in-memory map is empty after a restart and FileIndex
821 // persists no disk path, so resolve the hash straight from the
822 // content-addressed spool — a pure function of the hash + dir, like a
823 // git object store's blobs/<hash>. This is what lets a node keep serving
824 // its spooled files across restarts with no rehydration step.
825 for (const baseDir of this._syncBaseDirs) {
826 const candidate = path.join(baseDir, msg.contentHash)
827 if (fileExists(candidate)) {
828 diskPath = candidate
829 this._contentHashPaths.set(msg.contentHash, candidate) // warm the map
830 break
831 }
832 }
833 }
834
835 if (!diskPath) {
836 // Scan fileIndex for a file whose contentHash matches.
837 const fileIndex = this._syncEngine && this._syncEngine._fileIndex
838 if (fileIndex && typeof fileIndex.listFiles === 'function') {
839 try {
840 const files = await fileIndex.listFiles()
841 for (const f of files) {
842 if (f.contentHash === msg.contentHash) {
843 diskPath = this._filePaths.get(f.path)
844 if ((!diskPath || !fileExists(diskPath)) && f.path) diskPath = f.path
845 break
846 }
847 }
848 } catch {
849 // fall through
850 }
851 }
852 }
853
854 if (!diskPath) {
855 // Don't have it. Silently drop; requester will time out.
856 return
857 }
858
859 // Use a SYNTHETIC path keyed by contentHash so the requester can
860 // pre-register a destPath and the chunk flow routes correctly on
861 // both sides. Temporarily populate _filePaths so _onChunkNeed +
862 // chunkmap lookups resolve under this synthetic key.
863 const syntheticPath = 'content:' + msg.contentHash
864 this._filePaths.set(syntheticPath, diskPath)
865 peer.authorizedServe.set(syntheticPath, msg.from || null) // _onChunkNeed will now serve this path to THIS peer
866
867 // The chunk map is content-addressed — identical bytes always chunk
868 // identically. Reuse one computed at publish (or a prior serve, or before a
869 // restart) instead of re-reading the whole file to re-chunk it: re-chunking a
870 // multi-GB file adds a full-file-read of latency before the first byte ships.
871 const fileIndex = this._transferManager._fileIndex
872 let chunks = await fileIndex.getChunkMapByHash(msg.contentHash)
873 let tier
874 if (chunks) {
875 let size = 0
876 try { size = fs.statSync(diskPath).size } catch {}
877 tier = selectTier(size)
878 } else {
879 const prepared = await this._transferManager.prepareFile(diskPath, syntheticPath, { byHashOnly: true })
880 if (!prepared) {
881 this._filePaths.delete(syntheticPath)
882 peer.authorizedServe.delete(syntheticPath) // keep the authorize map in lockstep with _filePaths
883 return
884 }
885 chunks = prepared.chunks
886 tier = prepared.tier
887 // prepareFile persists the hash-keyed map only for large files; ensure it exists
888 // for small ones too, since _onChunkNeed resolves content:<hash> by that map.
889 if (!await fileIndex.hasChunkMapByHash(msg.contentHash)) await fileIndex.putChunkMapByHash(msg.contentHash, chunks)
890 }
891 this._sendChunkHashes(peer, syntheticPath, tier, chunks)
892 if (this._serveStartCb) {
893 let total = 0
894 for (const c of chunks) total += c.length || 0
895 try { this._serveStartCb({ path: syntheticPath, peer, from: msg.from || null, total }) } catch {}
896 }
897 }
898
899 // ── Sync logic ──────────────────────────────────────────────
900
901 /**
902 * Offer all local files to a peer.
903 * Reads from the local feed and sends file-offers for each entry.
904 * The peer will request any files it doesn't have.
905 */
906 async _offerLocalFiles (peer) {
907 const feed = this._syncEngine.feed
908 if (feed.length === 0) return
909
910 for (let i = 0; i < feed.length; i++) {
911 try {
912 const entry = await feed.get(i)
913 if (!entry) continue
914
915 // Register the file path mapping so _onFileRequest can find it
916 if (entry.op === 0) {
917 const diskPath = this._resolveFilePath(entry.path)
918 if (diskPath) this._filePaths.set(entry.path, diskPath)
919 }
920
921 peer.msgs.fileOffer.send({
922 path: entry.path,
923 contentHash: entry.hash || '0'.repeat(64),
924 size: entry.size || 0,
925 mtime: entry.mtime || 0,
926 op: entry.op
927 })
928 } catch {
929 // skip unreadable entries
930 }
931 }
932 }
933
934 /**
935 * Resolve an overlay path to a disk path.
936 */
937 _resolveFilePath (overlayPath) {
938 if (this._filePaths.has(overlayPath)) return this._filePaths.get(overlayPath)
939 for (const baseDir of this._syncBaseDirs) {
940 return path.join(baseDir, overlayPath)
941 }
942 return null
943 }
944}
945
946// True iff `p` is an existing regular file. Used to validate a resolved disk path
947// before serving (a cold-restart map entry or an overlay-path masquerading as a
948// disk path won't exist) and to probe the content-addressed spool.
949function fileExists (p) {
950 try { return fs.statSync(p).isFile() } catch { return false }
951}
lib/v3/sync-engine.js +364 -0
……@@ -0,0 +1,364 @@
1/**
2 * SyncEngine — Feed-based change tracking and conflict detection
3 *
4 * Each peer has a Hypercore feed logging file change operations.
5 * When peers connect, feeds replicate automatically (Hypercore handles this).
6 * The sync engine diffs feeds since the last-synced sequence number
7 * and detects conflicts using three-value comparison:
8 *
9 * my current hash vs their current hash vs last-synced hash (common ancestor)
10 *
11 * Conflict resolution is configurable per sync folder:
12 * prompt | newest-wins | keep-both | skip | mine-wins | theirs-wins
13 */
14
15import ReadyResource from 'ready-resource'
16
17// Operation types written to the change feed
18const OP_PUT = 0
19const OP_DEL = 1
20
21export const STRATEGIES = ['prompt', 'newest-wins', 'keep-both', 'skip', 'mine-wins', 'theirs-wins']
22
23export class SyncEngine extends ReadyResource {
24 constructor (fileIndex, corestore, opts = {}) {
25 super()
26 this._fileIndex = fileIndex
27 this._corestore = corestore
28 this._feed = null
29 this._peerFeeds = new Map() // peerKey hex → Hypercore
30 this._conflicts = [] // unresolved conflicts
31 this._opts = opts
32 }
33
34 async _open () {
35 // Local change feed — append-only log of file operations. Encrypted at rest when
36 // the caller passes opts.encryptionKey (peer feeds, opened by key in addPeerFeed,
37 // stay plaintext — their key is not ours to encrypt under).
38 const opts = { name: 'sync-feed', valueEncoding: 'json' }
39 if (this._opts.encryptionKey) opts.encryptionKey = this._opts.encryptionKey
40 this._feed = this._corestore.get(opts)
41 await this._feed.ready()
42 }
43
44 async _close () {
45 // Peer feeds are managed by corestore, no explicit close needed
46 }
47
48 /**
49 * The local change feed (Hypercore)
50 */
51 get feed () { return this._feed }
52
53 /**
54 * The local feed's public key as hex
55 */
56 get feedKey () { return this._feed.key.toString('hex') }
57
58 /**
59 * Current unresolved conflicts
60 */
61 get conflicts () { return this._conflicts }
62
63 // ── Change logging ──────────────────────────────────────────
64
65 /**
66 * Log a file put/update operation
67 * @param {string} path - file path
68 * @param {{ contentHash: string, size: number, mtime: number }} meta
69 */
70 async logPut (path, meta) {
71 await this._feed.append({
72 op: OP_PUT,
73 path,
74 hash: meta.contentHash,
75 size: meta.size,
76 mtime: meta.mtime,
77 timestamp: Date.now()
78 })
79 }
80
81 /**
82 * Log a file deletion
83 * @param {string} path - file path
84 */
85 async logDel (path) {
86 await this._feed.append({
87 op: OP_DEL,
88 path,
89 hash: null,
90 size: 0,
91 mtime: 0,
92 timestamp: Date.now()
93 })
94 }
95
96 // ── Peer feeds ──────────────────────────────────────────────
97
98 /**
99 * Add a remote peer's feed for syncing
100 * @param {Buffer|string} key - peer's feed public key
101 * @returns {Hypercore} the peer feed
102 */
103 addPeerFeed (key) {
104 const keyHex = typeof key === 'string' ? key : key.toString('hex')
105 if (this._peerFeeds.has(keyHex)) return this._peerFeeds.get(keyHex)
106
107 const feed = this._corestore.get({ key: typeof key === 'string' ? Buffer.from(key, 'hex') : key, valueEncoding: 'json' })
108 this._peerFeeds.set(keyHex, feed)
109 return feed
110 }
111
112 /**
113 * Get a peer feed by key
114 * @param {string} keyHex
115 * @returns {Hypercore|null}
116 */
117 getPeerFeed (keyHex) {
118 return this._peerFeeds.get(keyHex) || null
119 }
120
121 // ── Feed diffing ────────────────────────────────────────────
122
123 /**
124 * Get new operations from a feed since a given sequence number
125 * @param {Hypercore} feed
126 * @param {number} sinceSeq - start reading from this seq (exclusive)
127 * @returns {Array<{ seq: number, op: number, path: string, hash: string|null, size: number, mtime: number }>}
128 */
129 async getChanges (feed, sinceSeq) {
130 await feed.ready()
131 await feed.update()
132
133 const changes = []
134 const start = sinceSeq + 1
135 const end = feed.length
136
137 for (let i = start; i < end; i++) {
138 const entry = await feed.get(i)
139 changes.push({ seq: i, ...entry })
140 }
141
142 return changes
143 }
144
145 /**
146 * Build a map of latest state per path from a list of changes.
147 * Later entries override earlier ones for the same path.
148 * @param {Array} changes
149 * @returns {Map<string, { seq, op, hash, size, mtime }>}
150 */
151 buildChangeMap (changes) {
152 const map = new Map()
153 for (const c of changes) {
154 map.set(c.path, { seq: c.seq, op: c.op, hash: c.hash, size: c.size, mtime: c.mtime })
155 }
156 return map
157 }
158
159 // ── Sync diff ───────────────────────────────────────────────
160
161 /**
162 * Compute the sync diff between local and a remote peer.
163 * Returns actions to take: pull, push, conflict, or skip.
164 *
165 * @param {string} peerKey - hex key of the remote peer's feed
166 * @returns {Array<{ path, action, local?, remote?, ancestor? }>}
167 * action: 'pull' | 'push' | 'conflict' | 'skip' | 'delete-local' | 'delete-remote'
168 */
169 async diff (peerKey) {
170 const peerFeed = this._peerFeeds.get(peerKey)
171 if (!peerFeed) throw new Error('Unknown peer: ' + peerKey)
172
173 await peerFeed.ready()
174 await peerFeed.update()
175
176 // Get last-synced state for all files with this peer
177 const syncStates = await this._fileIndex.listSyncStates(peerKey)
178 const syncMap = new Map()
179 for (const s of syncStates) {
180 syncMap.set(s.path, s)
181 }
182
183 // Get my changes since last sync
184 const myLastSeq = this._getMaxSyncedSeq(syncStates, 'local')
185 const myChanges = await this.getChanges(this._feed, -1) // read all, filter by syncMap
186 const myMap = this.buildChangeMap(myChanges)
187
188 // Get their changes since last sync
189 const theirChanges = await this.getChanges(peerFeed, -1) // read all
190 const theirMap = this.buildChangeMap(theirChanges)
191
192 // Collect all paths mentioned by either side
193 const allPaths = new Set([...myMap.keys(), ...theirMap.keys()])
194
195 const actions = []
196
197 for (const path of allPaths) {
198 const mine = myMap.get(path)
199 const theirs = theirMap.get(path)
200 const ancestor = syncMap.get(path)
201
202 const myHash = mine ? mine.hash : null
203 const theirHash = theirs ? theirs.hash : null
204 const ancestorHash = ancestor ? ancestor.lastHash : null
205
206 // Both have it and hashes match — in sync
207 if (myHash && theirHash && myHash === theirHash) {
208 actions.push({ path, action: 'skip', local: mine, remote: theirs })
209 continue
210 }
211
212 // Only they have it (or they changed, I haven't)
213 if (theirHash && !mine) {
214 actions.push({ path, action: 'pull', remote: theirs })
215 continue
216 }
217
218 // Only I have it (or I changed, they haven't)
219 if (myHash && !theirs) {
220 actions.push({ path, action: 'push', local: mine })
221 continue
222 }
223
224 // Both null — both deleted
225 if (!myHash && !theirHash) {
226 actions.push({ path, action: 'skip' })
227 continue
228 }
229
230 // One deleted, one has content
231 if (mine && mine.op === OP_DEL && theirHash) {
232 // I deleted, they still have it
233 if (ancestorHash && ancestorHash === theirHash) {
234 // They haven't changed since ancestor — propagate my delete
235 actions.push({ path, action: 'delete-remote', local: mine, remote: theirs })
236 } else {
237 // They changed it after ancestor — conflict
238 actions.push({ path, action: 'conflict', local: mine, remote: theirs, ancestor })
239 }
240 continue
241 }
242
243 if (theirs && theirs.op === OP_DEL && myHash) {
244 // They deleted, I still have it
245 if (ancestorHash && ancestorHash === myHash) {
246 // I haven't changed since ancestor — accept their delete
247 actions.push({ path, action: 'delete-local', local: mine, remote: theirs })
248 } else {
249 // I changed it after ancestor — conflict
250 actions.push({ path, action: 'conflict', local: mine, remote: theirs, ancestor })
251 }
252 continue
253 }
254
255 // Both have different content — use ancestor to determine conflict
256 if (myHash && theirHash && myHash !== theirHash) {
257 if (ancestorHash === myHash) {
258 // Only they changed — pull
259 actions.push({ path, action: 'pull', local: mine, remote: theirs })
260 } else if (ancestorHash === theirHash) {
261 // Only I changed — push
262 actions.push({ path, action: 'push', local: mine, remote: theirs })
263 } else {
264 // Both changed — conflict
265 actions.push({ path, action: 'conflict', local: mine, remote: theirs, ancestor })
266 }
267 continue
268 }
269
270 // Fallback — shouldn't reach here
271 actions.push({ path, action: 'skip', local: mine, remote: theirs })
272 }
273
274 return actions
275 }
276
277 // ── Conflict resolution ─────────────────────────────────────
278
279 /**
280 * Resolve a conflict using a strategy
281 * @param {{ path, local, remote, ancestor }} conflict
282 * @param {string} strategy - one of STRATEGIES
283 * @returns {{ action: 'pull'|'push'|'keep-both'|'skip', winner?: 'local'|'remote' }}
284 */
285 resolveConflict (conflict, strategy) {
286 switch (strategy) {
287 case 'mine-wins':
288 return { action: 'push', winner: 'local' }
289
290 case 'theirs-wins':
291 return { action: 'pull', winner: 'remote' }
292
293 case 'newest-wins': {
294 const mySeq = conflict.local ? conflict.local.seq : -1
295 const theirSeq = conflict.remote ? conflict.remote.seq : -1
296 if (mySeq >= theirSeq) {
297 return { action: 'push', winner: 'local' }
298 }
299 return { action: 'pull', winner: 'remote' }
300 }
301
302 case 'keep-both':
303 return { action: 'keep-both' }
304
305 case 'skip':
306 return { action: 'skip' }
307
308 case 'prompt':
309 default:
310 // Add to conflict queue for user resolution
311 this._conflicts.push(conflict)
312 return { action: 'skip' }
313 }
314 }
315
316 /**
317 * Resolve a queued conflict by user choice
318 * @param {string} path
319 * @param {string} choice - 'overwrite' | 'keep-both' | 'skip'
320 * @returns {{ action: string } | null}
321 */
322 resolveQueued (path, choice) {
323 const idx = this._conflicts.findIndex(c => c.path === path)
324 if (idx === -1) return null
325
326 const conflict = this._conflicts.splice(idx, 1)[0]
327
328 switch (choice) {
329 case 'overwrite':
330 return { action: 'pull', winner: 'remote', conflict }
331 case 'keep-mine':
332 return { action: 'push', winner: 'local', conflict }
333 case 'keep-both':
334 return { action: 'keep-both', conflict }
335 case 'skip':
336 default:
337 return { action: 'skip', conflict }
338 }
339 }
340
341 /**
342 * Update sync state after a successful sync operation
343 * @param {string} peerKey
344 * @param {string} path
345 * @param {string} hash - the hash both sides now agree on
346 * @param {number} seq - the sequence number of the operation
347 */
348 async markSynced (peerKey, path, hash, seq) {
349 await this._fileIndex.putSyncState(peerKey, path, {
350 lastSeq: seq,
351 lastHash: hash
352 })
353 }
354
355 // ── Internal ────────────────────────────────────────────────
356
357 _getMaxSyncedSeq (syncStates, _side) {
358 let max = -1
359 for (const s of syncStates) {
360 if (s.lastSeq > max) max = s.lastSeq
361 }
362 return max
363 }
364}
lib/v3/transfer.js +808 -0
……@@ -0,0 +1,808 @@
1/**
2 * TransferManager — On-demand chunking and streaming file transfer
3 *
4 * Sender side:
5 * 1. Read file from disk
6 * 2. Chunk on demand (adaptive FastCDC)
7 * 3. Send chunk hashes → peer says which it needs → stream those chunks
8 * 4. Release chunks from memory after send
9 *
10 * Receiver side:
11 * 1. Receive chunk hashes, respond with which are missing
12 * 2. Receive chunks, write to <file>.overlay-partial
13 * 3. Verify each chunk hash
14 * 4. Atomic rename on completion
15 *
16 * No blob storage. Chunks exist in memory during transfer only.
17 */
18
19import fs from '#fs'
20import path from '#path'
21import os from '#os'
22import c from 'compact-encoding'
23import sodium from 'sodium-universal'
24import { chunk as chunkBuffer, chunkStream, hashChunk, createStreamingHasher, selectTier, getTierParams } from './chunker.js'
25
26export const PARTIAL_SUFFIX = '.overlay-partial'
27
28export const partialPathFor = (targetPath) => path.join(path.dirname(targetPath), path.basename(targetPath) + PARTIAL_SUFFIX)
29
30// Resume verification + whole-file hashing run async + yielding so a multi-GB pass
31// never blocks the event loop (IPC, pause/cancel, progress stay live).
32const VERIFY_YIELD_EVERY = 64
33const yieldToLoop = () => new Promise((resolve) => setTimeout(resolve, 0))
34
35// Receive journal: bitmap + whole-file hash snapshot, stored in app-private storage
36// (opts.journalDir) keyed by a digest of the destination path — never next to the
37// download — so resume is O(1) and the incremental verify continues with no re-read.
38const JOURNAL_SUFFIX = '.journal'
39const JOURNAL_MAGIC = 0x4d4a5633
40const JOURNAL_FLUSH_EVERY = 512
41const JOURNAL_FLUSH_INTERVAL_MS = 1500
42export const journalNameFor = (targetPath) => hashChunk(Buffer.from(targetPath)) + JOURNAL_SUFFIX
43
44// Block-read prepare — host-adaptive read block size.
45const READ_BLOCK_DEFAULT = 8 * 1024 * 1024
46const READ_BLOCK_LOW_RAM = 4 * 1024 * 1024
47const LOW_RAM_THRESHOLD = 6 * 1024 * 1024 * 1024
48
49// Cap on stashed just-received chunk buffers (per transfer, insert-refusal), and the
50// hash pump's yield budget — stash-fed iterations never await, so the pump yields by
51// bytes hashed, not chunk count.
52const MEM_STASH_BYTES_DEFAULT = 16 * 1024 * 1024
53const DRAIN_YIELD_BYTES = 2 * 1024 * 1024
54
55// Emit file blocks >= tier maxSize so chunkStream drains every block
56// instead of re-copying a growing pending buffer (64 KiB createReadStream reads force
57// ~56x memcpy at tier 3). Owns its fd and closes it on every terminal path: normal
58// end, consumer break/throw (for-await calls .return() → finally), read error.
59async function * readFileBlocks (filePath, size, blockSize) {
60 const fd = await fs.open(filePath, 'r')
61 try {
62 let pos = 0
63 while (pos < size) {
64 const want = Math.min(blockSize, size - pos)
65 const buf = Buffer.allocUnsafe(want)
66 let filled = 0
67 while (filled < want) {
68 const n = await fs.read(fd, buf, filled, want - filled, pos + filled)
69 if (n <= 0) break
70 filled += n
71 }
72 if (filled === 0) break
73 pos += filled
74 yield filled === buf.length ? buf : buf.subarray(0, filled)
75 }
76 } finally {
77 try { await fs.close(fd) } catch {}
78 }
79}
80
81function bitmapOf (received, total) {
82 const bm = Buffer.alloc((total + 7) >> 3)
83 for (const i of received) bm[i >> 3] |= 1 << (i & 7)
84 return bm
85}
86
87// Drop journals whose partial is gone, are stale, or are corrupt/foreign. Each
88// journal self-describes its partial path (read cheaply from the header). Standalone
89// (no TransferManager instance) so a boot-time sweep can reclaim orphans without
90// the overlay stack being live.
91export function cleanupOrphanedJournals (journalDir, maxAge = 7 * 86400000) {
92 const cleaned = []
93 if (!journalDir) return cleaned
94 let entries
95 try { entries = fs.readdirSync(journalDir) } catch { return cleaned }
96 const now = Date.now()
97 for (const entry of entries) {
98 if (!entry.endsWith(JOURNAL_SUFFIX)) continue
99 const full = path.join(journalDir, entry)
100 let remove = false
101 try {
102 const raw = fs.readFileSync(full)
103 const s = { start: 0, end: raw.length, buffer: raw }
104 if (c.uint32.decode(s) !== JOURNAL_MAGIC) remove = true
105 else {
106 c.uint32.decode(s)
107 const partialPath = c.string.decode(s)
108 if (!fs.existsSync(partialPath)) remove = true
109 else if (now - fs.statSync(full).mtimeMs > maxAge) remove = true
110 }
111 } catch { remove = true }
112 if (remove) { try { fs.unlinkSync(full); cleaned.push(full) } catch {} }
113 }
114 return cleaned
115}
116
117export class TransferManager {
118 constructor (fileIndex, opts = {}) {
119 this._fileIndex = fileIndex
120 this._maxConcurrency = opts.maxConcurrency || 4
121 this._active = new Map() // path → transfer state
122 this._memStashBytes = opts.memStashBytes ?? MEM_STASH_BYTES_DEFAULT
123 this._journalDir = opts.journalDir || null
124 if (this._journalDir) { try { fs.mkdirSync(this._journalDir, { recursive: true }) } catch {} }
125 }
126
127 _journalPathFor (targetPath) {
128 if (!this._journalDir) return null
129 return path.join(this._journalDir, journalNameFor(targetPath))
130 }
131
132 // ── Sender side ─────────────────────────────────────────────
133
134 /**
135 * Prepare a file for sending — read from disk, chunk, return hashes.
136 * Chunk map is persisted for large files (>1MB).
137 *
138 * @param {string} filePath - absolute path on disk
139 * @param {string} overlayPath - path in the overlay namespace
140 * @returns {{ tier: number, chunks: Array<{ hash, offset, length }>, size: number } | null}
141 */
142 async prepareFile (filePath, overlayPath, opts = {}) {
143 // Check file exists and get stats
144 let stat
145 try {
146 stat = fs.statSync(filePath)
147 } catch {
148 return null
149 }
150
151 if (!stat.isFile()) return null
152
153 const mtimeBefore = stat.mtimeMs
154
155 // Check for existing chunk map (skip re-chunking if valid)
156 const existing = await this._fileIndex.getChunkMap(overlayPath)
157 const fileMeta = await this._fileIndex.getFile(overlayPath)
158
159 if (existing && fileMeta && fileMeta.mtime === mtimeBefore) {
160 // Chunk map still valid — file hasn't changed.
161 // Backfill the content-addressed alias so the serve path (a
162 // different overlay key) reuses this map instead of re-reading the file —
163 // covers maps chunked before content-addressed caching existed.
164 if (fileMeta.contentHash && stat.size >= 1048576 && !await this._fileIndex.hasChunkMapByHash(fileMeta.contentHash)) {
165 await this._fileIndex.putChunkMapByHash(fileMeta.contentHash, existing)
166 }
167 return {
168 tier: selectTier(stat.size),
169 chunks: existing,
170 size: stat.size,
171 contentHash: fileMeta.contentHash // surfaced so the publish path gets the hash from one pass
172 }
173 }
174
175 // Read in blocks >= tier maxSize and chunk from views (copy:false).
176 // Cut points are content-stable across block sizes, so peers still match hashes.
177 // size is passed to the hasher so the streaming digest matches hashChunk(buffer).
178 const tier = selectTier(stat.size)
179 const blockSize = Math.max(
180 getTierParams(tier).maxSize,
181 os.totalmem() < LOW_RAM_THRESHOLD ? READ_BLOCK_LOW_RAM : READ_BLOCK_DEFAULT
182 )
183 const fileHasher = createStreamingHasher({ size: stat.size })
184 const chunks = []
185
186 for await (const c of chunkStream(readFileBlocks(filePath, stat.size, blockSize), { tier, copy: false })) {
187 // honor a publish-cancel signal (per-chunk granularity); on throw the
188 // for-await calls readFileBlocks.return() which closes the fd.
189 if (opts.signal?.aborted) { const e = new Error('publish aborted'); e.code = 'ECANCELLED'; throw e }
190 fileHasher.update(c.data)
191 chunks.push({ hash: c.hash, offset: c.offset, length: c.length })
192 opts.onProgress?.(c.length)
193 }
194
195 // mtime guard — abort if file changed during the streaming read. A
196 // vanished source (moved out of the folder mid-read) is just another "changed
197 // during read" case — the open fd kept the read alive after a same-volume
198 // rename, so it is this path-resolving stat that trips ENOENT. Guard it like
199 // the pre-read stat above and re-queue rather than throw.
200 let statAfter
201 try {
202 statAfter = fs.statSync(filePath)
203 } catch {
204 return null // source vanished during read, caller should re-queue
205 }
206 if (statAfter.mtimeMs !== mtimeBefore) {
207 return null // file changed during read, caller should re-queue
208 }
209
210 // Post-read guard: a mid-read shrink can feed the digest fewer bytes than the
211 // size baked into its leaf prefix — re-queue instead of persisting a bad hash.
212 if (fileHasher.bytes !== stat.size) return null
213
214 // Update file index
215 const contentHash = fileHasher.digest()
216
217 // Persist chunk map for large files — both path-keyed (legacy serve lookups)
218 // and content-addressed, so the serve path reuses it across keys.
219 // byHashOnly (the publish prepareForServe pass) keeps ONLY the
220 // content-addressed map — the serve path resolves by hash, so the path-keyed map
221 // + putFile under a throwaway overlay key would be dead, never-read state.
222 if (stat.size >= 1048576) {
223 if (!opts.byHashOnly) await this._fileIndex.putChunkMap(overlayPath, chunks)
224 if (!await this._fileIndex.hasChunkMapByHash(contentHash)) await this._fileIndex.putChunkMapByHash(contentHash, chunks)
225 }
226
227 if (!opts.byHashOnly) {
228 await this._fileIndex.putFile(overlayPath, {
229 contentHash,
230 size: stat.size,
231 mtime: mtimeBefore
232 })
233 }
234
235 return { tier, chunks, size: stat.size, contentHash }
236 }
237
238 /**
239 * Read a specific chunk from disk by its offset and length.
240 * Returns raw bytes — caller sends them to the peer.
241 *
242 * @param {string} filePath - absolute path on disk
243 * @param {number} offset - byte offset
244 * @param {number} length - bytes to read
245 * @returns {Buffer | null}
246 */
247 readChunk (filePath, offset, length) {
248 try {
249 const fd = fs.openSync(filePath, 'r')
250 const buf = Buffer.alloc(length)
251 const bytesRead = fs.readSync(fd, buf, 0, length, offset)
252 fs.closeSync(fd)
253 if (bytesRead !== length) return null
254 return buf
255 } catch {
256 return null
257 }
258 }
259
260 /**
261 * Determine which chunks a peer needs (chunks they don't already have).
262 *
263 * @param {Array<{ hash, offset, length }>} offered - chunks the sender has
264 * @param {Set<string>} peerHas - set of chunk hashes the peer already has
265 * @returns {number[]} indices of chunks the peer needs
266 */
267 computeNeeded (offered, peerHas) {
268 const needed = []
269 for (let i = 0; i < offered.length; i++) {
270 if (!peerHas.has(offered[i].hash)) {
271 needed.push(i)
272 }
273 }
274 return needed
275 }
276
277 // ── Receiver side ───────────────────────────────────────────
278
279 // Start receiving a file. A same-size partial is resumed: the journal restores the
280 // received-set + whole-file hash snapshot in O(1) (no re-read); failing that, an
281 // async, yielding re-verify rebuilds them without blocking the event loop.
282 // opts.isCancelled aborts a long re-verify; opts.onVerifyProgress(0..1) drives the UI.
283 async startReceive (targetPath, meta, opts = {}) {
284 const isCancelled = opts.isCancelled || (() => false)
285 const dir = path.dirname(targetPath)
286 const base = path.basename(targetPath)
287 // Visible partial (no leading dot) so an in-progress download shows in the
288 // destination folder; the distinct ".overlay-partial" suffix keeps host-app
289 // sweeps keyed on a plain ".partial" suffix from deleting a paused overlay
290 // partial.
291 const partialPath = path.join(dir, base + PARTIAL_SUFFIX)
292 const journalPath = this._journalPathFor(targetPath)
293
294 fs.mkdirSync(dir, { recursive: true })
295
296 let resumed = false
297 try { resumed = fs.statSync(partialPath).size === meta.size } catch {}
298
299 let received = new Set()
300 let hasher = meta.contentHash ? createStreamingHasher({ size: meta.size }) : null
301 let hashFrontier = 0
302
303 if (resumed) {
304 const j = journalPath ? this._loadJournal(journalPath, partialPath, meta) : null
305 if (j) {
306 received = j.received
307 hashFrontier = j.hashFrontier
308 if (meta.contentHash) hasher = createStreamingHasher({ size: meta.size, restore: { state: j.hasherState, bytes: j.hasherBytes } })
309 } else {
310 const rec = await this._recoverPartialAsync(partialPath, meta, isCancelled, opts.onVerifyProgress)
311 received = rec.received; hasher = rec.hasher; hashFrontier = rec.hashFrontier
312 }
313 } else {
314 if (journalPath) { try { fs.unlinkSync(journalPath) } catch {} }
315 }
316
317 if (isCancelled()) { const e = new Error('receive cancelled during setup'); e.code = 'ECANCELLED'; throw e }
318
319 // ONE persistent fd for the whole transfer — positioned writeSync/fs.read never
320 // move a shared cursor, so a single handle serves the chunk writes, the hash
321 // pump's gap read-back, and the journal fsync. Replaces an openSync+closeSync per
322 // chunk; 'w+' creates+truncates the fresh partial in the same open.
323 const fd = fs.openSync(partialPath, resumed ? 'r+' : 'w+')
324 if (!resumed) {
325 try { fs.ftruncateSync(fd, meta.size) } catch (err) { try { fs.closeSync(fd) } catch {}; throw err }
326 }
327
328 const total = meta.chunks.length
329 const state = {
330 partialPath,
331 targetPath,
332 journalPath,
333 chunks: meta.chunks,
334 received,
335 bitmap: bitmapOf(received, total),
336 total,
337 size: meta.size,
338 contentHash: meta.contentHash || null,
339 hasher,
340 hashFrontier,
341 advancePromise: null,
342 draining: false,
343 sinceFlush: 0,
344 lastFlush: 0,
345 flushing: false,
346 flushPending: false,
347 fd,
348 // Bounded stash of just-received verified chunk buffers so the hash pump feeds
349 // from memory instead of reading the bytes back off disk. Never load-bearing:
350 // bytes hit disk first; a miss falls back to read-back.
351 memChunks: new Map(),
352 memBytes: 0,
353 stats: { readbacks: 0, stashHits: 0 }
354 }
355
356 // A prior transfer for this path that ended via _fail (stall / all-peers-
357 // gone / disk error) leaves its state here with an open fd — close it before
358 // overwriting so a retry/resume of the same file can't orphan the fd.
359 const prev = this._active.get(targetPath)
360 if (prev) this._closeFd(prev)
361
362 this._active.set(targetPath, state)
363 // A fresh transfer has nothing worth journaling yet; a resume that recovered
364 // chunks persists them now so an immediate re-pause is O(1).
365 if (received.size > 0) this._flushJournal(state)
366 return state
367 }
368
369 // Rebuild the received-set from a same-size partial without blocking the loop:
370 // async reads, a yield every VERIFY_YIELD_EVERY chunks, feeding the contiguous
371 // prefix into the whole-file hasher inline (single pass, no second read).
372 async _recoverPartialAsync (partialPath, meta, isCancelled, onVerifyProgress) {
373 const received = new Set()
374 const hasher = meta.contentHash ? createStreamingHasher({ size: meta.size }) : null
375 let hashFrontier = 0
376 const total = meta.chunks.length
377 const maxLen = meta.chunks.reduce((m, c) => (c.length > m ? c.length : m), 0)
378 const buf = Buffer.allocUnsafe(maxLen)
379 let lastPct = -1
380 onVerifyProgress?.(0)
381 const fd = await fs.open(partialPath, 'r')
382 try {
383 for (let i = 0; i < total; i++) {
384 if (isCancelled()) { const e = new Error('receive recovery cancelled'); e.code = 'ECANCELLED'; throw e }
385 const ci = meta.chunks[i]
386 let n = 0
387 try { n = await fs.read(fd, buf, 0, ci.length, ci.offset) } catch {}
388 const view = buf.subarray(0, ci.length)
389 if (n === ci.length && hashChunk(view) === ci.hash) {
390 received.add(i)
391 if (hasher && i === hashFrontier) { hasher.update(view); hashFrontier++ }
392 }
393 if ((i & (VERIFY_YIELD_EVERY - 1)) === VERIFY_YIELD_EVERY - 1) {
394 const pct = Math.floor(((i + 1) / total) * 100)
395 if (pct !== lastPct) { lastPct = pct; onVerifyProgress?.(pct / 100) }
396 await yieldToLoop()
397 }
398 }
399 } finally {
400 try { await fs.close(fd) } catch {}
401 }
402 onVerifyProgress?.(1)
403 return { received, hasher, hashFrontier }
404 }
405
406 _encodeJournal (state) {
407 const total = state.total
408 const bm = state.bitmap
409 const ch = state.contentHash ? Buffer.from(state.contentHash, 'hex') : Buffer.alloc(32)
410 const snap = state.hasher ? state.hasher.snapshot() : { state: Buffer.alloc(0), bytes: 0 }
411 const s = { start: 0, end: 0, buffer: null }
412 c.uint32.preencode(s, JOURNAL_MAGIC)
413 c.uint32.preencode(s, sodium.crypto_generichash_STATEBYTES)
414 c.string.preencode(s, state.partialPath)
415 c.uint64.preencode(s, state.size)
416 c.uint32.preencode(s, total)
417 c.fixed32.preencode(s, ch)
418 c.uint32.preencode(s, state.hashFrontier)
419 c.uint64.preencode(s, snap.bytes)
420 c.buffer.preencode(s, snap.state)
421 c.buffer.preencode(s, bm)
422 s.buffer = Buffer.allocUnsafe(s.end)
423 c.uint32.encode(s, JOURNAL_MAGIC)
424 c.uint32.encode(s, sodium.crypto_generichash_STATEBYTES)
425 c.string.encode(s, state.partialPath)
426 c.uint64.encode(s, state.size)
427 c.uint32.encode(s, total)
428 c.fixed32.encode(s, ch)
429 c.uint32.encode(s, state.hashFrontier)
430 c.uint64.encode(s, snap.bytes)
431 c.buffer.encode(s, snap.state)
432 c.buffer.encode(s, bm)
433 return s.buffer
434 }
435
436 // Returns the resume state only when the journal is for this partial, binds to
437 // this content, and its snapshot is restorable on this libsodium build; else null
438 // so the caller falls back to the async re-verify.
439 _loadJournal (journalPath, partialPath, meta) {
440 let raw
441 try { raw = fs.readFileSync(journalPath) } catch { return null }
442 try {
443 const s = { start: 0, end: raw.length, buffer: raw }
444 if (c.uint32.decode(s) !== JOURNAL_MAGIC) return null
445 const stateBytes = c.uint32.decode(s)
446 const storedPartial = c.string.decode(s)
447 const size = c.uint64.decode(s)
448 const total = c.uint32.decode(s)
449 const ch = c.fixed32.decode(s).toString('hex')
450 const hashFrontier = c.uint32.decode(s)
451 const hasherBytes = c.uint64.decode(s)
452 const hasherState = c.buffer.decode(s)
453 const bm = c.buffer.decode(s)
454 if (storedPartial !== partialPath) return null
455 if (size !== meta.size || total !== meta.chunks.length) return null
456 if (meta.contentHash && ch !== meta.contentHash) return null
457 if (meta.contentHash) {
458 if (stateBytes !== sodium.crypto_generichash_STATEBYTES) return null
459 if (hasherState.length !== sodium.crypto_generichash_STATEBYTES) return null
460 }
461 const received = new Set()
462 for (let i = 0; i < total; i++) if (bm[i >> 3] & (1 << (i & 7))) received.add(i)
463 return { received, hashFrontier, hasherState, hasherBytes }
464 } catch { return null }
465 }
466
467 // Best-effort async crash-hedge flush: never blocks the receive loop, single-flight
468 // (a flush requested while one runs coalesces into one more pass). fsync the partial
469 // BEFORE the journal references it so a power-loss can never leave the journal
470 // marking a chunk received whose bytes aren't durable.
471 _flushJournal (state) {
472 if (!state.contentHash || !state.journalPath || state.fd == null) return
473 if (state.flushing) { state.flushPending = true; return }
474 if (Date.now() - state.lastFlush < JOURNAL_FLUSH_INTERVAL_MS) return
475 state.flushing = true
476 this._drainFlush(state)
477 }
478
479 async _drainFlush (state) {
480 do {
481 state.flushPending = false
482 state.lastFlush = Date.now()
483 if (state.fd == null) break
484 let buf
485 try { buf = this._encodeJournal(state) } catch { break }
486 const tmp = state.journalPath + '.tmp'
487 try {
488 try { await fs.fsync(state.fd) } catch {}
489 await fs.writeFile(tmp, buf)
490 if (state.fd == null) break // paused/cancelled mid-flush — don't re-create the journal
491 await fs.rename(tmp, state.journalPath)
492 } catch { try { await fs.unlink(tmp) } catch {} }
493 } while (state.flushPending && state.fd != null && state.contentHash)
494 try { await fs.unlink(state.journalPath + '.tmp') } catch {}
495 state.flushing = false
496 }
497
498 // Synchronous durable flush for pause/shutdown, where the app may quit before an
499 // async flush would land.
500 _flushJournalSync (state) {
501 if (!state.contentHash || !state.journalPath) return
502 let buf
503 try { buf = this._encodeJournal(state) } catch { return }
504 if (state.fd != null) { try { fs.fsyncSync(state.fd) } catch {} }
505 try {
506 const tmp = state.journalPath + '.tmp'
507 fs.writeFileSync(tmp, buf)
508 fs.renameSync(tmp, state.journalPath)
509 state.lastFlush = Date.now()
510 } catch { try { fs.unlinkSync(state.journalPath + '.tmp') } catch {} }
511 }
512
513 cleanJournals (maxAge) { return cleanupOrphanedJournals(this._journalDir, maxAge) }
514
515 async _hashWholeFileAsync (filePath, size) {
516 const h = createStreamingHasher({ size })
517 for await (const buf of readFileBlocks(filePath, size, 8 * 1024 * 1024)) {
518 h.update(buf)
519 await yieldToLoop()
520 }
521 return h.digest()
522 }
523
524 // Advance the whole-file hash over the contiguous run of received chunks
525 // as a single-flight BACKGROUND pump: chunk index order == file offset order, so
526 // once chunk `hashFrontier` is present we feed its bytes and move on. The drain is
527 // async + yielding so a large contiguous run that suddenly becomes available (a
528 // gap filling after a low-frontier resume) cannot block the event loop.
529 // Single-flight is `state.draining`, set/cleared SYNCHRONOUSLY inside _drainHash —
530 // a stash-fed drain completes without awaiting, so a promise-based gate would stay
531 // stale until its microtask clear and block same-macrotask restarts (protomux
532 // delivers frame bursts synchronously). advancePromise is only the awaitable
533 // handle for finalize/pause; its clear is guarded against nulling a newer drain.
534 _advanceHash (state) {
535 if (!state.hasher || state.draining) return
536 if (!(state.hashFrontier < state.chunks.length && state.received.has(state.hashFrontier))) return
537 const p = this._drainHash(state)
538 state.advancePromise = p
539 p.catch(() => {}).then(() => { if (state.advancePromise === p) state.advancePromise = null })
540 }
541
542 async _drainHash (state) {
543 state.draining = true
544 try {
545 let sinceYield = 0
546 while (state.fd != null && state.hashFrontier < state.chunks.length && state.received.has(state.hashFrontier)) {
547 const i = state.hashFrontier
548 const c = state.chunks[i]
549 let buf = state.memChunks.get(i)
550 if (buf !== undefined) {
551 state.memChunks.delete(i)
552 state.memBytes -= buf.length
553 state.stats.stashHits++
554 } else {
555 buf = Buffer.alloc(c.length)
556 try { await fs.read(state.fd, buf, 0, c.length, c.offset) } catch { return }
557 state.stats.readbacks++
558 }
559 state.hasher.update(buf)
560 state.hashFrontier++
561 // Yield by BYTE volume: the stash path has no awaits, so a chunk-count
562 // cadence would let a gap-fill hash the whole stash synchronously inside
563 // the network event that delivered the gap chunk.
564 sinceYield += buf.length
565 if (sinceYield >= DRAIN_YIELD_BYTES) { sinceYield = 0; await yieldToLoop() }
566 }
567 } finally {
568 state.draining = false
569 }
570 }
571
572 // Close the persistent fd — MUST run before finalize's rename (an open handle
573 // blocks the rename on Windows) — and drop the chunk stash.
574 _closeFd (state) {
575 if (state.fd != null) {
576 try { fs.closeSync(state.fd) } catch {}
577 state.fd = null
578 }
579 state.memChunks.clear()
580 state.memBytes = 0
581 }
582
583 /**
584 * Write a received chunk to the partial file. Verifies the hash.
585 *
586 * @param {string} targetPath - the final file path (used as key)
587 * @param {number} index - chunk index
588 * @param {Buffer} data - raw chunk bytes
589 * @returns {{ ok: boolean, error?: string }}
590 */
591 writeChunk (targetPath, index, data) {
592 const state = this._active.get(targetPath)
593 if (!state) return { ok: false, error: 'no active transfer' }
594 if (index < 0 || index >= state.chunks.length) return { ok: false, error: 'index out of range' }
595 // Terminal race (chunk landed after close) — codeless so the scheduler
596 // re-assigns instead of failing the fetch.
597 if (state.fd == null) return { ok: false, error: 'transfer closed' }
598
599 const expected = state.chunks[index]
600
601 // Verify hash
602 const actual = hashChunk(data)
603 if (actual !== expected.hash) {
604 return { ok: false, error: `hash mismatch at index ${index}: expected ${expected.hash.slice(0, 16)}... got ${actual.slice(0, 16)}...` }
605 }
606
607 // Verify length
608 if (data.length !== expected.length) {
609 return { ok: false, error: `length mismatch at index ${index}: expected ${expected.length} got ${data.length}` }
610 }
611
612 // Write to partial at the chunk's offset through the persistent fd. A local
613 // I/O failure here (ENOSPC / EACCES / EROFS) is fatal and non-retryable —
614 // return its code so the scheduler fails the fetch instead of looping over
615 // other peers. Loop on a SHORT count: libuv swallows an error after partial
616 // progress and returns short without throwing; retrying the remainder
617 // surfaces the real coded error instead of leaving a silent hole the
618 // stash-fed digest would never see.
619 try {
620 let written = 0
621 while (written < data.length) {
622 const n = fs.writeSync(state.fd, data, written, data.length - written, expected.offset + written)
623 if (!(n > 0)) { const e = new Error(`short write at index ${index}: ${written}/${data.length} bytes`); e.code = 'EIO'; throw e }
624 written += n
625 }
626 } catch (err) {
627 return { ok: false, error: err.message, code: err.code }
628 }
629
630 state.received.add(index)
631 state.bitmap[index >> 3] |= 1 << (index & 7)
632 // Stash only what the pump can consume cleanly: skip when already hashed or
633 // stashed, when the pump is mid-drain on exactly this index, or over cap.
634 // Copied, not aliased — wire buffers are views into shared network slabs (udx
635 // read slab, decrypted in place by secret-stream), so retaining `data` would
636 // pin whole slabs and bet the digest on their write-once behavior.
637 if (state.hasher && !state.memChunks.has(index) && state.memBytes + data.length <= this._memStashBytes &&
638 (index > state.hashFrontier || (index === state.hashFrontier && !state.draining))) {
639 state.memChunks.set(index, Buffer.from(data))
640 state.memBytes += data.length
641 }
642 this._advanceHash(state)
643 if (++state.sinceFlush >= JOURNAL_FLUSH_EVERY) { state.sinceFlush = 0; this._flushJournal(state) }
644
645 return { ok: true }
646 }
647
648 /**
649 * Check if a transfer is complete (all chunks received).
650 *
651 * @param {string} targetPath
652 * @returns {boolean}
653 */
654 isComplete (targetPath) {
655 const state = this._active.get(targetPath)
656 if (!state) return false
657 return state.received.size === state.total
658 }
659
660 /**
661 * Finalize a completed transfer — atomic rename from partial to target.
662 *
663 * @param {string} targetPath
664 * @returns {{ ok: boolean, error?: string }}
665 */
666 async finalize (targetPath) {
667 const state = this._active.get(targetPath)
668 if (!state) return { ok: false, error: 'no active transfer' }
669 if (state.received.size !== state.total) {
670 return { ok: false, error: `incomplete: ${state.received.size}/${state.total} chunks` }
671 }
672
673 // whole-file integrity — computed incrementally by the background hash
674 // pump during the transfer. Kick the pump and await it to quiescence, then read
675 // the digest (no full re-read). On a mismatch, drop the corrupt partial so a
676 // retry starts clean.
677 if (state.hasher) {
678 this._advanceHash(state)
679 while (state.advancePromise) { try { await state.advancePromise } catch {} }
680 }
681 if (state.contentHash) {
682 const actual = state.hashFrontier === state.chunks.length ? state.hasher.digest() : null
683 if (actual !== state.contentHash) {
684 this._closeFd(state)
685 try { fs.unlinkSync(state.partialPath) } catch {}
686 if (state.journalPath) { try { fs.unlinkSync(state.journalPath) } catch {} }
687 this._active.delete(targetPath)
688 return { ok: false, error: 'content-hash mismatch', code: 'EHASHMISMATCH' }
689 }
690 }
691 this._closeFd(state)
692
693 try {
694 fs.renameSync(state.partialPath, state.targetPath)
695 } catch (err) {
696 // Carry the code (routes through the coded local-I/O rethrow) and clear the
697 // state — the partial + journal stay for a clean resume-retry.
698 this._active.delete(targetPath)
699 return { ok: false, error: 'rename failed: ' + err.message, code: err.code }
700 }
701
702 if (state.journalPath) { try { fs.unlinkSync(state.journalPath) } catch {} }
703 this._active.delete(targetPath)
704 return { ok: true }
705 }
706
707 // Stop receiving but KEEP the partial on disk so a later startReceive
708 // verifies and resumes it. Distinct from cancel(), which unlinks the partial.
709 // Drains the in-flight hash advance to the full contiguous frontier first, so the
710 // journal captures it and resume stays O(1) (no re-read) and deterministic.
711 async pause (targetPath) {
712 const state = this._active.get(targetPath)
713 if (!state) return
714 if (state.hasher) {
715 this._advanceHash(state)
716 while (state.advancePromise) { try { await state.advancePromise } catch {} }
717 }
718 this._flushJournalSync(state)
719 this._closeFd(state)
720 this._active.delete(targetPath)
721 }
722
723 /**
724 * Cancel an active transfer and clean up the partial file.
725 *
726 * @param {string} targetPath
727 */
728 cancel (targetPath) {
729 const state = this._active.get(targetPath)
730 if (!state) return
731
732 this._closeFd(state)
733 try {
734 fs.unlinkSync(state.partialPath)
735 } catch {
736 // partial file may not exist
737 }
738 if (state.journalPath) { try { fs.unlinkSync(state.journalPath) } catch {} }
739
740 this._active.delete(targetPath)
741 }
742
743 /**
744 * Get transfer progress for an active transfer.
745 *
746 * @param {string} targetPath
747 * @returns {{ received: number, total: number, percentage: number } | null}
748 */
749 getProgress (targetPath) {
750 const state = this._active.get(targetPath)
751 if (!state) return null
752
753 return {
754 received: state.received.size,
755 total: state.total,
756 percentage: state.total > 0 ? Math.round(state.received.size / state.total * 100) : 0
757 }
758 }
759
760 /**
761 * List all active transfers.
762 *
763 * @returns {Array<{ path: string, received: number, total: number, percentage: number }>}
764 */
765 listActive () {
766 const result = []
767 for (const [targetPath, state] of this._active) {
768 result.push({
769 path: targetPath,
770 received: state.received.size,
771 total: state.total,
772 percentage: state.total > 0 ? Math.round(state.received.size / state.total * 100) : 0
773 })
774 }
775 return result
776 }
777
778 /**
779 * Clean up stale partial files in a directory (older than maxAge).
780 *
781 * @param {string} dir - directory to scan
782 * @param {number} [maxAge=86400000] - max age in ms (default 24h)
783 * @returns {string[]} paths of cleaned files
784 */
785 cleanPartials (dir, maxAge = 86400000) {
786 const cleaned = []
787 try {
788 const entries = fs.readdirSync(dir)
789 const now = Date.now()
790 for (const entry of entries) {
791 if (!entry.endsWith(PARTIAL_SUFFIX)) continue
792 const full = path.join(dir, entry)
793 try {
794 const stat = fs.statSync(full)
795 if (now - stat.mtimeMs > maxAge) {
796 fs.unlinkSync(full)
797 cleaned.push(full)
798 }
799 } catch {
800 // skip unreadable files
801 }
802 }
803 } catch {
804 // directory may not exist
805 }
806 return cleaned
807 }
808}
node_modules +1 -0
……@@ -0,0 +1 @@
1/storage/projects/code/hypercore-kb/hyper-fleeky/hyper-overlay/node_modules