| … | … | @@ -178,11 +178,160 @@ export class DeployServer extends ReadyResource { |
| 178 | 178 | return this._sessions.size |
| 179 | 179 | } |
| 180 | 180 | |
| 181 | async rescan () { | |
| 181 | /** | |
| 182 | * Refresh the file/tree index from disk. | |
| 183 | * | |
| 184 | * Default behavior (legacy): full rescan — walks the entire `this.dir`, | |
| 185 | * re-hashes every file, rebuilds the manifest from scratch. O(total | |
| 186 | * bytes in the served directory). Caller-cost: many minutes for a | |
| 187 | * pgh-sidecar tree-store with tens of thousands of files. | |
| 188 | * | |
| 189 | * Incremental mode: pass `{ newHashes: ['<hash1>', ...] }` to scan | |
| 190 | * ONLY the named subdirectories (e.g. `<dir>/<hash1>/`). The result | |
| 191 | * is appended to the existing manifest. O(bytes in those subdirs). | |
| 192 | * Use this when you know exactly which trees were just staged — | |
| 193 | * publish-release does, since the caller is naming the tree. | |
| 194 | * | |
| 195 | * Removed-on-disk files are NOT reaped in incremental mode (they | |
| 196 | * stay in the manifest until the next full rescan). For pgh's | |
| 197 | * append-only publish flow this is correct; if you need eviction | |
| 198 | * call rescan() with no args to do a full rebuild. | |
| 199 | */ | |
| 200 | async rescan (opts = {}) { | |
| 201 | if (opts && Array.isArray(opts.newHashes) && opts.newHashes.length > 0) { | |
| 202 | const added = await this._scanIncremental(opts.newHashes) | |
| 203 | return { | |
| 204 | files: this.manifest.files.length, | |
| 205 | totalSize: this.manifest.totalSize, | |
| 206 | incremental: true, | |
| 207 | addedFiles: added.addedFiles, | |
| 208 | addedBytes: added.addedBytes, | |
| 209 | scannedHashes: opts.newHashes.length | |
| 210 | } | |
| 211 | } | |
| 182 | 212 | await this._scan() |
| 183 | 213 | return { files: this.manifest.files.length, totalSize: this.manifest.totalSize } |
| 184 | 214 | } |
| 185 | 215 | |
| 216 | /** | |
| 217 | * Walk only the listed top-level subdirs (typically tree-hash dirs | |
| 218 | * staged by `pgh publish`) and add their files + trees to the | |
| 219 | * existing fileIndex + manifest. No re-hashing of paths already in | |
| 220 | * the manifest. | |
| 221 | * | |
| 222 | * @param {string[]} subPaths - directory names relative to this.dir | |
| 223 | * @returns {{ addedFiles: number, addedBytes: number }} | |
| 224 | */ | |
| 225 | async _scanIncremental (subPaths) { | |
| 226 | if (!this._symlinkBlobsDir) { | |
| 227 | this._symlinkBlobsDir = path.join(os.tmpdir(), 'hyper-deploy-symlinks-' + crypto.randomBytes(8).toString('hex')) | |
| 228 | fs.mkdirSync(this._symlinkBlobsDir, { recursive: true }) | |
| 229 | } | |
| 230 | if (!this.manifest) { | |
| 231 | // First-ever scan or after a full reset — fall back to legacy full scan, | |
| 232 | // since incremental can't synthesize the manifest skeleton from nothing. | |
| 233 | await this._scan() | |
| 234 | return { addedFiles: 0, addedBytes: 0 } | |
| 235 | } | |
| 236 | ||
| 237 | let addedFiles = 0 | |
| 238 | let addedBytes = 0 | |
| 239 | ||
| 240 | // Track paths already in manifest so we don't double-add on | |
| 241 | // overlapping incremental calls. | |
| 242 | const existing = new Set(this.manifest.files.map(f => f.path)) | |
| 243 | ||
| 244 | for (const sub of subPaths) { | |
| 245 | const subDir = path.join(this.dir, sub) | |
| 246 | let stat | |
| 247 | try { stat = fs.lstatSync(subDir) } catch { continue } | |
| 248 | if (!stat.isDirectory()) continue | |
| 249 | ||
| 250 | const files = walkDir(subDir, sub) | |
| 251 | for (let i = 0; i < files.length; i++) { | |
| 252 | const f = files[i] | |
| 253 | if (existing.has(f.path)) continue // already indexed | |
| 254 | ||
| 255 | let diskPath, contentHash, mtime | |
| 256 | if (f.isSymlink) { | |
| 257 | const linkBytes = Buffer.from(f.linkTarget, 'utf8') | |
| 258 | contentHash = hashChunk(linkBytes) | |
| 259 | diskPath = path.join(this._symlinkBlobsDir, contentHash) | |
| 260 | if (!fs.existsSync(diskPath)) fs.writeFileSync(diskPath, linkBytes) | |
| 261 | mtime = Date.now() | |
| 262 | } else { | |
| 263 | diskPath = path.join(this.dir, f.path) | |
| 264 | contentHash = hashFile(diskPath) | |
| 265 | mtime = fs.statSync(diskPath).mtimeMs | |
| 266 | } | |
| 267 | f.diskPath = diskPath | |
| 268 | ||
| 269 | await this._fileIndex.putFile(f.path, { | |
| 270 | contentHash, | |
| 271 | size: f.size, | |
| 272 | mtime | |
| 273 | }) | |
| 274 | ||
| 275 | if (f.size >= CHUNK_THRESHOLD) { | |
| 276 | await this._transferManager.prepareFile(diskPath, f.path) | |
| 277 | } | |
| 278 | ||
| 279 | this.manifest.files.push({ | |
| 280 | path: f.path, | |
| 281 | size: f.size, | |
| 282 | diskPath: f.diskPath, | |
| 283 | hash: contentHash | |
| 284 | }) | |
| 285 | existing.add(f.path) | |
| 286 | addedFiles++ | |
| 287 | addedBytes += f.size | |
| 288 | this.manifest.totalSize += f.size | |
| 289 | ||
| 290 | // Also push the new entry into the overlay protocol's lookup | |
| 291 | // maps so peers connected BEFORE this rescan can find the | |
| 292 | // new bytes via content-request without reconnecting. Without | |
| 293 | // this, the maps are stale until the next inbound connection | |
| 294 | // overwrites them via `_onConnection`. | |
| 295 | if (this._overlayProtocol) { | |
| 296 | if (this._overlayProtocol._contentHashPaths) { | |
| 297 | this._overlayProtocol._contentHashPaths.set(contentHash, diskPath) | |
| 298 | } | |
| 299 | if (this._overlayProtocol._filePaths) { | |
| 300 | this._overlayProtocol._filePaths.set(f.path, diskPath) | |
| 301 | } | |
| 302 | } | |
| 303 | } | |
| 304 | ||
| 305 | // Tree-level publish for the newly-added subdir. walkTree(subDir) | |
| 306 | // produces tree hashes for the new subtree only; we putTree each | |
| 307 | // one and putTreePath at sub-relative paths plus '/'. | |
| 308 | try { | |
| 309 | const trees = walkTree(subDir) | |
| 310 | for (const [relPath, tree] of trees) { | |
| 311 | await this._fileIndex.putTree(tree.hash, { | |
| 312 | entries: tree.entries, | |
| 313 | size: tree.size | |
| 314 | }) | |
| 315 | // The relPath here is sub-tree-relative. Tree-by-path lookups | |
| 316 | // use this.dir-relative paths — prefix `sub`. The empty | |
| 317 | // relPath is the subtree root, which maps to `sub` in our | |
| 318 | // top-level naming. | |
| 319 | const fullRel = relPath ? sub + '/' + relPath : sub | |
| 320 | await this._fileIndex.putTreePath(fullRel, tree.hash) | |
| 321 | } | |
| 322 | // The new subtree's root is itself a tree advertised at hash = | |
| 323 | // tree-by-name. The PUBLISHER calls us with the tree hash so we | |
| 324 | // also want path = '<sub>' to map there, which the loop above | |
| 325 | // covers for relPath = ''. | |
| 326 | this.emit('incremental-indexed', sub, trees.size) | |
| 327 | } catch (err) { | |
| 328 | this.emit('tree-publish-error', err) | |
| 329 | } | |
| 330 | } | |
| 331 | ||
| 332 | return { addedFiles, addedBytes } | |
| 333 | } | |
| 334 | ||
| 186 | 335 | async _scan () { |
| 187 | 336 | const files = walkDir(this.dir) |
| 188 | 337 | let totalSize = 0 |
| … | … | @@ -1,6 +1,6 @@ |
| 1 | 1 | { |
| 2 | 2 | "name": "hyper-deploy", |
| 3 | "version": "0.4.2", | |
| 3 | "version": "0.4.3", | |
| 4 | 4 | "description": "P2P release distribution — serve build artifacts to peers over Hyperswarm", |
| 5 | 5 | "type": "module", |
| 6 | 6 | "main": "index.js", |
| … | … | @@ -0,0 +1,130 @@ |
| 1 | /** | |
| 2 | * Incremental rescan proof. | |
| 3 | * | |
| 4 | * Asserts: | |
| 5 | * 1. After `rescan({ newHashes: [hash] })`, the named subdir's | |
| 6 | * files appear in the manifest + fileIndex. | |
| 7 | * 2. Files NOT in the named subdir are not re-hashed (we test this | |
| 8 | * by mutating an existing file's bytes WITHOUT updating mtime — | |
| 9 | * a full rescan would detect new content; incremental ignores). | |
| 10 | * 3. Subsequent incremental scans accumulate without duplicates. | |
| 11 | * 4. The legacy full-rescan still works (no opts). | |
| 12 | */ | |
| 13 | ||
| 14 | import test from 'brittle' | |
| 15 | import fs from 'bare-fs' | |
| 16 | import path from 'bare-path' | |
| 17 | import os from 'bare-os' | |
| 18 | import crypto from 'hypercore-crypto' | |
| 19 | ||
| 20 | import { DeployServer } from '../index.js' | |
| 21 | ||
| 22 | function mktemp (prefix) { | |
| 23 | const dir = path.join(os.tmpdir(), 'hd-incr-' + prefix + '-' + Date.now() + '-' + Math.random().toString(36).slice(2, 6)) | |
| 24 | fs.mkdirSync(dir, { recursive: true }) | |
| 25 | return dir | |
| 26 | } | |
| 27 | ||
| 28 | function writeTree (root, treeName, files) { | |
| 29 | const treeDir = path.join(root, treeName) | |
| 30 | fs.mkdirSync(treeDir, { recursive: true }) | |
| 31 | for (const [name, content] of Object.entries(files)) { | |
| 32 | fs.writeFileSync(path.join(treeDir, name), content) | |
| 33 | } | |
| 34 | return treeDir | |
| 35 | } | |
| 36 | ||
| 37 | test('rescan({ newHashes }) — indexes only the named subdir', async (t) => { | |
| 38 | const dir = mktemp('only-new') | |
| 39 | ||
| 40 | // Pre-seed an existing tree the server will see at first scan. | |
| 41 | writeTree(dir, 'tree-A', { 'a.txt': 'alpha', 'b.txt': 'beta' }) | |
| 42 | ||
| 43 | const server = new DeployServer(dir, { topic: crypto.randomBytes(32) }) | |
| 44 | await server.ready() | |
| 45 | ||
| 46 | // After ready: full scan ran. Manifest should have tree-A's 2 files. | |
| 47 | t.is(server.manifest.files.length, 2, 'initial full scan saw 2 files') | |
| 48 | ||
| 49 | // Stage a NEW tree on disk. | |
| 50 | writeTree(dir, 'tree-B', { 'c.txt': 'gamma' }) | |
| 51 | ||
| 52 | // Incremental rescan naming tree-B only. | |
| 53 | const result = await server.rescan({ newHashes: ['tree-B'] }) | |
| 54 | t.ok(result.incremental, 'reported as incremental') | |
| 55 | t.is(result.addedFiles, 1, 'added exactly 1 file (c.txt)') | |
| 56 | t.is(server.manifest.files.length, 3, 'manifest now has 3 files') | |
| 57 | ||
| 58 | // Verify the new file is in fileIndex. | |
| 59 | const meta = await server._fileIndex.getFile('tree-B/c.txt') | |
| 60 | t.ok(meta, 'tree-B/c.txt indexed') | |
| 61 | t.is(meta.size, 'gamma'.length) | |
| 62 | ||
| 63 | await server.close() | |
| 64 | fs.rmSync(dir, { recursive: true, force: true }) | |
| 65 | }) | |
| 66 | ||
| 67 | test('rescan({ newHashes }) — does NOT re-walk other subdirs', async (t) => { | |
| 68 | const dir = mktemp('no-rewalk') | |
| 69 | writeTree(dir, 'tree-A', { 'a.txt': 'alpha' }) | |
| 70 | ||
| 71 | const server = new DeployServer(dir, { topic: crypto.randomBytes(32) }) | |
| 72 | await server.ready() | |
| 73 | ||
| 74 | // Capture tree-A's recorded hash, then change its content on disk | |
| 75 | // WITHOUT calling full rescan. A full _scan() would notice and update | |
| 76 | // the contentHash; an incremental scan naming only tree-B must NOT. | |
| 77 | const before = await server._fileIndex.getFile('tree-A/a.txt') | |
| 78 | ||
| 79 | fs.writeFileSync(path.join(dir, 'tree-A', 'a.txt'), 'MUTATED') | |
| 80 | ||
| 81 | writeTree(dir, 'tree-B', { 'b.txt': 'beta' }) | |
| 82 | await server.rescan({ newHashes: ['tree-B'] }) | |
| 83 | ||
| 84 | // tree-A's recorded contentHash MUST be unchanged — we didn't ask | |
| 85 | // for it to be rescanned. | |
| 86 | const after = await server._fileIndex.getFile('tree-A/a.txt') | |
| 87 | t.is(after.contentHash, before.contentHash, 'tree-A entry untouched') | |
| 88 | ||
| 89 | await server.close() | |
| 90 | fs.rmSync(dir, { recursive: true, force: true }) | |
| 91 | }) | |
| 92 | ||
| 93 | test('rescan({ newHashes }) — idempotent on duplicate calls', async (t) => { | |
| 94 | const dir = mktemp('idempotent') | |
| 95 | writeTree(dir, 'tree-A', { 'a.txt': 'alpha' }) | |
| 96 | ||
| 97 | const server = new DeployServer(dir, { topic: crypto.randomBytes(32) }) | |
| 98 | await server.ready() | |
| 99 | ||
| 100 | writeTree(dir, 'tree-B', { 'b.txt': 'beta' }) | |
| 101 | ||
| 102 | const first = await server.rescan({ newHashes: ['tree-B'] }) | |
| 103 | t.is(first.addedFiles, 1, 'first call adds the file') | |
| 104 | ||
| 105 | const second = await server.rescan({ newHashes: ['tree-B'] }) | |
| 106 | t.is(second.addedFiles, 0, 'second call adds nothing (already indexed)') | |
| 107 | t.is(server.manifest.files.length, 2, 'manifest still has 2 files total') | |
| 108 | ||
| 109 | await server.close() | |
| 110 | fs.rmSync(dir, { recursive: true, force: true }) | |
| 111 | }) | |
| 112 | ||
| 113 | test('rescan() with no args still does the legacy full scan', async (t) => { | |
| 114 | const dir = mktemp('legacy') | |
| 115 | writeTree(dir, 'tree-A', { 'a.txt': 'alpha' }) | |
| 116 | ||
| 117 | const server = new DeployServer(dir, { topic: crypto.randomBytes(32) }) | |
| 118 | await server.ready() | |
| 119 | ||
| 120 | t.is(server.manifest.files.length, 1) | |
| 121 | ||
| 122 | // Add a new tree on disk + call rescan() with NO opts → legacy full scan. | |
| 123 | writeTree(dir, 'tree-B', { 'b.txt': 'beta' }) | |
| 124 | const result = await server.rescan() | |
| 125 | t.absent(result.incremental, 'no incremental flag for legacy path') | |
| 126 | t.is(server.manifest.files.length, 2, 'full scan picked up the new file') | |
| 127 | ||
| 128 | await server.close() | |
| 129 | fs.rmSync(dir, { recursive: true, force: true }) | |
| 130 | }) |