OverviewFilesCommitsIssuesDocsReleasesPeers

hyper-deploy commits / 765988a

0.4.2: index symlinks as content-by-hash (fixes install-stuck-at-symlink)
walkDir used statSync (follow-symlinks). For symlink entries pointing at a directory, it either recursed into the target (leaking out-of-tree files into the served index) or silently dropped them. The link-target STRING was never registered as content-by-hash. But hyper-overlay/lib/tree.js:walkTree (used by pgh publish to compute tree hashes) uses lstatSync and emits KIND_SYMLINK entries with childHash = blake2b(linkTargetBytes). So every published tree containing a symlink referenced a content hash the server had no entry for. content-request silently dropped (protocol-v2.js:430), client hung indefinitely. End-to-end symptom reproduced on hyper-tps install: stortron pulled all files up to and including the alphabetically-last regular file before addons/, then froze on addons/SunshineClouds2 (a symlink). Loopback NetworkFetcher.getContent(symlinkHash) hung the same way. Fix: - walkDir uses lstatSync, emits {isSymlink, linkTarget} leaf entries (closes the out-of-tree leak too — never recurses through symlinks) - _scan writes each unique link-target string to a temp blob at os.tmpdir()/hyper-deploy-symlinks-<rand>/<contentHash>; manifest entries carry an f.diskPath field pointing to the blob (symlinks) or the source file (regular files) - _onConnection populates contentHashPaths/filePaths from f.diskPath instead of path.join(this.dir, f.path). Falls back to dir-relative for back-compat with older manifest shapes - _close removes the temp blob dir Test: test/symlink-serve.test.js — 3 tests / 11 asserts. End-to-end verified: `pgh app install hyper-tps` on stortron now completes in 3m46s (was: silent hang at first symlink, forever). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
765988a25b91455c6c0aeef6c174b5b25c80e5e9
by plankatron <fleeky@gmail.com> · 2026-05-12T17:21:47.000Z

Changed files (3) +149 -9

index.js +54 -8
……@@ -38,6 +38,15 @@ const CHUNK_THRESHOLD = 1048576 // 1 MB — files below this transfer as whole b
3838
3939// ── Helpers ──────────────────────────────────────────────────
4040
41// Walks `dir` and returns an array of entries describing every file and
42// symlink found. Symlink ENTRIES are returned with the link-target string
43// as their served content (not the file they resolve to) — this matches
44// hyper-overlay/lib/tree.js:walkTree, which records KIND_SYMLINK entries
45// the same way. Without this, the link-target string's content hash is
46// never indexed for serving, so any client materializing a tree with
47// symlinks hangs on the symlink's content-request (root cause of the
48// "install stuck at SunshineClouds2" hang). Uses lstatSync so symlinks
49// are NOT followed.
4150function walkDir (dir, prefix) {
4251 prefix = prefix || ''
4352 const results = []
……@@ -45,9 +54,16 @@ function walkDir (dir, prefix) {
4554 const full = path.join(dir, entry)
4655 const rel = prefix ? prefix + '/' + entry : entry
4756 try {
48 const stat = fs.statSync(full)
49 if (stat.isDirectory()) results.push(...walkDir(full, rel))
50 else if (stat.isFile()) results.push({ path: rel, size: stat.size })
57 const stat = fs.lstatSync(full)
58 if (stat.isSymbolicLink()) {
59 const linkTarget = fs.readlinkSync(full)
60 const targetBytes = Buffer.from(linkTarget, 'utf8')
61 results.push({ path: rel, size: targetBytes.length, isSymlink: true, linkTarget })
62 } else if (stat.isDirectory()) {
63 results.push(...walkDir(full, rel))
64 } else if (stat.isFile()) {
65 results.push({ path: rel, size: stat.size })
66 }
5167 } catch {}
5268 }
5369 return results
……@@ -148,6 +164,10 @@ export class DeployServer extends ReadyResource {
148164 }
149165 if (this._fileIndex) await this._fileIndex.close()
150166 if (this._ownStore && this._corestore) await this._corestore.close()
167 if (this._symlinkBlobsDir) {
168 try { fs.rmSync(this._symlinkBlobsDir, { recursive: true, force: true }) } catch {}
169 this._symlinkBlobsDir = null
170 }
151171 }
152172
153173 get topicHex () {
……@@ -167,20 +187,41 @@ export class DeployServer extends ReadyResource {
167187 const files = walkDir(this.dir)
168188 let totalSize = 0
169189
190 // Scratch dir for symlink-target blobs. Each unique link-target
191 // string gets materialized to <scratch>/<contentHash> so that the
192 // existing diskPath-based content-request flow can serve it via
193 // fs.readFileSync. Single source of bytes; the protocol layer does
194 // not need to know about symlinks.
195 if (!this._symlinkBlobsDir) {
196 this._symlinkBlobsDir = path.join(os.tmpdir(), 'hyper-deploy-symlinks-' + crypto.randomBytes(8).toString('hex'))
197 fs.mkdirSync(this._symlinkBlobsDir, { recursive: true })
198 }
199
170200 for (let i = 0; i < files.length; i++) {
171201 const f = files[i]
172 const fullPath = path.join(this.dir, f.path)
173 const contentHash = hashFile(fullPath)
202 let diskPath, contentHash, mtime
203 if (f.isSymlink) {
204 const linkBytes = Buffer.from(f.linkTarget, 'utf8')
205 contentHash = hashChunk(linkBytes)
206 diskPath = path.join(this._symlinkBlobsDir, contentHash)
207 if (!fs.existsSync(diskPath)) fs.writeFileSync(diskPath, linkBytes)
208 mtime = Date.now()
209 } else {
210 diskPath = path.join(this.dir, f.path)
211 contentHash = hashFile(diskPath)
212 mtime = fs.statSync(diskPath).mtimeMs
213 }
214 f.diskPath = diskPath
174215
175216 await this._fileIndex.putFile(f.path, {
176217 contentHash,
177218 size: f.size,
178 mtime: fs.statSync(fullPath).mtimeMs
219 mtime
179220 })
180221
181222 // For large files, prepare chunk map via TransferManager
182223 if (f.size >= CHUNK_THRESHOLD) {
183 await this._transferManager.prepareFile(fullPath, f.path)
224 await this._transferManager.prepareFile(diskPath, f.path)
184225 }
185226
186227 totalSize += f.size
……@@ -191,6 +232,7 @@ export class DeployServer extends ReadyResource {
191232 files: files.map(f => ({
192233 path: f.path,
193234 size: f.size,
235 diskPath: f.diskPath, // points to symlink blob OR source file, used by _onConnection
194236 hash: null // populated lazily from fileIndex
195237 })),
196238 totalSize,
……@@ -253,7 +295,11 @@ export class DeployServer extends ReadyResource {
253295 const filePaths = new Map()
254296 for (const f of (this.manifest && this.manifest.files) || []) {
255297 if (f.hash) {
256 const diskPath = path.join(this.dir, f.path)
298 // f.diskPath points to the symlink-target blob for symlink
299 // entries and to the actual source file for regular files.
300 // Falls back to dir-relative path for back-compat with any
301 // older manifest shapes that lack f.diskPath.
302 const diskPath = f.diskPath || path.join(this.dir, f.path)
257303 contentHashPaths.set(f.hash, diskPath)
258304 filePaths.set(f.path, diskPath)
259305 }
package.json +1 -1
……@@ -1,6 +1,6 @@
11{
22 "name": "hyper-deploy",
3 "version": "0.4.1",
3 "version": "0.4.2",
44 "description": "P2P release distribution — serve build artifacts to peers over Hyperswarm",
55 "type": "module",
66 "main": "index.js",
test/symlink-serve.test.js +94 -0
……@@ -0,0 +1,94 @@
1// Regression test for the install-stuck-at-symlink bug. Before the fix,
2// DeployServer's `walkDir` used statSync and silently dropped symlink
3// entries from the index. The tree-hashing pass (hyper-overlay/tree)
4// uses lstatSync and DOES record KIND_SYMLINK entries, so the published
5// tree contained content hashes pointing at the link-target string. The
6// server then had no entry for those hashes, content-requests were
7// silently dropped, and the client hung forever on its first symlink.
8//
9// This test stands up a real DeployServer pointing at a directory with
10// a symlink, then directly asks the server to look up the symlink's
11// content. After the fix, the server's _fileIndex contains the entry
12// with contentHash == blake2b(linkTargetBytes), and a diskPath
13// reachable via _onConnection's contentHashPaths map.
14
15import test from 'brittle'
16import fs from 'fs'
17import path from 'path'
18import os from 'os'
19import crypto from 'hypercore-crypto'
20import { DeployServer } from '../index.js'
21
22function mktemp (prefix) {
23 const dir = path.join(os.tmpdir(), 'hd-symlink-test-' + prefix + '-' + Date.now() + '-' + Math.random().toString(36).slice(2, 8))
24 fs.mkdirSync(dir, { recursive: true })
25 return dir
26}
27
28test('DeployServer indexes symlinks as content-by-hash', async (t) => {
29 const dir = mktemp('idx')
30 fs.writeFileSync(path.join(dir, 'regular.txt'), 'plain content')
31 const linkTarget = '/some/absolute/path/to/elsewhere'
32 fs.symlinkSync(linkTarget, path.join(dir, 'a-symlink'))
33
34 const server = new DeployServer(dir)
35 await server.ready()
36
37 const linkEntry = await server._fileIndex.getFile('a-symlink')
38 t.ok(linkEntry, 'symlink indexed in _fileIndex')
39 t.is(linkEntry.size, Buffer.byteLength(linkTarget, 'utf8'),
40 'size equals byte length of link target')
41
42 const expectedHash = crypto.data(Buffer.from(linkTarget, 'utf8')).toString('hex')
43 t.is(linkEntry.contentHash, expectedHash,
44 'contentHash matches hash of link target string')
45
46 const manifestEntry = server.manifest.files.find(f => f.path === 'a-symlink')
47 t.ok(manifestEntry && manifestEntry.diskPath, 'manifest entry has diskPath')
48 const blobBytes = fs.readFileSync(manifestEntry.diskPath)
49 t.is(blobBytes.toString('utf8'), linkTarget,
50 'diskPath blob contains exactly the link target string')
51
52 const regEntry = await server._fileIndex.getFile('regular.txt')
53 t.ok(regEntry, 'regular file still indexed')
54 t.is(regEntry.size, 'plain content'.length, 'regular file size correct')
55
56 await server.close()
57 fs.rmSync(dir, { recursive: true, force: true })
58})
59
60test('DeployServer cleans up symlink blob dir on close', async (t) => {
61 const dir = mktemp('cleanup')
62 fs.symlinkSync('/some/path', path.join(dir, 'link'))
63
64 const server = new DeployServer(dir)
65 await server.ready()
66
67 const blobDir = server._symlinkBlobsDir
68 t.ok(blobDir && fs.existsSync(blobDir), 'blob dir created at ready')
69
70 await server.close()
71 t.absent(fs.existsSync(blobDir), 'blob dir removed on close')
72
73 fs.rmSync(dir, { recursive: true, force: true })
74})
75
76test('walkDir does not follow symlinks (no traversal of link target)', async (t) => {
77 const dir = mktemp('nofollow')
78 const outsideDir = mktemp('outside')
79 fs.writeFileSync(path.join(outsideDir, 'leaked.txt'), 'should not appear')
80 fs.symlinkSync(outsideDir, path.join(dir, 'link-to-outside-dir'))
81
82 const server = new DeployServer(dir)
83 await server.ready()
84
85 const paths = server.manifest.files.map(f => f.path)
86 t.ok(paths.includes('link-to-outside-dir'),
87 'symlink itself indexed as a leaf entry')
88 t.absent(paths.some(p => p.includes('leaked.txt')),
89 'walkDir did not recurse through the symlink')
90
91 await server.close()
92 fs.rmSync(dir, { recursive: true, force: true })
93 fs.rmSync(outsideDir, { recursive: true, force: true })
94})