OverviewFilesCommitsIssuesDocsReleasesPeers

pgh commits / 041bc84

0.1.125: serve cross-arch bootstraps — /bootstrap 404s for every arm64 VPS
hostLabel() reads os.platform()/os.arch(), so reconcileBootstrapArtifacts can only ever TAR the arch it runs on. An all-x64 fleet therefore never produces pgh-bootstrap-bare-linux-arm64.tar.gz, and install.sh -- which fetches exactly that name from a flat base -- gets the "bootstrap artifact not found" page on any arm64 box (Graviton / Ampere / Hetzner ARM). The bytes were already on disk. Bootstrap apps vendor prebuilt cross-arch tarballs at node_modules/<pkg>/dist/<slug>-bootstrap-bare-<label>.tar.gz; they were just never wired into the served dir. Promote them instead: - vendoredBootstraps() finds them per app tree - a promotion pass hardlinks any arch we could not build locally into <store>/bootstrap/ (same filesystem => zero extra bytes) - local tar always wins for the host arch - promoted names join wantTarballs, or the prune pass would delete them on the very next reconcile - re-links when the vendored inode changes, so an upstream bump is not served stale forever The promotion pass deliberately walks ALL app trees, not just pgh.bootstrap ones. Measured on tabertron: pgh-bootstrap-bare-linux-arm64.tar.gz -> pgh-sidecar tree (bootstrap app) pgh-sidecar-bootstrap-bare-linux-arm64.tar.gz -> pear-git-v3 tree (NOT one) install.sh installs pgh AND pgh-sidecar under `set -e`, so scanning only bootstrap apps would serve pgh for arm64 and still abort at the sidecar step -- a half-fixed install is still a broken one. Does not touch payload size: the nested tarballs stay in the sidecar tree for now (302.6 MiB of its 775.3 MiB) because they are currently the ONLY cross-arch seed. Excluding them from the tar has to wait until each arch is published as its own artifact, or cross-arch seeding dies one generation later. test: 14/14 in bootstrap-artifacts.test.js, full suite 272/272 (823 asserts) under bare. Node-only runs fail in bare-path (pre-existing, verified by stash). Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
041bc8475e45f1d7bedf4f5201fba6b809728584
by plankatron <fleeky@gmail.com> · 2026-07-29T23:39:31.000Z

Changed files (3) +159 -1

lib/bootstrap-artifacts.js +60 -0
……@@ -68,6 +68,32 @@ function slugify (name) {
6868 return String(name || '').replace(/[^a-z0-9-]/gi, '-')
6969}
7070
71/**
72 * Cross-arch seeding. A machine can only TAR its OWN arch — hostLabel() reads
73 * os.platform()/os.arch() — so an all-x64 fleet can never produce the
74 * linux-arm64 tarball install.sh asks for, and /bootstrap 404s for every arm64
75 * VPS. Bootstrap apps already vendor prebuilt cross-arch tarballs at
76 * node_modules/<pkg>/dist/<slug>-bootstrap-bare-<label>.tar.gz; this finds them
77 * so reconcile can promote them into the served dir. Local tar always wins for
78 * the host arch — these only ever fill in the arches we cannot build here.
79 */
80function vendoredBootstraps (appDir) {
81 const out = []
82 const nm = path.join(appDir, 'node_modules')
83 let pkgs = []
84 try { pkgs = fs.readdirSync(nm) } catch { return out }
85 for (const pkg of pkgs) {
86 if (pkg.startsWith('.')) continue
87 const dist = path.join(nm, pkg, 'dist')
88 let files = []
89 try { files = fs.readdirSync(dist) } catch { continue }
90 for (const f of files) {
91 if (/-bootstrap-bare-.+\.tar\.gz$/.test(f)) out.push({ name: f, src: path.join(dist, f) })
92 }
93 }
94 return out
95}
96
7197/**
7298 * Reconcile <store>/bootstrap from the local app trees. Idempotent.
7399 *
……@@ -142,6 +168,40 @@ export async function reconcileBootstrapArtifacts (store, o = {}) {
142168 }
143169 }
144170
171 // ── Cross-arch promotion pass ───────────────────────────────────
172 // Deliberately over ALL app trees, not just pgh.bootstrap ones: a vendored
173 // bootstrap tarball is arch-portable BYTES, and it does not live in the tree
174 // of the app it bootstraps. Measured on tabertron:
175 // pgh-bootstrap-bare-linux-arm64.tar.gz → pgh-sidecar tree (bootstrap app)
176 // pgh-sidecar-bootstrap-bare-linux-arm64.tar.gz → pear-git-v3 tree (NOT a bootstrap app)
177 // install.sh fetches BOTH and runs under `set -e`, so scanning only bootstrap
178 // apps would serve pgh for arm64 and still fail at the sidecar step.
179 // Anything already in wantTarballs was tarred locally for the host arch and
180 // wins — promotion only ever fills arches we cannot build here.
181 for (const name of appNames) {
182 if (name.startsWith('.') || /\.new-\d+$/.test(name)) continue
183 for (const v of vendoredBootstraps(path.join(appsDir, name))) {
184 if (wantTarballs.has(v.name)) continue
185 wantTarballs.add(v.name)
186 nextState[v.name] = 'vendored'
187 const vdest = path.join(dir, v.name)
188 // Same-inode means it's already the promoted hardlink — nothing to do.
189 // A DIFFERENT inode means the vendored artifact was updated underneath us,
190 // so re-link rather than serving stale bytes forever.
191 let linked = false
192 try { linked = fs.statSync(vdest).ino === fs.statSync(v.src).ino } catch {}
193 if (linked) { out.kept.push(v.name); continue }
194 try { fs.rmSync(vdest, { force: true }) } catch {}
195 try {
196 fs.linkSync(v.src, vdest) // hardlink: zero extra bytes
197 } catch {
198 try { fs.copyFileSync(v.src, vdest) } catch { continue }
199 }
200 out.tarred.push(v.name)
201 if (log) log('[bootstrap] promoted vendored ' + v.name + ' (cross-arch, from ' + name + ')')
202 }
203 }
204
145205 // install.sh — a symlink (not a copy) so it tracks the current tree.
146206 const installLink = path.join(dir, 'install.sh')
147207 if (installShTarget) {
package.json +1 -1
……@@ -1,6 +1,6 @@
11{
22 "name": "pgh",
3 "version": "0.1.124",
3 "version": "0.1.125",
44 "description": "pgh — P2P content-addressed package manager and app launcher",
55 "type": "module",
66 "main": "index.js",
test/bootstrap-artifacts.test.js +98 -0
……@@ -153,6 +153,104 @@ test('default tarTree (real async spawn) produces a valid tarball without blocki
153153 rmSync(root, { recursive: true, force: true })
154154})
155155
156// Cross-arch: hostLabel() only ever names the LOCAL arch, so an all-x64 fleet
157// can never tar linux-arm64 and /bootstrap 404s for arm64 VPSes. Bootstrap apps
158// vendor prebuilt cross-arch tarballs under node_modules/<pkg>/dist/ — reconcile
159// promotes those into the served dir.
160
161// Drop a vendored cross-arch tarball into an app tree's node_modules/<pkg>/dist/.
162function mkVendored (appDir, pkg, tarName, body = 'VENDORED') {
163 const dist = join(appDir, 'node_modules', pkg, 'dist')
164 mkdirSync(dist, { recursive: true })
165 writeFileSync(join(dist, tarName), body)
166 return join(dist, tarName)
167}
168
169test('reconcileBootstrapArtifacts promotes a vendored cross-arch tarball into the served dir', async (t) => {
170 const root = mkdtempSync(join(tmpdir(), 'pgh-ba-xarch-'))
171 const store = fakeStore(root); store.init()
172 const appDir = mkAppTree(store.appsDir, 'pgh', { label: 'linux-x64', treeHash: 'h1', installSh: true })
173 const src = mkVendored(appDir, 'pgh', 'pgh-bootstrap-bare-linux-arm64.tar.gz', 'ARM64BYTES')
174
175 const res = await reconcileBootstrapArtifacts(store, { tarTree: fakeTar([]), label: 'linux-x64' })
176
177 const dir = bootstrapDir(store)
178 const arm = join(dir, 'pgh-bootstrap-bare-linux-arm64.tar.gz')
179 t.ok(existsSync(arm), 'arm64 tarball now served')
180 t.is(readFileSync(arm, 'utf8'), 'ARM64BYTES', 'promoted the vendored bytes')
181 t.ok(res.tarred.includes('pgh-bootstrap-bare-linux-arm64.tar.gz'), 'reported as produced')
182 t.ok(existsSync(join(dir, 'pgh-bootstrap-bare-linux-x64.tar.gz')), 'host-arch tarball still tarred locally')
183 // hardlink, not copy — same inode means serving arm64 costs no extra bytes
184 t.is(lstatSync(arm).ino, lstatSync(src).ino, 'hardlinked to the vendored file (zero extra disk)')
185})
186
187test('a promoted cross-arch tarball survives the prune pass', async (t) => {
188 const root = mkdtempSync(join(tmpdir(), 'pgh-ba-xarch-p-'))
189 const store = fakeStore(root); store.init()
190 const appDir = mkAppTree(store.appsDir, 'pgh', { label: 'linux-x64', treeHash: 'h1', installSh: true })
191 mkVendored(appDir, 'pgh', 'pgh-bootstrap-bare-linux-arm64.tar.gz')
192
193 const dir = bootstrapDir(store)
194 await reconcileBootstrapArtifacts(store, { tarTree: fakeTar([]), label: 'linux-x64' })
195 t.ok(existsSync(join(dir, 'pgh-bootstrap-bare-linux-arm64.tar.gz')), 'present after first reconcile')
196
197 // Second pass is where a non-wanted artifact would get pruned.
198 const res = await reconcileBootstrapArtifacts(store, { tarTree: fakeTar([]), label: 'linux-x64' })
199 t.absent(res.pruned.includes('pgh-bootstrap-bare-linux-arm64.tar.gz'), 'not pruned')
200 t.ok(existsSync(join(dir, 'pgh-bootstrap-bare-linux-arm64.tar.gz')), 'still served after re-reconcile')
201})
202
203test('local tar wins for the host arch — a vendored same-label tarball never overwrites it', async (t) => {
204 const root = mkdtempSync(join(tmpdir(), 'pgh-ba-xarch-w-'))
205 const store = fakeStore(root); store.init()
206 const appDir = mkAppTree(store.appsDir, 'pgh', { label: 'linux-x64', treeHash: 'h1', installSh: true })
207 // a stale vendored tarball for OUR arch — must not shadow the fresh local tar
208 mkVendored(appDir, 'pgh', 'pgh-bootstrap-bare-linux-x64.tar.gz', 'STALE')
209
210 await reconcileBootstrapArtifacts(store, { tarTree: fakeTar([]), label: 'linux-x64' })
211
212 const hostTb = join(bootstrapDir(store), 'pgh-bootstrap-bare-linux-x64.tar.gz')
213 t.not(readFileSync(hostTb, 'utf8'), 'STALE', 'host-arch tarball came from local tar, not the vendored copy')
214})
215
216// Measured on tabertron: pgh-sidecar-bootstrap-bare-linux-arm64.tar.gz lives ONLY
217// in the pear-git-v3 tree, which is not a pgh.bootstrap app. install.sh fetches
218// both pgh and pgh-sidecar under `set -e`, so missing the sidecar's arm64 tarball
219// still breaks every arm64 install.
220test('promotes a vendored tarball carried by a NON-bootstrap app tree', async (t) => {
221 const root = mkdtempSync(join(tmpdir(), 'pgh-ba-xarch-nb-'))
222 const store = fakeStore(root); store.init()
223 mkAppTree(store.appsDir, 'pgh', { label: 'linux-x64', treeHash: 'h1', installSh: true })
224 // pear-git-v3 is NOT a bootstrap app, but it carries the sidecar's arm64 bytes
225 const pgv3 = mkAppTree(store.appsDir, 'pear-git-v3', { bootstrap: false, treeHash: 'h2' })
226 mkVendored(pgv3, 'pgh-sidecar', 'pgh-sidecar-bootstrap-bare-linux-arm64.tar.gz', 'SIDECAR-ARM64')
227
228 const res = await reconcileBootstrapArtifacts(store, { tarTree: fakeTar([]), label: 'linux-x64' })
229
230 const sc = join(bootstrapDir(store), 'pgh-sidecar-bootstrap-bare-linux-arm64.tar.gz')
231 t.ok(existsSync(sc), 'sidecar arm64 tarball served despite carrier not being a bootstrap app')
232 t.is(readFileSync(sc, 'utf8'), 'SIDECAR-ARM64', 'promoted the right bytes')
233 t.ok(res.tarred.includes('pgh-sidecar-bootstrap-bare-linux-arm64.tar.gz'), 'reported as produced')
234 t.absent(existsSync(join(bootstrapDir(store), 'pear-git-v3-bootstrap-bare-linux-x64.tar.gz')),
235 'non-bootstrap carrier is still not tarred as a bootstrap app itself')
236})
237
238test('re-promotes when the vendored artifact is replaced underneath us', async (t) => {
239 const root = mkdtempSync(join(tmpdir(), 'pgh-ba-xarch-up-'))
240 const store = fakeStore(root); store.init()
241 const appDir = mkAppTree(store.appsDir, 'pgh', { label: 'linux-x64', treeHash: 'h1', installSh: true })
242 mkVendored(appDir, 'pgh', 'pgh-bootstrap-bare-linux-arm64.tar.gz', 'V1')
243
244 const dir = bootstrapDir(store)
245 await reconcileBootstrapArtifacts(store, { tarTree: fakeTar([]), label: 'linux-x64' })
246 t.is(readFileSync(join(dir, 'pgh-bootstrap-bare-linux-arm64.tar.gz'), 'utf8'), 'V1', 'v1 served')
247
248 // new upstream version replaces the vendored file (new inode)
249 mkVendored(appDir, 'pgh', 'pgh-bootstrap-bare-linux-arm64.tar.gz', 'V2')
250 await reconcileBootstrapArtifacts(store, { tarTree: fakeTar([]), label: 'linux-x64' })
251 t.is(readFileSync(join(dir, 'pgh-bootstrap-bare-linux-arm64.tar.gz'), 'utf8'), 'V2', 're-promoted, not stale')
252})
253
156254test('hostLabel returns os-arch', (t) => {
157255 t.ok(/^[a-z0-9]+-[a-z0-9]+$/.test(hostLabel()), 'looks like <os>-<arch>')
158256})