OverviewFilesCommitsIssuesDocsReleasesPeers

pgh commits / eb20cf8

0.1.158: store gc keeps each app's newest N releases (keepReleases)
The 14-day age gate never collected an app that publishes several times a day: tabertron's store held 75 copies of hyper-tps's ~600 MB hypertps.pck (~44 GiB), all younger than 14 days. A release entry names its whole closure (root tree + dep trees + per-arch prebuild trees), so gc({ keepReleases, releases }) keeps every tree of each app's newest N releases and sweeps trees only older releases use, whatever their age. Installed/pinned trees are kept; trees in no release keep the age gate (a publish not yet registered). `pgh store gc --keep-releases N` asks pgh-sidecar to run it (store-gc RPC): the full release list carries every dep closure and overflows the 16 MiB UDS frame, so only the registry holder can read it. Falls back to opening the registry locally when no sidecar runs. test/store-gc-keep-releases.test.js (red: releaseTreeHashes not exported). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
eb20cf81e63b64fb4db5db9754fbeeb595e6cb64
by plankatron <fleeky@gmail.com> · 2026-10-11T21:37:41.000Z

Changed files (5) +212 -5

lib/commands/releases.js +6 -1
……@@ -28,7 +28,7 @@ const SIDECAR_SOCK = process.env.PGH_SIDECAR_SOCK || path.join(homedir(), '.pgh'
2828
2929// Prefer pgh-sidecar (which owns the registry lock). Fall back to
3030// opening the corestore directly when the sidecar isn't running.
31async function withRegistry (fn) {
31export async function withRegistry (fn) {
3232 if (isAbstract(SIDECAR_SOCK) || fs.existsSync(SIDECAR_SOCK)) {
3333 let client = null
3434 try {
……@@ -73,6 +73,11 @@ function sidecarRegistryAdapter (client) {
7373 },
7474 async remove (name, version) {
7575 return client.call('remove-release', { name, version })
76 },
77 // Not a registry method: the sidecar runs release-retention gc itself,
78 // because `list()` of every release overflows the UDS frame.
79 async storeGc (opts) {
80 return client.call('store-gc', opts)
7681 }
7782 }
7883}
lib/commands/store.js +35 -1
……@@ -1,6 +1,7 @@
11/** pgh store list|size|gc|pin|unpin — manage the tree store */
22
33import { TreeStore } from '../store.js'
4import { withRegistry } from './releases.js'
45
56// Default age-gate for `pgh store gc`. 14 days matches the retention
67// policy adopted on the retention-policy branch — short enough to keep
……@@ -16,6 +17,9 @@ export const usage = `pgh store — manage the content-addressed tree store
1617 pgh store gc [--dry-run] [--max-age <d>] Remove unreferenced, unpinned trees;
1718 --max-age <d> grace period in days
1819 (default: ${DEFAULT_GC_MAX_AGE_DAYS}). 0 = no grace.
20 [--keep-releases <n>] Also keep each app's newest n releases
21 and drop trees only older releases use,
22 whatever their age.
1923 pgh store pin <hash> Mark tree as ungcable
2024 pgh store unpin <hash> Remove pin
2125 pgh store show <hash> Show details for one tree
……@@ -94,12 +98,42 @@ export async function run (client, args) {
9498 }
9599 maxAgeDays = parsed
96100 }
97 const result = store.gc({ dryRun, maxAgeDays })
101 const keepRaw = flagValue(args, '--keep-releases')
102 let keepReleases
103 let releases
104 if (keepRaw !== undefined) {
105 keepReleases = parseInt(keepRaw, 10)
106 if (!Number.isFinite(keepReleases) || keepReleases < 1) {
107 throw new Error('--keep-releases requires a positive integer')
108 }
109 // With the sidecar up, IT runs the gc: it holds the registry, and the
110 // full release list (every entry's dep closure) overflows the UDS frame.
111 const remote = await withRegistry(async (reg) => {
112 if (typeof reg.storeGc === 'function') return reg.storeGc({ dryRun, keepReleases })
113 releases = await reg.list()
114 return null
115 })
116 if (remote) {
117 return {
118 candidates: remote.candidates.length,
119 removed: remote.removed.length,
120 kept: remote.kept,
121 blobsRemoved: remote.blobsRemoved,
122 maxAgeDays: 14,
123 keepReleases,
124 dryRun,
125 by: 'pgh-sidecar',
126 removedHashes: dryRun ? remote.candidates : remote.removed
127 }
128 }
129 }
130 const result = store.gc({ dryRun, maxAgeDays, keepReleases, releases })
98131 return {
99132 candidates: result.candidates.length,
100133 removed: result.removed.length,
101134 kept: result.kept.length,
102135 maxAgeDays,
136 keepReleases: keepReleases || null,
103137 dryRun,
104138 removedHashes: dryRun ? result.candidates : result.removed
105139 }
lib/store.js +54 -2
……@@ -510,6 +510,13 @@ export class TreeStore {
510510 * REPORTED in the return value, but only removed when this is set and
511511 * dryRun is off). Removing them frees their now-redundant blob hardlinks
512512 * in the same pass.
513 * @param {number} [opts.keepReleases] - with `opts.releases`: keep every tree
514 * of each app's newest N releases, and sweep the trees of its older
515 * releases REGARDLESS of age. The age gate alone never collects an app
516 * that publishes several times a day (2026-10-11: 75 copies of a ~600 MB
517 * hypertps.pck, all younger than 14 days). Trees in no release keep the
518 * age gate — it is what protects a publish not yet in the registry.
519 * @param {object[]} [opts.releases] - release entries (ReleaseRegistry.list())
513520 * @returns {{ candidates, removed, kept, blobsRemoved, orphanApps, appsRemoved }}
514521 */
515522 gc (opts = {}) {
……@@ -524,17 +531,22 @@ export class TreeStore {
524531 ? Date.now() - (opts.maxAgeDays * 24 * 60 * 60 * 1000)
525532 : null
526533
534 const { current, superseded } = (typeof opts.keepReleases === 'number' && opts.keepReleases > 0)
535 ? splitReleaseTrees(opts.releases || [], opts.keepReleases)
536 : { current: new Set(), superseded: new Set() }
537
527538 for (const hash of allHashes) {
528539 const referenced = (refs[hash] && refs[hash].length > 0)
529540 const pinned = this.isPinned(hash)
530541
531 if (referenced || pinned) {
542 if (referenced || pinned || current.has(hash)) {
532543 kept.push(hash)
533544 continue
534545 }
535546
536547 // Age gate — if a cutoff is set, only sweep trees pulled before it.
537 if (cutoff !== null) {
548 // A tree only an older release uses is past its retention whatever its age.
549 if (cutoff !== null && !superseded.has(hash)) {
538550 const meta = this.getMeta(hash)
539551 const pulledAt = (meta && typeof meta.pulledAt === 'number') ? meta.pulledAt : 0
540552 if (pulledAt > cutoff) {
……@@ -751,6 +763,46 @@ export class TreeStore {
751763 }
752764}
753765
766// ── Release retention ───────────────────────────────────────────
767
768/**
769 * Every tree a release entry names: the app root, each dep's tree, and each
770 * dep's per-arch prebuild trees (publish.js depEntry).
771 */
772export function releaseTreeHashes (entry) {
773 const out = new Set()
774 if (!entry) return out
775 if (entry.treeHash) out.add(entry.treeHash)
776 for (const d of entry.deps || []) {
777 if (d && d.tree) out.add(d.tree)
778 if (d && d.prebuilds) for (const h of Object.values(d.prebuilds)) if (h) out.add(h)
779 }
780 return out
781}
782
783/**
784 * Per app, newest `keep` releases (by publishedAt) → `current`; trees ONLY in
785 * older releases → `superseded`. A tree shared with a current release is current.
786 */
787function splitReleaseTrees (releases, keep) {
788 const byApp = new Map()
789 for (const e of releases) {
790 if (!e || !e.name) continue
791 if (!byApp.has(e.name)) byApp.set(e.name, [])
792 byApp.get(e.name).push(e)
793 }
794 const current = new Set()
795 const older = new Set()
796 for (const list of byApp.values()) {
797 list.sort((a, b) => (a.publishedAt === b.publishedAt ? 0 : (a.publishedAt > b.publishedAt ? -1 : 1)))
798 list.forEach((e, i) => {
799 for (const h of releaseTreeHashes(e)) (i < keep ? current : older).add(h)
800 })
801 }
802 const superseded = new Set([...older].filter(h => !current.has(h)))
803 return { current, superseded }
804}
805
754806// ── Content-address verification ────────────────────────────────
755807
756808/**
package.json +1 -1
……@@ -1,6 +1,6 @@
11{
22 "name": "pgh",
3 "version": "0.1.157",
3 "version": "0.1.158",
44 "description": "pgh \u2014 P2P content-addressed package manager and app launcher",
55 "type": "module",
66 "main": "index.js",
test/store-gc-keep-releases.test.js +116 -0
……@@ -0,0 +1,116 @@
1/**
2 * TreeStore.gc({ keepReleases, releases }) — retention by RELEASE COUNT.
3 *
4 * Why: the age gate alone (14 days) never collected anything from an app that
5 * publishes several times a day. On tabertron 2026-10-11 the store held 75
6 * copies of hyper-tps's ~600 MB hypertps.pck (~44 GiB), every one younger than
7 * 14 days. A release entry names its whole closure (root tree + each dep's
8 * tree + each per-arch prebuild tree), so "keep the newest N releases per app"
9 * is an exact set of tree hashes.
10 */
11
12import test from 'brittle'
13import fs from '#node:fs'
14import path from '#node:path'
15import os from '#node:os'
16import { TreeStore, releaseTreeHashes } from '../lib/store.js'
17
18const DAY = 24 * 60 * 60 * 1000
19
20function mkstore () {
21 const root = fs.mkdtempSync(path.join(os.tmpdir(), 'pgh-keep-releases-'))
22 const store = new TreeStore({ root })
23 store.init()
24 return { root, store }
25}
26
27function addTree (store, hash, pulledAt = Date.now()) {
28 fs.mkdirSync(path.join(store.dependenciesDir, hash))
29 store._writeMeta(hash, { pulledAt, schema: TreeStore.BLOB_LAYER_SCHEMA })
30}
31
32const h = (c) => c.repeat(64)
33
34function release (name, version, publishedAt, treeHash, deps) {
35 return { name, version, publishedAt, treeHash, deps }
36}
37
38test('releaseTreeHashes — root + dep trees + per-arch prebuild trees', (t) => {
39 const e = release('app', '1.0.0', '2026-10-01T00:00:00.000Z', h('1'), [
40 { name: 'a', version: '1', tree: h('2') },
41 { name: 'b', version: '1', tree: h('3'), prebuilds: { 'linux-x64': h('4'), 'win32-x64': h('5') } },
42 { name: 'broken', version: '1', tree: null, error: 'unreadable' }
43 ])
44 t.alike([...releaseTreeHashes(e)].sort(), [h('1'), h('2'), h('3'), h('4'), h('5')])
45})
46
47test('gc keepReleases — superseded release trees go even when fresh; newest N and shared trees stay', (t) => {
48 const { root, store } = mkstore()
49 // app: v1 (oldest), v2, v3 (newest). Shared dep S is in all three.
50 // v1-only: root R1, dep D1, prebuild P1. v2: R2. v3: R3, prebuild P3.
51 for (const c of ['a', 'b', 'c', 'd', 'e', 'f', '9']) addTree(store, h(c))
52 const releases = [
53 release('app', '0.1.1', '2026-10-01T00:00:00.000Z', h('a'), [
54 { name: 'shared', version: '1', tree: h('9') },
55 { name: 'old', version: '1', tree: h('b'), prebuilds: { 'linux-x64': h('c') } }
56 ]),
57 release('app', '0.1.3', '2026-10-03T00:00:00.000Z', h('e'), [
58 { name: 'shared', version: '1', tree: h('9'), prebuilds: { 'linux-x64': h('f') } }
59 ]),
60 release('app', '0.1.2', '2026-10-02T00:00:00.000Z', h('d'), [
61 { name: 'shared', version: '1', tree: h('9') }
62 ])
63 ]
64
65 const r = store.gc({ maxAgeDays: 14, keepReleases: 2, releases })
66 t.alike(r.removed.sort(), [h('a'), h('b'), h('c')], 'only the oldest release\'s own trees are removed')
67 for (const c of ['d', 'e', 'f', '9']) t.ok(store.hasTree(h(c)), 'kept ' + c)
68 fs.rmSync(root, { recursive: true, force: true })
69})
70
71test('gc keepReleases — installed or pinned trees of a superseded release are kept', (t) => {
72 const { root, store } = mkstore()
73 for (const c of ['a', 'b', 'c', 'd']) addTree(store, h(c))
74 const releases = [
75 release('app', '1', '2026-10-01T00:00:00.000Z', h('a'), [{ name: 'x', version: '1', tree: h('b') }]),
76 release('app', '2', '2026-10-02T00:00:00.000Z', h('c'), []),
77 release('app', '3', '2026-10-03T00:00:00.000Z', h('d'), [])
78 ]
79 store.addRef(h('a'), 'app') // v1 is what is installed here
80 store.pin(h('b')) // an operator's rollback pin
81 const r = store.gc({ maxAgeDays: 14, keepReleases: 2, releases })
82 t.alike(r.removed, [], 'nothing removed')
83 fs.rmSync(root, { recursive: true, force: true })
84})
85
86test('gc keepReleases — trees in no release keep the age gate', (t) => {
87 const { root, store } = mkstore()
88 addTree(store, h('a')) // fresh, in no release (a publish not yet registered)
89 addTree(store, h('b'), Date.now() - 30 * DAY) // old, in no release
90 const r = store.gc({ maxAgeDays: 14, keepReleases: 2, releases: [] })
91 t.alike(r.removed, [h('b')], 'only the old unreleased tree goes')
92 fs.rmSync(root, { recursive: true, force: true })
93})
94
95test('gc keepReleases — apps are counted separately', (t) => {
96 const { root, store } = mkstore()
97 for (const c of ['a', 'b', 'c', 'd']) addTree(store, h(c))
98 const releases = [
99 release('one', '1', '2026-10-01T00:00:00.000Z', h('a'), []),
100 release('one', '2', '2026-10-02T00:00:00.000Z', h('b'), []),
101 release('two', '1', '2026-09-01T00:00:00.000Z', h('c'), []),
102 release('two', '2', '2026-09-02T00:00:00.000Z', h('d'), [])
103 ]
104 const r = store.gc({ maxAgeDays: 14, keepReleases: 2, releases })
105 t.alike(r.removed, [], 'each app keeps its own two')
106 fs.rmSync(root, { recursive: true, force: true })
107})
108
109test('gc without keepReleases — unchanged: fresh release trees are kept', (t) => {
110 const { root, store } = mkstore()
111 for (const c of ['a', 'b', 'c']) addTree(store, h(c))
112 const releases = ['a', 'b', 'c'].map((c, i) => release('app', String(i), '2026-10-0' + (i + 1) + 'T00:00:00.000Z', h(c), []))
113 const r = store.gc({ maxAgeDays: 14, releases })
114 t.alike(r.removed, [], 'age gate alone keeps all three')
115 fs.rmSync(root, { recursive: true, force: true })
116})