| … | … | @@ -24,6 +24,7 @@ import os from '#node:os' |
| 24 | 24 | import { spawn } from '#node:child_process' |
| 25 | 25 | |
| 26 | 26 | import { loadCIConfig, pickChannel, deriveVersion, runSteps, collectArtifacts } from './ci-runner.js' |
| 27 | import { publishVersionToDrive } from './vdrive-publish.js' | |
| 27 | 28 | |
| 28 | 29 | export async function buildAndPublish ({ |
| 29 | 30 | sync, |
| … | … | @@ -120,6 +121,27 @@ export async function buildAndPublish ({ |
| 120 | 121 | } |
| 121 | 122 | }) |
| 122 | 123 | |
| 124 | // 8. Optional: publish the same artifacts into the repo's virtual | |
| 125 | // Hyperdrive, giving the repo a durable pear:// link. Non-fatal — | |
| 126 | // a release must not fail because its drive did. | |
| 127 | let vdrive = null | |
| 128 | if (config.vdrive && config.vdrive.enabled && releaseManager.releaseDir) { | |
| 129 | try { | |
| 130 | vdrive = await publishVersionToDrive({ | |
| 131 | releaseDir: releaseManager.releaseDir, | |
| 132 | repo, | |
| 133 | version, | |
| 134 | channel, | |
| 135 | vdrive: config.vdrive, | |
| 136 | onLog | |
| 137 | }) | |
| 138 | if (vdrive.skipped) log(onLog, `[ci] vdrive skipped: ${vdrive.reason}`) | |
| 139 | } catch (err) { | |
| 140 | vdrive = { ok: false, error: err.message } | |
| 141 | log(onLog, `[ci] vdrive failed (release unaffected): ${err.message}`) | |
| 142 | } | |
| 143 | } | |
| 144 | ||
| 123 | 145 | return { |
| 124 | 146 | ok: true, |
| 125 | 147 | version, |
| … | … | @@ -127,7 +149,8 @@ export async function buildAndPublish ({ |
| 127 | 149 | sha, |
| 128 | 150 | steps: runResult.steps, |
| 129 | 151 | buildLog, |
| 130 | manifest: published.manifest | |
| 152 | manifest: published.manifest, | |
| 153 | vdrive | |
| 131 | 154 | } |
| 132 | 155 | } finally { |
| 133 | 156 | // Clean up the temp checkout — artifacts have already been read |
| … | … | @@ -101,9 +101,44 @@ export function loadCIConfig (checkoutDir) { |
| 101 | 101 | } |
| 102 | 102 | } |
| 103 | 103 | |
| 104 | // vdrive: optional. `true` publishes every release into the repo's virtual | |
| 105 | // Hyperdrive; a mapping can restrict it to named channels. Normalised to | |
| 106 | // { enabled, channels } or null so callers don't re-parse. | |
| 107 | // NB: FAILSAFE_SCHEMA means every scalar arrives as a STRING. | |
| 108 | config.vdrive = normaliseVdrive(config.vdrive) | |
| 109 | ||
| 104 | 110 | return config |
| 105 | 111 | } |
| 106 | 112 | |
| 113 | function normaliseVdrive (raw) { | |
| 114 | if (raw === undefined || raw === null) return null | |
| 115 | ||
| 116 | if (typeof raw === 'string') { | |
| 117 | const v = raw.trim().toLowerCase() | |
| 118 | if (v === 'true' || v === 'yes' || v === 'on') return { enabled: true, channels: null } | |
| 119 | if (v === 'false' || v === 'no' || v === 'off') return { enabled: false, channels: null } | |
| 120 | throw new Error('ci.yaml "vdrive" must be true/false or a mapping') | |
| 121 | } | |
| 122 | ||
| 123 | if (typeof raw === 'boolean') return { enabled: raw, channels: null } | |
| 124 | ||
| 125 | if (typeof raw !== 'object' || Array.isArray(raw)) { | |
| 126 | throw new Error('ci.yaml "vdrive" must be true/false or a mapping') | |
| 127 | } | |
| 128 | ||
| 129 | if (raw.channels !== undefined && !Array.isArray(raw.channels)) { | |
| 130 | throw new Error('ci.yaml "vdrive.channels" must be a list') | |
| 131 | } | |
| 132 | for (const c of raw.channels || []) { | |
| 133 | if (typeof c !== 'string' || !c) throw new Error('ci.yaml "vdrive.channels" entries must be strings') | |
| 134 | } | |
| 135 | ||
| 136 | return { | |
| 137 | enabled: true, | |
| 138 | channels: raw.channels ? raw.channels.slice() : null | |
| 139 | } | |
| 140 | } | |
| 141 | ||
| 107 | 142 | /** |
| 108 | 143 | * Pick the channel this ref should publish to based on the ci.yaml rules. |
| 109 | 144 | * |
| … | … | @@ -54,6 +54,9 @@ export class ReleaseManager { |
| 54 | 54 | this._eventLog = opts.eventLog || null |
| 55 | 55 | } |
| 56 | 56 | |
| 57 | /** Release root on disk — the version dirs under it are the vdrive byte store. */ | |
| 58 | get releaseDir () { return this._releaseDir } | |
| 59 | ||
| 57 | 60 | // ── Path helpers ─────────────────────────────────────────── |
| 58 | 61 | _repoDir (repo) { return join(this._releaseDir, repo) } |
| 59 | 62 | _versionsDir (repo) { return join(this._releaseDir, repo, 'versions') } |
| … | … | @@ -0,0 +1,102 @@ |
| 1 | /** | |
| 2 | * Publish a released version into the repo's virtual Hyperdrive. | |
| 3 | * | |
| 4 | * Opt-in per repo via `.pear-git/ci.yaml`: | |
| 5 | * | |
| 6 | * vdrive: true | |
| 7 | * # or | |
| 8 | * vdrive: | |
| 9 | * channels: [production] # only these channels go into the drive | |
| 10 | * | |
| 11 | * ReleaseManager has already written the artifacts as plain files under | |
| 12 | * <releaseDir>/<repo>/versions/<version>/. That directory IS the byte store — | |
| 13 | * the drive keeps a 32-byte pointer per block and reads the bytes back from | |
| 14 | * those files on demand, so a drive costs roughly 0.1% of what it publishes. | |
| 15 | * | |
| 16 | * ONE drive per repo, republished per version — not a drive per version. The | |
| 17 | * pear:// link stays valid across releases and resolves to the newest, which | |
| 18 | * is what makes it usable as a distribution link. Per-version pinning is still | |
| 19 | * available through the verlink form that `pear install` prints. | |
| 20 | * | |
| 21 | * Failures are NON-FATAL: a release must not fail because its drive did. | |
| 22 | */ | |
| 23 | ||
| 24 | import fsp from '#node:fs/promises' | |
| 25 | import path from '#node:path' | |
| 26 | ||
| 27 | export function vdriveStoreDir (releaseDir, repo) { | |
| 28 | return path.join(releaseDir, repo, '.vdrive') | |
| 29 | } | |
| 30 | ||
| 31 | async function walk (dir, base = dir, out = []) { | |
| 32 | for (const entry of await fsp.readdir(dir, { withFileTypes: true })) { | |
| 33 | const full = path.join(dir, entry.name) | |
| 34 | if (entry.isDirectory()) await walk(full, base, out) | |
| 35 | else if (entry.isFile()) out.push('/' + path.relative(base, full).split(path.sep).join('/')) | |
| 36 | } | |
| 37 | return out | |
| 38 | } | |
| 39 | ||
| 40 | /** | |
| 41 | * @param {object} opts | |
| 42 | * @param {string} opts.releaseDir ReleaseManager's release root | |
| 43 | * @param {string} opts.repo | |
| 44 | * @param {string} opts.version the version just published | |
| 45 | * @param {string} [opts.channel] | |
| 46 | * @param {object} opts.vdrive normalised config from loadCIConfig | |
| 47 | * @param {function} [opts.chunk] injectable chunker (tests) | |
| 48 | * @param {function} [opts.onLog] | |
| 49 | */ | |
| 50 | export async function publishVersionToDrive ({ | |
| 51 | releaseDir, | |
| 52 | repo, | |
| 53 | version, | |
| 54 | channel, | |
| 55 | vdrive, | |
| 56 | chunk, | |
| 57 | onLog | |
| 58 | } = {}) { | |
| 59 | if (!vdrive || !vdrive.enabled) return { ok: false, skipped: true, reason: 'not enabled' } | |
| 60 | if (!releaseDir || !repo || !version) { | |
| 61 | throw new Error('publishVersionToDrive requires { releaseDir, repo, version }') | |
| 62 | } | |
| 63 | ||
| 64 | if (vdrive.channels && vdrive.channels.length > 0) { | |
| 65 | if (!channel || !vdrive.channels.includes(channel)) { | |
| 66 | return { | |
| 67 | ok: false, | |
| 68 | skipped: true, | |
| 69 | reason: 'channel not selected for vdrive: ' + (channel || '(none)') | |
| 70 | } | |
| 71 | } | |
| 72 | } | |
| 73 | ||
| 74 | // Lazily resolved so a missing or incompatible module degrades to "no drive" | |
| 75 | // instead of taking the release path down with it. | |
| 76 | let publishTree | |
| 77 | try { | |
| 78 | const mod = await import('hyper-vdrive/publish') | |
| 79 | publishTree = mod.publishTree || (mod.default && mod.default.publishTree) | |
| 80 | if (typeof publishTree !== 'function') throw new Error('publishTree not exported') | |
| 81 | } catch (err) { | |
| 82 | return { ok: false, skipped: true, reason: 'hyper-vdrive unavailable: ' + err.message } | |
| 83 | } | |
| 84 | ||
| 85 | const chunker = chunk || (await import('hyper-overlay/chunker')).chunk | |
| 86 | ||
| 87 | const versionDir = path.join(releaseDir, repo, 'versions', version) | |
| 88 | const names = await walk(versionDir) | |
| 89 | if (names.length === 0) return { ok: false, skipped: true, reason: 'no artifacts to publish' } | |
| 90 | ||
| 91 | const result = await publishTree({ | |
| 92 | storeDir: vdriveStoreDir(releaseDir, repo), | |
| 93 | files: names.map((name) => ({ name, file: path.join(versionDir, name) })), | |
| 94 | chunk: chunker | |
| 95 | }) | |
| 96 | ||
| 97 | if (onLog) { | |
| 98 | onLog(`[ci] vdrive ${repo} ${version}: ${result.files} files, ${result.blocks} blocks -> ${result.link}`) | |
| 99 | } | |
| 100 | ||
| 101 | return { ok: true, ...result } | |
| 102 | } |
| … | … | @@ -1,7 +1,7 @@ |
| 1 | 1 | { |
| 2 | 2 | "name": "pear-git-v3", |
| 3 | 3 | "version": "0.1.225", |
| 4 | "description": "P2P git forge — multi-writer issue/org/release/peer-mirror daemon. Built bottom-up after the failed v2.", | |
| 4 | "description": "P2P git forge \u2014 multi-writer issue/org/release/peer-mirror daemon. Built bottom-up after the failed v2.", | |
| 5 | 5 | "type": "module", |
| 6 | 6 | "main": "index.js", |
| 7 | 7 | "scripts": { |
| … | … | @@ -171,6 +171,7 @@ |
| 171 | 171 | "hyper-rooms": "file:../hyper-rooms", |
| 172 | 172 | "hyper-svc": "file:../hyper-svc", |
| 173 | 173 | "hyper-sync": "file:../hyper-sync", |
| 174 | "hyper-vdrive": "file:../hyper-vdrive", | |
| 174 | 175 | "hyperbee": "^2.27.3", |
| 175 | 176 | "hypercore": "^11.32.0", |
| 176 | 177 | "hypercore-crypto": "^3.0.0", |
| … | … | @@ -0,0 +1,127 @@ |
| 1 | /** | |
| 2 | * vdrive in ci.yaml — config parsing and the publish hook. | |
| 3 | * | |
| 4 | * The hook writes a real drive over a release version dir and returns a | |
| 5 | * pear:// link, without the drive storing any of the bytes. | |
| 6 | */ | |
| 7 | import test from 'brittle' | |
| 8 | import fs from '#node:fs' | |
| 9 | import fsp from '#node:fs/promises' | |
| 10 | import path from '#node:path' | |
| 11 | import os from '#node:os' | |
| 12 | ||
| 13 | import { loadCIConfig } from '../lib/ci-runner.js' | |
| 14 | import { publishVersionToDrive, vdriveStoreDir } from '../lib/vdrive-publish.js' | |
| 15 | ||
| 16 | function tmpdir (name) { | |
| 17 | return fsp.mkdtemp(path.join(os.tmpdir(), 'pgv3-vdrive-' + name + '-')) | |
| 18 | } | |
| 19 | ||
| 20 | function writeConfig (dir, yaml) { | |
| 21 | fs.mkdirSync(path.join(dir, '.pear-git'), { recursive: true }) | |
| 22 | fs.writeFileSync(path.join(dir, '.pear-git', 'ci.yaml'), yaml) | |
| 23 | return dir | |
| 24 | } | |
| 25 | ||
| 26 | const STEPS = 'steps:\n - name: build\n run: "true"\n' | |
| 27 | ||
| 28 | test('ci.yaml vdrive: absent → null', async (t) => { | |
| 29 | const dir = await tmpdir('cfg1') | |
| 30 | writeConfig(dir, STEPS) | |
| 31 | t.is(loadCIConfig(dir).vdrive, null) | |
| 32 | }) | |
| 33 | ||
| 34 | test('ci.yaml vdrive: true → enabled, all channels', async (t) => { | |
| 35 | const dir = await tmpdir('cfg2') | |
| 36 | writeConfig(dir, STEPS + 'vdrive: true\n') | |
| 37 | const cfg = loadCIConfig(dir) | |
| 38 | t.is(cfg.vdrive.enabled, true) | |
| 39 | t.is(cfg.vdrive.channels, null) | |
| 40 | }) | |
| 41 | ||
| 42 | test('ci.yaml vdrive: false → disabled', async (t) => { | |
| 43 | const dir = await tmpdir('cfg3') | |
| 44 | writeConfig(dir, STEPS + 'vdrive: false\n') | |
| 45 | t.is(loadCIConfig(dir).vdrive.enabled, false) | |
| 46 | }) | |
| 47 | ||
| 48 | test('ci.yaml vdrive: mapping with channels', async (t) => { | |
| 49 | const dir = await tmpdir('cfg4') | |
| 50 | writeConfig(dir, STEPS + 'vdrive:\n channels: [production]\n') | |
| 51 | const cfg = loadCIConfig(dir) | |
| 52 | t.is(cfg.vdrive.enabled, true) | |
| 53 | t.alike(cfg.vdrive.channels, ['production']) | |
| 54 | }) | |
| 55 | ||
| 56 | test('ci.yaml vdrive: garbage is rejected', async (t) => { | |
| 57 | const dir = await tmpdir('cfg5') | |
| 58 | writeConfig(dir, STEPS + 'vdrive: maybe\n') | |
| 59 | t.exception(() => loadCIConfig(dir), /vdrive/) | |
| 60 | ||
| 61 | const dir2 = await tmpdir('cfg6') | |
| 62 | writeConfig(dir2, STEPS + 'vdrive:\n channels: notalist\n') | |
| 63 | t.exception(() => loadCIConfig(dir2), /channels/) | |
| 64 | }) | |
| 65 | ||
| 66 | test('hook is inert when not enabled', async (t) => { | |
| 67 | const res = await publishVersionToDrive({ vdrive: null }) | |
| 68 | t.is(res.ok, false) | |
| 69 | t.is(res.skipped, true) | |
| 70 | }) | |
| 71 | ||
| 72 | test('hook skips a channel that is not selected', async (t) => { | |
| 73 | const res = await publishVersionToDrive({ | |
| 74 | releaseDir: '/tmp', | |
| 75 | repo: 'x', | |
| 76 | version: '1', | |
| 77 | channel: 'experimental', | |
| 78 | vdrive: { enabled: true, channels: ['production'] } | |
| 79 | }) | |
| 80 | t.is(res.skipped, true) | |
| 81 | t.ok(res.reason.includes('channel not selected')) | |
| 82 | }) | |
| 83 | ||
| 84 | test('hook publishes a version dir into a drive holding no blob bytes', async (t) => { | |
| 85 | const releaseDir = await tmpdir('rel') | |
| 86 | const repo = 'demo' | |
| 87 | const version = '1.0.0' | |
| 88 | const versionDir = path.join(releaseDir, repo, 'versions', version) | |
| 89 | await fsp.mkdir(path.join(versionDir, 'by-arch', 'linux-x64', 'app'), { recursive: true }) | |
| 90 | ||
| 91 | const payload = Buffer.alloc(2 * 1024 * 1024, 7) | |
| 92 | await fsp.writeFile(path.join(versionDir, 'by-arch', 'linux-x64', 'app', 'demo'), payload) | |
| 93 | await fsp.writeFile(path.join(versionDir, 'package.json'), JSON.stringify({ name: repo })) | |
| 94 | await fsp.writeFile(path.join(versionDir, 'empty.stamp'), '') | |
| 95 | ||
| 96 | const { chunk } = await import('hyper-overlay/chunker') | |
| 97 | const res = await publishVersionToDrive({ | |
| 98 | releaseDir, | |
| 99 | repo, | |
| 100 | version, | |
| 101 | channel: 'production', | |
| 102 | vdrive: { enabled: true, channels: null }, | |
| 103 | chunk | |
| 104 | }) | |
| 105 | ||
| 106 | t.is(res.ok, true, 'published') | |
| 107 | t.ok(res.link.startsWith('pear://'), 'has a pear link: ' + res.link) | |
| 108 | t.is(res.files, 3, 'all three files, including the 0-byte one') | |
| 109 | t.ok(res.pointerBlocks > 0, 'blocks were redirected, not stored') | |
| 110 | ||
| 111 | // the drive must be far smaller than the bytes it publishes | |
| 112 | const storeDir = vdriveStoreDir(releaseDir, repo) | |
| 113 | const size = await dirSize(storeDir) | |
| 114 | t.ok(size < payload.length / 4, `drive store ${size}B << payload ${payload.length}B`) | |
| 115 | ||
| 116 | await fsp.rm(releaseDir, { recursive: true, force: true }) | |
| 117 | }) | |
| 118 | ||
| 119 | async function dirSize (dir) { | |
| 120 | let total = 0 | |
| 121 | for (const e of await fsp.readdir(dir, { withFileTypes: true })) { | |
| 122 | const full = path.join(dir, e.name) | |
| 123 | if (e.isDirectory()) total += await dirSize(full) | |
| 124 | else total += (await fsp.stat(full)).size | |
| 125 | } | |
| 126 | return total | |
| 127 | } |