| … | … | @@ -0,0 +1,131 @@ |
| 1 | /** |
| 2 | * locations.json must travel with the release it describes (org #146). |
| 3 | * |
| 4 | * A vdrive stores `index -> contentHash`. Serving needs the other half, |
| 5 | * `contentHash -> (file, offset, length)`, and rebuilding that by re-chunking |
| 6 | * every published file is O(all bytes) on every boot. So it is written once at |
| 7 | * publish time as `locations.json`. |
| 8 | * |
| 9 | * It lived only at `<releaseDir>/<repo>/.vdrive/locations.json` — OUTSIDE the |
| 10 | * version dir, so it is not part of what replicates with a release. A mirror |
| 11 | * therefore holds the artifacts and the key (which rides in manifest.json) but |
| 12 | * cannot resolve a single block, and has to pay for a second full copy of every |
| 13 | * byte to seed a drive it could otherwise serve for ~281KB. |
| 14 | * |
| 15 | * The index is CUMULATIVE: publishTree loads the prior index, adds the new |
| 16 | * version's chunks, and saves the union. So the copy written into a version dir |
| 17 | * is a snapshot of the whole drive at that point, not a per-version delta — |
| 18 | * which is what makes the NEWEST version's copy sufficient on its own. |
| 19 | */ |
| 20 | import test from 'brittle' |
| 21 | import fsp from '#node:fs/promises' |
| 22 | import path from '#node:path' |
| 23 | import os from '#node:os' |
| 24 | |
| 25 | import { publishVersionToDrive, vdriveStoreDir } from '../lib/vdrive-publish.js' |
| 26 | |
| 27 | function tmpdir (name) { |
| 28 | return fsp.mkdtemp(path.join(os.tmpdir(), 'pgv3-vdloc-' + name + '-')) |
| 29 | } |
| 30 | |
| 31 | async function publish (releaseDir, repo, version, files) { |
| 32 | const versionDir = path.join(releaseDir, repo, 'versions', version) |
| 33 | await fsp.mkdir(versionDir, { recursive: true }) |
| 34 | for (const [name, data] of Object.entries(files)) { |
| 35 | await fsp.writeFile(path.join(versionDir, name), data) |
| 36 | } |
| 37 | const { chunk } = await import('hyper-overlay/chunker') |
| 38 | const res = await publishVersionToDrive({ |
| 39 | releaseDir, |
| 40 | repo, |
| 41 | version, |
| 42 | channel: 'production', |
| 43 | vdrive: { enabled: true, channels: null }, |
| 44 | chunk |
| 45 | }) |
| 46 | return { res, versionDir } |
| 47 | } |
| 48 | |
| 49 | test('locations.json is written into the version dir so it replicates', async (t) => { |
| 50 | const releaseDir = await tmpdir('travel') |
| 51 | const repo = 'demo' |
| 52 | |
| 53 | const { res, versionDir } = await publish(releaseDir, repo, '1.0.0', { |
| 54 | 'app.bin': Buffer.alloc(512 * 1024, 3) |
| 55 | }) |
| 56 | t.is(res.ok, true, 'published') |
| 57 | |
| 58 | const inVersion = path.join(versionDir, 'locations.json') |
| 59 | const inStore = path.join(vdriveStoreDir(releaseDir, repo), 'locations.json') |
| 60 | |
| 61 | t.ok(await exists(inStore), 'the drive keeps its working copy') |
| 62 | t.ok(await exists(inVersion), 'and a copy travels with the version') |
| 63 | |
| 64 | const a = JSON.parse(await fsp.readFile(inStore, 'utf8')) |
| 65 | const b = JSON.parse(await fsp.readFile(inVersion, 'utf8')) |
| 66 | t.alike(b, a, 'the travelling copy is the same index') |
| 67 | t.ok(b.entries.length > 0, 'and it is not empty') |
| 68 | |
| 69 | await fsp.rm(releaseDir, { recursive: true, force: true }) |
| 70 | }) |
| 71 | |
| 72 | test('every recorded path is relative and inside the root', async (t) => { |
| 73 | // An absolute path would not resolve on another machine and would bake this |
| 74 | // machine's layout into an artifact peers can read. |
| 75 | const releaseDir = await tmpdir('rel') |
| 76 | const repo = 'demo' |
| 77 | const { versionDir } = await publish(releaseDir, repo, '1.0.0', { |
| 78 | 'app.bin': Buffer.alloc(256 * 1024, 9), |
| 79 | 'notes.txt': 'hello' |
| 80 | }) |
| 81 | |
| 82 | const idx = JSON.parse(await fsp.readFile(path.join(versionDir, 'locations.json'), 'utf8')) |
| 83 | for (const [, rel] of idx.entries) { |
| 84 | t.absent(path.isAbsolute(rel), 'relative: ' + rel) |
| 85 | t.absent(rel.startsWith('..'), 'inside the root: ' + rel) |
| 86 | } |
| 87 | |
| 88 | await fsp.rm(releaseDir, { recursive: true, force: true }) |
| 89 | }) |
| 90 | |
| 91 | test("the newest version's copy covers earlier versions too", async (t) => { |
| 92 | // The index accumulates, so a mirror needs only the latest version's copy to |
| 93 | // serve the whole drive. If this ever became a per-version delta, a mirror |
| 94 | // would silently fail to resolve blocks belonging to older releases. |
| 95 | const releaseDir = await tmpdir('accum') |
| 96 | const repo = 'demo' |
| 97 | |
| 98 | const first = await publish(releaseDir, repo, '1.0.0', { |
| 99 | 'one.bin': Buffer.alloc(256 * 1024, 1) |
| 100 | }) |
| 101 | t.is(first.res.ok, true, 'first published') |
| 102 | |
| 103 | const second = await publish(releaseDir, repo, '2.0.0', { |
| 104 | 'two.bin': Buffer.alloc(256 * 1024, 2) |
| 105 | }) |
| 106 | t.is(second.res.ok, true, 'second published') |
| 107 | |
| 108 | const older = JSON.parse( |
| 109 | await fsp.readFile(path.join(first.versionDir, 'locations.json'), 'utf8') |
| 110 | ) |
| 111 | const newest = JSON.parse( |
| 112 | await fsp.readFile(path.join(second.versionDir, 'locations.json'), 'utf8') |
| 113 | ) |
| 114 | |
| 115 | const newestHashes = new Set(newest.entries.map((e) => e[0])) |
| 116 | for (const [hash] of older.entries) { |
| 117 | t.ok(newestHashes.has(hash), 'newest snapshot still carries ' + hash.slice(0, 12)) |
| 118 | } |
| 119 | t.ok(newest.entries.length >= older.entries.length, 'and it only grew') |
| 120 | |
| 121 | await fsp.rm(releaseDir, { recursive: true, force: true }) |
| 122 | }) |
| 123 | |
| 124 | async function exists (p) { |
| 125 | try { |
| 126 | await fsp.stat(p) |
| 127 | return true |
| 128 | } catch { |
| 129 | return false |
| 130 | } |
| 131 | } |