OverviewFilesCommitsIssuesDocsReleasesPeers

pear-git-v3 commits / 1847ca0

0.1.233: sort the vdrive file list — readdir order was reaching the merkle tree
publishTree writes files in exactly the order it is handed them, and that order lands in the drive's merkle tree. `walk()` fed it straight from readdir, whose order is filesystem-defined — hash order on ext4, unstable across filesystems, inode layouts, and directories. Measured on the real path (hyper-overlay's chunker, the 6-file ifsrenderer-linux release, two independent stores): same bytes, same order tree 0997c75b… blobs 179a4fff… 1844 blocks same bytes, REVERSED order tree dacbc54b… blobs 8bb25544… 1844 blocks Identical content, different drive. After the fix all three builds — including one fed a reversed list — produce 0997c75b… again. Invisible today because exactly one machine ever builds a drive. It stops being invisible the moment a second machine builds the same release, which is the premise of seeding by reproduction rather than replication: it would silently produce a DIFFERENT drive from identical artifacts. Under a content-derived (manifest) key that is worse than a mismatch — two machines would claim the same key for different content. Byte-wise, not localeCompare, matching hyper-overlay's own tree format ("Entries MUST be sorted byte-wise by name"). This removes an inconsistency between the two layers rather than inventing a rule. On the test: the first version passed BEFORE the fix, because ext4 happened to return a 5-file directory alphabetically — which is exactly why this stayed hidden. A 200-file variant was flaky for the same reason (hash order is per-directory). So the deterministic proof is sortDriveNames tested directly on a scrambled list, with the walk cases kept as invariants. 20/20, 66/66 across the vdrive tests. Forge org #145. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1847ca0689b749d136032bb0410b3296d3f58ae2
by plankatron <fleeky@gmail.com> · 2026-08-25T17:59:19.000Z

Changed files (3) +159 -2

lib/vdrive-publish.js +30 -1
……@@ -65,10 +65,39 @@ async function dirSize (dir) {
6565 return total
6666}
6767
68/**
69 * Byte-wise sort of drive paths.
70 *
71 * publishTree writes files in exactly the order it is given, and that order
72 * lands in the drive's merkle tree — measured on the real path against a
73 * 6-file release: identical bytes in reversed order produced tree dacbc54b…
74 * instead of 0997c75b…, same 1844 blocks. So the order must come from the
75 * names, never from the filesystem.
76 *
77 * Byte-wise rather than locale-aware, matching hyper-overlay's tree format
78 * ("Entries MUST be sorted byte-wise by name"). localeCompare would order
79 * 'Build.log' after 'aaa.txt' on some locales and before it on others, which
80 * is the same class of bug one layer up.
81 */
82export function sortDriveNames (names) {
83 return [...names].sort((a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b)))
84}
85
86// Exported for test/vdrive-walk-order.test.js — the ordering this produces is
87// baked into the drive's merkle tree, so it needs a direct test.
88export { walk as walkForTest }
89
6890async function walk (dir, base = dir, out = []) {
91 await collect(dir, base, out)
92 // readdir order is filesystem-defined (hash order on ext4, and unstable
93 // across filesystems and directories), so it must not reach publishTree.
94 return sortDriveNames(out)
95}
96
97async function collect (dir, base, out) {
6998 for (const entry of await fsp.readdir(dir, { withFileTypes: true })) {
7099 const full = path.join(dir, entry.name)
71 if (entry.isDirectory()) await walk(full, base, out)
100 if (entry.isDirectory()) await collect(full, base, out)
72101 else if (entry.isFile()) out.push('/' + path.relative(base, full).split(path.sep).join('/'))
73102 }
74103 return out
package.json +1 -1
……@@ -1,6 +1,6 @@
11{
22 "name": "pear-git-v3",
3 "version": "0.1.232",
3 "version": "0.1.233",
44 "description": "P2P git forge \u2014 multi-writer issue/org/release/peer-mirror daemon. Built bottom-up after the failed v2.",
55 "type": "module",
66 "main": "index.js",
test/vdrive-walk-order.test.js +128 -0
……@@ -0,0 +1,128 @@
1/**
2 * The file list a vdrive is published from must be deterministic.
3 *
4 * publishTree writes files in exactly the order it is handed them, and the
5 * order lands in the drive's merkle tree. Measured on the real path with
6 * hyper-overlay's chunker, against a real 6-file release:
7 *
8 * same bytes, same order -> tree 0997c75b… (identical across stores)
9 * same bytes, REVERSED order -> tree dacbc54b… (1844 blocks either way)
10 *
11 * `walk()` fed that list straight from `readdir`, whose order is filesystem
12 * defined — hash order on ext4, and not stable across filesystems, inode
13 * layouts, or a directory rewritten in a different sequence.
14 *
15 * Today only one machine ever builds a drive, so nothing notices. The moment a
16 * second machine builds the same release — the premise of seeding by
17 * reproduction rather than replication — it silently produces a DIFFERENT
18 * drive from identical artifacts. Under a content-derived (manifest) key that
19 * is worse than a mismatch: two machines would claim the same key for
20 * different content.
21 *
22 * hyper-overlay's own tree format already requires this ("Entries MUST be
23 * sorted byte-wise by name"), so sorting here removes an inconsistency between
24 * the two layers rather than inventing a rule.
25 */
26
27import test from 'brittle'
28import fs from 'fs'
29import path from 'path'
30import os from 'os'
31
32import { walkForTest as walk, sortDriveNames } from '../lib/vdrive-publish.js'
33
34function mktemp () {
35 return fs.mkdtempSync(path.join(os.tmpdir(), 'vdrive-walk-'))
36}
37
38function isByteSorted (names) {
39 for (let i = 1; i < names.length; i++) {
40 if (Buffer.compare(Buffer.from(names[i - 1]), Buffer.from(names[i])) >= 0) return false
41 }
42 return true
43}
44
45test('walk returns a byte-wise sorted list regardless of creation order', async (t) => {
46 const dir = mktemp()
47 t.teardown(() => fs.rmSync(dir, { recursive: true, force: true }))
48
49 // Created in deliberately reverse-ish order, across nested dirs, so any
50 // filesystem that reports creation or hash order shows it.
51 fs.mkdirSync(path.join(dir, 'zeta'), { recursive: true })
52 fs.mkdirSync(path.join(dir, 'alpha'), { recursive: true })
53 fs.writeFileSync(path.join(dir, 'zzz.txt'), 'z')
54 fs.writeFileSync(path.join(dir, 'zeta', 'b.bin'), 'zb')
55 fs.writeFileSync(path.join(dir, 'mmm.txt'), 'm')
56 fs.writeFileSync(path.join(dir, 'alpha', 'a.bin'), 'aa')
57 fs.writeFileSync(path.join(dir, 'aaa.txt'), 'a')
58
59 const names = await walk(dir)
60
61 t.is(names.length, 5, 'found every file')
62 t.ok(isByteSorted(names), 'byte-wise sorted: ' + JSON.stringify(names))
63})
64
65test('walk sorts even when readdir does not — many files', async (t) => {
66 const dir = mktemp()
67 t.teardown(() => fs.rmSync(dir, { recursive: true, force: true }))
68
69 // A handful of files can come back alphabetical by luck — ext4 returns hash
70 // order, which only diverges visibly once the directory is large. That luck
71 // is exactly why this bug stayed invisible: the real 6-file ifsrenderer
72 // release walks in sorted order on this machine, so nothing looked wrong.
73 const names = []
74 for (let i = 0; i < 200; i++) {
75 const n = 'file-' + String(i).padStart(3, '0') + '-' + 'abcdefghij'[i % 10] + '.bin'
76 fs.writeFileSync(path.join(dir, n), 'x')
77 names.push('/' + n)
78 }
79
80 // NB: no assertion that readdir is unsorted — ext4's hash order is
81 // per-directory, so that is luck and made this test flaky. The invariant
82 // below holds either way; sortDriveNames is tested directly for the case
83 // the filesystem will not reliably produce.
84 const walked = await walk(dir)
85 t.is(walked.length, 200, 'found every file')
86 t.ok(isByteSorted(walked), 'walk output is sorted despite readdir order')
87})
88
89test('walk is stable across two calls on the same tree', async (t) => {
90 const dir = mktemp()
91 t.teardown(() => fs.rmSync(dir, { recursive: true, force: true }))
92
93 fs.mkdirSync(path.join(dir, 'dist'), { recursive: true })
94 fs.mkdirSync(path.join(dir, 'by-arch', 'linux-x64'), { recursive: true })
95 fs.writeFileSync(path.join(dir, 'build.log'), 'log')
96 fs.writeFileSync(path.join(dir, 'dist', 'b.AppImage'), 'B')
97 fs.writeFileSync(path.join(dir, 'dist', 'a.tar.gz'), 'A')
98 fs.writeFileSync(path.join(dir, 'by-arch', 'linux-x64', 'app.bin'), 'X')
99 fs.writeFileSync(path.join(dir, 'package.json'), '{}')
100
101 const first = await walk(dir)
102 const second = await walk(dir)
103
104 t.alike(second, first, 'two walks agree')
105 t.ok(isByteSorted(first), 'and are sorted')
106
107 // The property that actually matters: a nested path sorts as a whole path,
108 // so the order cannot depend on which directory was descended into first.
109 const expected = [...first].sort((a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b)))
110 t.alike(first, expected, 'matches an independent byte-wise sort')
111})
112
113
114test('sortDriveNames orders byte-wise, deterministically', (t) => {
115 // The case the filesystem will not reliably hand us. Deliberately scrambled,
116 // with nested paths, so ordering cannot depend on descent order.
117 const scrambled = [
118 '/zzz.txt', '/dist/b.AppImage', '/aaa.txt', '/by-arch/linux-x64/app.bin',
119 '/dist/a.tar.gz', '/manifest.json', '/Build.log'
120 ]
121 const sorted = sortDriveNames(scrambled)
122
123 t.ok(isByteSorted(sorted), 'sorted: ' + JSON.stringify(sorted))
124 t.is(sorted.length, scrambled.length, 'nothing lost')
125 t.alike(sortDriveNames([...scrambled].reverse()), sorted, 'input order is irrelevant')
126 // Uppercase sorts before lowercase byte-wise; a locale-aware sort would not.
127 t.is(sorted[0], '/Build.log', 'byte-wise, not locale-aware')
128})