OverviewFilesCommitsIssuesDocsReleasesPeers

p2pbuilders commits / 1a3af73

Fix swarm connectivity in sandboxed environments, gossip late-learned keys, wire web tests into npm test
- SwarmHub/Node/startRelay now accept firewalled/host/port DHT options; testnet-based tests (m2, m3, m7) pass firewalled:false + loopback so server-side connections don't depend on holepunching, which fails in containerized CI and made the swarm tests flaky or hang. - Announce channel gossips newly tracked keys to already-open channels, so transitive peer discovery no longer depends on connection ordering. New m3 test covers the late-announce path. - web/: add package.json (type: module) so test/engine.mjs runs under Node with the root package set to commonjs; chain web tests into npm test via test:web. - Docs: pear v3 installs to ~/.local/bin and configures PATH itself — update README, landing, and public pages off the old macOS-only Application Support path; refresh test counts. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01QAqfE2dpo3vsVd9BorLY6w
1a3af73ef1cd75a92f61d6562f4b565eae754bec
by Claude <noreply@anthropic.com> · 2026-07-29T11:41:56.000Z

Changed files (13) +132 -38

README.md +7 -5
……@@ -12,10 +12,12 @@ Install Pear once:
1212
1313```
1414npm i -g pear
15pear # bootstraps the runtime
16export PATH="$HOME/Library/Application Support/pear/bin:$PATH"
15pear # bootstraps the runtime + adds pear to your PATH
1716```
1817
18Then open a new terminal so the PATH change takes effect (`which pear` should
19print `~/.local/bin/pear` on macOS/Linux).
20
1921Run p2pbuilders:
2022
2123```
……@@ -106,7 +108,7 @@ scripts/
106108 publish.js publish a directory to hiverelay
107109 seed-pear.js ask hiverelay to seed a pear:// key
108110 probe-relays.mjs list relays currently pinning the live key
109test/ 41 tests covering every layer
111test/ 42 tests covering every layer
110112```
111113
112114## How it works
……@@ -138,7 +140,7 @@ npm install
138140# run the optional Node dev server (browser at http://localhost:8787)
139141npm run dev
140142
141# run the full test suite (41 tests)
143# run the full test suite (42 backend tests + 24 web engine checks)
142144npm test
143145```
144146
……@@ -180,7 +182,7 @@ node scripts/publish.js # publish ./public as a hyperdrive
180182
181183**Shipping now:**
182184- Terminal app (TUI) via `pear run pear://…`
183- Full backend verified with 41 tests
185- Full backend verified with 42 tests (plus 24 web engine checks)
184186- Auto-seeding to the public hiverelay network
185187
186188**Parked:**
landing/index.html +7 -6
……@@ -291,15 +291,16 @@
291291 <p><strong>install pear</strong> — the p2p runtime holepunch ships.</p>
292292<pre><code>npm install -g pear
293293pear</code></pre>
294 <p>the second command bootstraps the real runtime. takes 30s on first run.</p>
294 <p>
295 the second command bootstraps the real runtime and adds it to your
296 <code>PATH</code>. takes 30s on first run.
297 </p>
295298 </li>
296299
297300 <li>
298 <p><strong>add pear to your path.</strong> one-time setup.</p>
299<pre><code>echo 'export PATH="$HOME/Library/Application Support/pear/bin:$PATH"' &gt;&gt; ~/.zshrc
300source ~/.zshrc
301which pear
302# should print: /Users/you/Library/Application Support/pear/bin/pear</code></pre>
301 <p><strong>open a new terminal</strong> so the <code>PATH</code> change takes effect.</p>
302<pre><code>which pear
303# should print: ~/.local/bin/pear</code></pre>
303304 </li>
304305
305306 <li>
package.json +2 -1
……@@ -37,7 +37,8 @@
3737 "pear:stage": "pear stage dev .",
3838 "relay": "node src/relay/server.js",
3939 "publish": "node scripts/publish.js",
40 "test": "node test/m1.js && node test/m2.js && node test/m3.js && node test/m4.js && node test/m5.js && node test/m6.js && node test/m7.js && node test/m8.js && node test/m9.js && node test/m10-bare.js && node test/m11.js"
40 "test": "node test/m1.js && node test/m2.js && node test/m3.js && node test/m4.js && node test/m5.js && node test/m6.js && node test/m7.js && node test/m8.js && node test/m9.js && node test/m10-bare.js && node test/m11.js && npm run test:web",
41 "test:web": "node web/test/engine.mjs"
4142 },
4243 "pear": {
4344 "name": "p2pbuilders",
public/index.html +7 -6
……@@ -291,15 +291,16 @@
291291 <p><strong>install pear</strong> — the p2p runtime holepunch ships.</p>
292292<pre><code>npm install -g pear
293293pear</code></pre>
294 <p>the second command bootstraps the real runtime. takes 30s on first run.</p>
294 <p>
295 the second command bootstraps the real runtime and adds it to your
296 <code>PATH</code>. takes 30s on first run.
297 </p>
295298 </li>
296299
297300 <li>
298 <p><strong>add pear to your path.</strong> one-time setup.</p>
299<pre><code>echo 'export PATH="$HOME/Library/Application Support/pear/bin:$PATH"' &gt;&gt; ~/.zshrc
300source ~/.zshrc
301which pear
302# should print: /Users/you/Library/Application Support/pear/bin/pear</code></pre>
301 <p><strong>open a new terminal</strong> so the <code>PATH</code> change takes effect.</p>
302<pre><code>which pear
303# should print: ~/.local/bin/pear</code></pre>
303304 </li>
304305
305306 <li>
src/backend/announce.js +18 -2
……@@ -14,13 +14,22 @@ const pubkeyArray = cenc.array(cenc.fixed(PUBKEY_BYTES))
1414// selfPubkey — our own user pubkey (so we don't echo it back)
1515// getKnownKeys() — returns Buffer[] of pubkeys we'd like to gossip
1616// onKey(pubkey) — callback for each new pubkey the peer announces
17function setupAnnounce (muxer, { selfPubkey, getKnownKeys, onKey }) {
17// onOpen(handle) — channel is live; handle.sendKeys() can push later updates
18// onClose(handle) — channel closed; drop the handle
19// Returns a handle with sendKeys(keys) for announcing keys learned after the
20// initial exchange (transitive gossip), or null if the peer lacks the protocol.
21function setupAnnounce (muxer, { selfPubkey, getKnownKeys, onKey, onOpen, onClose }) {
1822 let keysMsg
23 let handle
1924 const channel = muxer.createChannel({
2025 protocol: PROTOCOL,
2126 onopen () {
2227 const keys = getKnownKeys().slice(0, MAX_KEYS_PER_ANNOUNCE)
2328 keysMsg.send(keys)
29 if (onOpen) onOpen(handle)
30 },
31 onclose () {
32 if (onClose) onClose(handle)
2433 }
2534 })
2635 if (!channel) return null // peer doesn't support the protocol
……@@ -34,8 +43,15 @@ function setupAnnounce (muxer, { selfPubkey, getKnownKeys, onKey }) {
3443 }
3544 }
3645 })
46 handle = {
47 channel,
48 sendKeys (keys) {
49 if (!keys.length) return
50 keysMsg.send(keys.slice(0, MAX_KEYS_PER_ANNOUNCE))
51 }
52 }
3753 channel.open()
38 return channel
54 return handle
3955}
4056
4157module.exports = { setupAnnounce, PROTOCOL, MAX_KEYS_PER_ANNOUNCE }
src/backend/node.js +13 -3
……@@ -26,6 +26,7 @@ class Node extends EventEmitter {
2626 this._cleanup = cleanup
2727 this.swarm = null
2828 this._tracked = new Map() // hex(pubkey) -> remote Hypercore
29 this._announces = new Set() // live announce-channel handles
2930 }
3031
3132 static async openDisk (dir, opts) {
……@@ -43,7 +44,7 @@ class Node extends EventEmitter {
4344 get pubkey () { return this.core.key }
4445 get length () { return this.core.length }
4546
46 async attachSwarm ({ bootstrap, keyPair, autoTrack = true, relays = [] } = {}) {
47 async attachSwarm ({ bootstrap, keyPair, autoTrack = true, relays = [], firewalled, host, port } = {}) {
4748 if (this.swarm) throw new Error('swarm already attached')
4849 const onConnection = ({ muxer }) => {
4950 setupAnnounce(muxer, {
……@@ -52,10 +53,12 @@ class Node extends EventEmitter {
5253 onKey: (pubkey) => {
5354 if (autoTrack) this.trackUser(pubkey).catch(err => this.emit('error', err))
5455 this.emit('announce', pubkey)
55 }
56 },
57 onOpen: (handle) => this._announces.add(handle),
58 onClose: (handle) => this._announces.delete(handle)
5659 })
5760 }
58 this.swarm = new SwarmHub(this.store, { bootstrap, keyPair, onConnection })
61 this.swarm = new SwarmHub(this.store, { bootstrap, keyPair, onConnection, firewalled, host, port })
5962 for (const pk of relays) this.swarm.joinPeer(pk)
6063 return this.swarm
6164 }
……@@ -94,6 +97,13 @@ class Node extends EventEmitter {
9497 // it costs a small counter. close() takes care of teardown.
9598 const done = remote.findingPeers()
9699 remote.on('close', done)
100 // Gossip the newly learned key to peers we're already connected to — the
101 // channel-open exchange only covers keys known at connect time, so without
102 // this, transitive discovery depends on connection ordering. Each node only
103 // re-announces a key the first time it tracks it, so the flood terminates.
104 for (const handle of this._announces) {
105 try { handle.sendKeys([pubkey]) } catch { /* channel tearing down */ }
106 }
97107 this.emit('track', pubkey)
98108 return remote
99109 }
src/backend/swarm.js +10 -2
……@@ -1,14 +1,22 @@
11'use strict'
22
33const Hyperswarm = require('hyperswarm')
4const DHT = require('hyperdht')
45
56// Wraps Hyperswarm, pipes every connection into the corestore replicator.
67// If onConnection is provided, it's invoked per connection with { conn, stream, muxer }
78// so callers can attach additional protomux channels (e.g. announce).
9// `firewalled`/`host`/`port` are forwarded to a dedicated HyperDHT instance —
10// tests and LAN deployments pass { firewalled: false, host: '127.0.0.1' } so
11// server connections don't depend on holepunching. Hyperswarm destroys the DHT
12// it's handed, so no extra teardown is needed here.
813class SwarmHub {
9 constructor (store, { bootstrap, keyPair, onConnection } = {}) {
14 constructor (store, { bootstrap, keyPair, onConnection, firewalled, host, port } = {}) {
1015 this.store = store
11 this.swarm = new Hyperswarm({ bootstrap, keyPair })
16 const needsOwnDht = firewalled !== undefined || host !== undefined || port !== undefined
17 this.swarm = needsOwnDht
18 ? new Hyperswarm({ keyPair, dht: new DHT({ bootstrap, firewalled, host, port }) })
19 : new Hyperswarm({ bootstrap, keyPair })
1220 this.swarm.on('connection', (conn) => {
1321 const stream = this.store.replicate(conn)
1422 const muxer = stream.noiseStream.userData
src/relay/server.js +3 -2
……@@ -33,11 +33,12 @@ async function startRelay ({
3333 boards = [],
3434 bootstrap,
3535 dhtRelayPort,
36 announcePubkey = true
36 announcePubkey = true,
37 swarmOpts = {} // extra SwarmHub/DHT options, e.g. { firewalled: false, host: '127.0.0.1' } for local testnets
3738} = {}) {
3839 if (!dir) throw new Error('dir required')
3940 const node = await Node.openDisk(dir, {
40 swarm: bootstrap ? { bootstrap } : {}
41 swarm: { ...(bootstrap ? { bootstrap } : {}), ...swarmOpts }
4142 })
4243 for (const name of boards) await node.joinBoard(name)
4344
test/_helpers.js +11 -0
……@@ -0,0 +1,11 @@
1'use strict'
2
3// Swarm options for tests running against a local hyperdht testnet.
4// firewalled:false + loopback binding skip firewall detection and holepunching,
5// which don't work in sandboxed/containerized CI environments — server-side
6// connections would otherwise never complete and the tests flake.
7function localSwarm (bootstrap) {
8 return { bootstrap, firewalled: false, host: '127.0.0.1' }
9}
10
11module.exports = { localSwarm }
test/m2.js +3 -2
……@@ -12,6 +12,7 @@ const { Node } = require('../src/backend/node')
1212const { connectDirect } = require('../src/backend/swarm')
1313const { boardTopic } = require('../src/backend/board')
1414const createTestnet = require('hyperdht/testnet')
15const { localSwarm } = require('./_helpers')
1516
1617const tests = []
1718function test (name, fn) { tests.push({ name, fn }) }
……@@ -95,8 +96,8 @@ test('swarm: two nodes joining same board connect + replicate', async () => {
9596 const testnet = await createTestnet(3, { teardown: () => {} })
9697 const bootstrap = testnet.bootstrap
9798
98 const a = await Node.openTemp({ swarm: { bootstrap } })
99 const b = await Node.openTemp({ swarm: { bootstrap } })
99 const a = await Node.openTemp({ swarm: localSwarm(bootstrap) })
100 const b = await Node.openTemp({ swarm: localSwarm(bootstrap) })
100101
101102 await a.post('general', 'swarm-test', 'over the wire')
102103
test/m3.js +37 -5
……@@ -9,6 +9,7 @@ const b4a = require('b4a')
99const { Node } = require('../src/backend/node')
1010const createTestnet = require('hyperdht/testnet')
1111const { decodeOp } = require('../src/backend/ops')
12const { localSwarm } = require('./_helpers')
1213
1314const tests = []
1415function test (name, fn) { tests.push({ name, fn }) }
……@@ -17,8 +18,8 @@ test('swarm+announce: B auto-tracks A after joining same board', async () => {
1718 const testnet = await createTestnet(3, { teardown: () => {} })
1819 const bootstrap = testnet.bootstrap
1920
20 const a = await Node.openTemp({ swarm: { bootstrap } })
21 const b = await Node.openTemp({ swarm: { bootstrap } })
21 const a = await Node.openTemp({ swarm: localSwarm(bootstrap) })
22 const b = await Node.openTemp({ swarm: localSwarm(bootstrap) })
2223
2324 await a.post('general', 'auto-discovery test', 'body')
2425
……@@ -44,9 +45,9 @@ test('swarm+announce: transitive discovery (C learns A via B)', async () => {
4445 const testnet = await createTestnet(3, { teardown: () => {} })
4546 const bootstrap = testnet.bootstrap
4647
47 const a = await Node.openTemp({ swarm: { bootstrap } })
48 const b = await Node.openTemp({ swarm: { bootstrap } })
49 const c = await Node.openTemp({ swarm: { bootstrap } })
48 const a = await Node.openTemp({ swarm: localSwarm(bootstrap) })
49 const b = await Node.openTemp({ swarm: localSwarm(bootstrap) })
50 const c = await Node.openTemp({ swarm: localSwarm(bootstrap) })
5051
5152 await a.post('general', 'hello', 'from A')
5253
……@@ -71,6 +72,37 @@ test('swarm+announce: transitive discovery (C learns A via B)', async () => {
7172 await testnet.destroy()
7273})
7374
75test('swarm+announce: late announce — C learns a key B tracked after C connected', async () => {
76 const testnet = await createTestnet(3, { teardown: () => {} })
77 const bootstrap = testnet.bootstrap
78
79 const a = await Node.openTemp({ swarm: localSwarm(bootstrap) })
80 const b = await Node.openTemp({ swarm: localSwarm(bootstrap) })
81 const c = await Node.openTemp({ swarm: localSwarm(bootstrap) })
82
83 // B and C connect first, before B knows anything about A. The channel-open
84 // key exchange between B and C therefore cannot carry A's key.
85 await b.joinBoard('meta')
86 await c.joinBoard('meta')
87 await waitForEvent(c, 'track', (pk) => b4a.equals(pk, b.pubkey), 15000)
88
89 // Now A shows up on a board shared with B only.
90 await a.joinBoard('general')
91 await b.joinBoard('general')
92 await waitForEvent(b, 'track', (pk) => b4a.equals(pk, a.pubkey), 15000)
93
94 // C must learn A via B's re-broadcast on the already-open channel.
95 if (!c.userCore(a.pubkey)) {
96 await waitForEvent(c, 'track', (pk) => b4a.equals(pk, a.pubkey), 15000)
97 }
98 assert.ok(c.userCore(a.pubkey), 'C discovered A via late announce')
99
100 await a.close()
101 await b.close()
102 await c.close()
103 await testnet.destroy()
104})
105
74106// ---------------------------------------------------------------------------
75107
76108function waitForEvent (emitter, event, predicate, timeoutMs) {
test/m7.js +11 -4
……@@ -16,6 +16,7 @@ const createTestnet = require('hyperdht/testnet')
1616const { Node } = require('../src/backend/node')
1717const { startRelay } = require('../src/relay/server')
1818const { decodeOp } = require('../src/backend/ops')
19const { localSwarm } = require('./_helpers')
1920const fs = require('fs'), os = require('os'), path = require('path')
2021
2122const tests = []
……@@ -37,9 +38,15 @@ test('relay seeds offline author to late-arriving peer', async () => {
3738 const bootstrap = testnet.bootstrap
3839
3940 const relayDir = mkdir()
40 const relay = await startRelay({ dir: relayDir, boards: ['general'], bootstrap, announcePubkey: false })
41
42 const a = await Node.openTemp({ swarm: { bootstrap } })
41 const relay = await startRelay({
42 dir: relayDir,
43 boards: ['general'],
44 bootstrap,
45 announcePubkey: false,
46 swarmOpts: { firewalled: false, host: '127.0.0.1' }
47 })
48
49 const a = await Node.openTemp({ swarm: localSwarm(bootstrap) })
4350 await a.joinBoard('general')
4451
4552 // wait for the relay to see A
……@@ -57,7 +64,7 @@ test('relay seeds offline author to late-arriving peer', async () => {
5764 await a.close()
5865
5966 // B comes online, joins same board
60 const b = await Node.openTemp({ swarm: { bootstrap } })
67 const b = await Node.openTemp({ swarm: localSwarm(bootstrap) })
6168 await b.joinBoard('general')
6269
6370 // B learns about A's pubkey via the relay's announce
web/package.json +3 -0
……@@ -0,0 +1,3 @@
1{
2 "type": "module"
3}