| … | … | @@ -70,7 +70,9 @@ function prepareEnv () { |
| 70 | 70 | // which boots this file for real — died with EADDRINUSE whenever a favela-web was already running on |
| 71 | 71 | // this machine, i.e. on any developer box that had actually deployed one. The phone always gets 4041. |
| 72 | 72 | const FAVELA_PORT = Number(process.env.FAVELA_PORT) || 4041 |
| 73 | | const state = { appview: null, hpp: null, ready: false, favelaUrl: null, favelaSession: null } |
| 73 | // No `appview` here any more — favela-web owns the one index on this node. See the note where the |
| 74 | // worklet's second one used to be built. |
| 75 | const state = { hpp: null, ready: false, favelaUrl: null, favelaSession: null } |
| 74 | 76 | |
| 75 | 77 | async function boot () { |
| 76 | 78 | prepareEnv() |
| … | … | @@ -141,75 +143,32 @@ async function boot () { |
| 141 | 143 | log('no DID paired — read-only until pairing (see spec §6.3)') |
| 142 | 144 | } |
| 143 | 145 | |
| 144 | | // ---- the appview: favela-web's index, verbatim ---- |
| 146 | // ---- there is ONE index on this phone, and favela-web owns it ---- |
| 145 | 147 | // |
| 146 | | // The index is fed a `getRecordStore` hook so it can source a baseline from the CONTENT-ADDRESSED |
| 147 | | // record files hpp already wrote — the same OverlayRepoStore construction favela-web uses |
| 148 | | // (roomKey16(did) under ATPROTO_OVERLAY_ROOT). Without it the appview opens empty and STAYS empty: |
| 149 | | // the node replicates the DID's repo perfectly while the timeline, which reads the INDEX rather than |
| 150 | | // the repo, shows "nothing yet" forever. That is not a sync problem and no amount of waiting fixes it. |
| 151 | | const Corestore = (await import('corestore')).default |
| 152 | | const { Appview } = await import('favela-web/lib/appview.js') |
| 153 | | const { OverlayRepoStore } = await import('atproto-store/lib/overlay-repo-store.js') |
| 154 | | const { roomKey16 } = await import('favela-web/lib/blob-serve.js') |
| 155 | | const store = new Corestore(path.join(dirs.favela, 'index')) |
| 156 | | await store.ready() |
| 157 | | const recordStores = new Map() |
| 158 | | const getRecordStore = async (did) => { |
| 159 | | if (!recordStores.has(did)) { |
| 160 | | recordStores.set(did, await new OverlayRepoStore(path.join(dirs.overlay, roomKey16(did)), { did }).ready()) |
| 161 | | } |
| 162 | | return recordStores.get(did) |
| 163 | | } |
| 164 | | state.appview = await new Appview(store, { getRecordStore }).ready() |
| 165 | | log('appview ready') |
| 166 | | |
| 167 | | // ---- feed the index from THIS node's own records ---- |
| 148 | // ⛔ WHAT USED TO BE HERE, AND WHY IT WAS WRONG. The worklet built its OWN `Appview` on its OWN |
| 149 | // Corestore (<storage>/favela/index) and fed it by folding hpp's records straight off local disk |
| 150 | // (`indexPeer({_noRepo:true})`) on every StateBase advance. The premise was that a co-owner does not |
| 151 | // need favela-web's ingest loop because the records are already here. Both halves of that failed: |
| 168 | 152 | // |
| 169 | | // ⛔ NOT favela-web's ingest loop. That one is keyless: it resolves the freshest co-owner over the DHT |
| 170 | | // and tails their firehose, because it holds no repo of its own. A co-owner does not need any of that |
| 171 | | // — the records are on local disk, written by the hpp in this process. `indexPeer` with `_noRepo` |
| 172 | | // takes exactly that path (appview.js:990), reading the overlay store rather than a repo core. |
| 153 | // 1. NOBODY READ IT. Every screen reads favela-web's index (see the compose handler's note — this |
| 154 | // file already recorded the same discovery once, for optimistic writes). So the fold logged a |
| 155 | // healthy "indexed … statebase-update" on every advance while updating an index with no readers. |
| 156 | // 2. IT SKIPPED THE VERIFICATION. atproto's sync spec is explicit that "receiving services are |
| 157 | // responsible for validating message structure and verifying authenticity" — verify the commit |
| 158 | // signature against current identity data, check the message against the signed commit in the |
| 159 | // blocks CAR. Reading records off disk does none of that. Defensible for a DID we sign for; |
| 160 | // wrong for anyone else, and this path made no distinction. |
| 173 | 161 | // |
| 174 | | // Re-indexed on every StateBase advance so replicated writes from other devices land too. Debounced, |
| 175 | | // because a replication burst fires this repeatedly and indexPeer re-folds the whole record set; |
| 176 | | // putPost/putProfile are dedup-gated by content signature, so a re-fold re-reads without re-writing. |
| 177 | | if (state.hpp && state.hpp.nodes) { |
| 178 | | const reindex = async (why) => { |
| 179 | | for (const node of state.hpp.nodes) { |
| 180 | | try { |
| 181 | | const n = await state.appview.indexPeer({ did: node.did, _noRepo: true }) |
| 182 | | log('indexed ' + node.did.slice(0, 20) + '…', { why, records: n && n.indexed != null ? n.indexed : undefined }) |
| 183 | | } catch (e) { log('index failed for ' + node.did.slice(0, 20) + '…', { error: (e && e.message) || String(e) }) } |
| 184 | | } |
| 185 | | } |
| 186 | | await reindex('boot') |
| 187 | | // Tracked DIDs are re-viewed on boot for the same reason hosted ones are re-indexed: a view that |
| 188 | | // evaporates on restart is not a subscription. Best-effort and non-blocking — a community nobody is |
| 189 | | // serving right now must not hold up the app. |
| 190 | | // Pull followed accounts too — a scoped home feed with nothing but your own repo indexed is still |
| 191 | | // your own posts. Non-blocking for the same reason the tracked loop is. |
| 192 | | handlers.syncFollows({}).then( |
| 193 | | (r) => { if (r && r.queued) log('syncing ' + r.queued + ' followed account(s)', { following: r.following }) }, |
| 194 | | () => {} |
| 195 | | ) |
| 196 | | for (const did of readTracked()) { |
| 197 | | handlers.viewDid({ did, track: false }).then( |
| 198 | | (r) => log('viewing ' + did.slice(0, 20) + '…', { rev: r.rev }), |
| 199 | | (e) => log('cannot view ' + did.slice(0, 20) + '… yet', { error: e.message }) |
| 200 | | ) |
| 201 | | } |
| 202 | | let pending = null |
| 203 | | for (const node of state.hpp.nodes) { |
| 204 | | try { |
| 205 | | node.sb.watch(() => { |
| 206 | | if (pending) return |
| 207 | | pending = setTimeout(() => { pending = null; reindex('statebase-update').catch(() => {}) }, 2000) |
| 208 | | if (pending.unref) pending.unref() |
| 209 | | }) |
| 210 | | } catch (e) { log('could not watch statebase for ' + node.did.slice(0, 20) + '…', { error: (e && e.message) || String(e) }) } |
| 211 | | } |
| 212 | | } |
| 162 | // It also opened a SECOND OverlayRepoStore over hpp's private record directory, in the same process |
| 163 | // where favela-web already has one. Each instance keeps its own in-memory oid→disk map, rebuilt only |
| 164 | // from its own ready() seed (atproto-store invariant #5) — the shape behind the 75-hour wj62 |
| 165 | // byte-resolution outage. One process, one reader. |
| 166 | // |
| 167 | // favela-web already implements exactly the split this needs (bin/favela-web.js:577-616): a DID it |
| 168 | // co-hosts baselines from the shared store and tails from the current end — no refetch of what is on |
| 169 | // disk — and everyone else backfills from a VERIFIED firehose front, failover-aware, applying each |
| 170 | // #commit incrementally instead of re-folding every record. It just needed to be told who to watch, |
| 171 | // which is what FAVELA_ROSTER below does. |
| 213 | 172 | |
| 214 | 173 | // ── PORT, NOT REIMPLEMENT ──────────────────────────────────────────────────────────────────────── |
| 215 | 174 | // favela-web is a bare app (runtime "bare", bare-http1) and it holds the appview and every social |
| … | … | @@ -297,6 +256,28 @@ async function boot () { |
| 297 | 256 | log('favela-web could NOT start in-process — UI stays on the legacy surface', { error: e.message }) |
| 298 | 257 | } |
| 299 | 258 | |
| 259 | // Re-subscribe to what the user chose, now that favela-web is up to receive it. |
| 260 | // |
| 261 | // ⛔ ORDER IS THE WHOLE POINT. This used to run BEFORE favela-web started, because it fed the worklet's |
| 262 | // own index and had nothing to wait for. It now goes through favela-web, so it must come after — and |
| 263 | // after sign-in, because /api/track requires a viewer. |
| 264 | // |
| 265 | // FAVELA_ROSTER already carried the tracked DIDs into favela-web's boot, so this is not how they get |
| 266 | // subscribed; it is how we find out whether each one is REACHABLE and say so. Both loops are |
| 267 | // non-blocking: a community nobody is serving right now must not hold up the app. |
| 268 | if (state.favelaUrl) { |
| 269 | handlers.syncFollows({}).then( |
| 270 | (r) => { if (r && r.queued) log('syncing ' + r.queued + ' followed account(s)', { following: r.following }) }, |
| 271 | (e) => log('could not read follows yet', { error: (e && e.message) || String(e) }) |
| 272 | ) |
| 273 | for (const did of readTracked()) { |
| 274 | handlers.viewDid({ did, track: false }).then( |
| 275 | (r) => log('viewing ' + did.slice(0, 20) + '…', { rev: r.rev }), |
| 276 | (e) => log('cannot view ' + did.slice(0, 20) + '… yet', { error: e.message }) |
| 277 | ) |
| 278 | } |
| 279 | } |
| 280 | |
| 300 | 281 | state.ready = true |
| 301 | 282 | send({ t: 'ready' }) |
| 302 | 283 | |
| … | … | @@ -311,33 +292,28 @@ async function boot () { |
| 311 | 292 | } |
| 312 | 293 | |
| 313 | 294 | // ---- the IPC surface (spec §3) --------------------------------------------------------------------- |
| 314 | | // ── viewing DIDs this node does NOT host ────────────────────────────────────────────────────────── |
| 315 | | // Hosting and VIEWING are different things, and only the first existed. This node hosts one DID and |
| 316 | | // indexed only what it hosts, so a co-owned community it has not joined (loser-club) and any stranger |
| 317 | | // were both unreachable — not "empty", absent. |
| 295 | // ── talking to our own favela-web ───────────────────────────────────────────────────────────────── |
| 318 | 296 | // |
| 319 | | // favela-web already solved the read: P2PSource joins a DID's discovery topic, verifies its head |
| 320 | | // against the PLC #atproto key, and exposes repo reads. It is imported, not re-implemented — the same |
| 321 | | // module the web client uses. Two adaptations for a phone: |
| 322 | | // - the verify-key resolver is INJECTED over bare-fetch, because favela-web's bare-globals |
| 323 | | // deliberately does not shim fetch (its comment says so) and the default resolver assumes Node. |
| 324 | | // - tracked DIDs persist, so a view survives a restart instead of being re-typed. |
| 297 | // Same client the screens use (lib/favela-api.js), for the same reason its header gives: the moment a |
| 298 | // second caller builds its own request, session handling has two definitions and the one that forgets |
| 299 | // the cookie degrades SILENTLY — favela-web answers 200 with viewerDid null. |
| 325 | 300 | // |
| 326 | | // Indexing writes into the SAME appview the hosted DIDs use, so once viewed, a community's posts flow |
| 327 | | // through the ordinary author/thread/search paths with no special-casing downstream. |
| 328 | | let _p2p = null |
| 329 | | async function p2pSource () { |
| 330 | | if (_p2p) return _p2p |
| 331 | | const { P2PSource } = await import('favela-web/lib/p2p-handler.js') |
| 301 | // ⛔ THIS REPLACED A SECOND P2P STACK. Viewing a foreign DID used to construct favela-web's `P2PSource` |
| 302 | // HERE, in the worklet: its own storage, its own Hyperswarm, its own topic joins — a third subscriber |
| 303 | // beside favela-web's swarm and hpp's, in one process, doing what favela-web's addPeer already does with |
| 304 | // failover-aware front selection and an incremental #commit tail. `/api/track` is that same work, done |
| 305 | // once, by the component that owns the index. |
| 306 | let _api = null |
| 307 | async function api () { |
| 308 | if (!state.favelaUrl) throw new Error('favela-web is not up on this node') |
| 332 | 309 | const fetchImpl = (await import('bare-fetch')).default |
| 333 | | _p2p = new P2PSource({ |
| 334 | | storage: path.join(dirs.root, 'p2p'), |
| 335 | | resolveVerifyKey: async (did) => { |
| 336 | | const { resolvePlcVerifyKey } = await import('favela-web/lib/p2p-handler.js') |
| 337 | | return resolvePlcVerifyKey(did, { doFetch: fetchImpl }) |
| 338 | | } |
| 339 | | }) |
| 340 | | return _p2p |
| 310 | // Rebuilt when the session changes: the token is captured at construction, and a client still holding |
| 311 | // a pre-sign-in null would read anonymously forever. |
| 312 | if (!_api || _api.session !== state.favelaSession) { |
| 313 | const { FavelaApi } = await import('./favela-api.js') |
| 314 | _api = new FavelaApi({ baseUrl: state.favelaUrl, session: state.favelaSession, fetchImpl }) |
| 315 | } |
| 316 | return _api |
| 341 | 317 | } |
| 342 | 318 | |
| 343 | 319 | const TRACKED = () => path.join(dirs.root, 'tracked-dids.json') |
| … | … | @@ -383,7 +359,9 @@ const handlers = { |
| 383 | 359 | if (!me) return { synced: 0 } |
| 384 | 360 | const hosted = new Set(((state.hpp && state.hpp.nodes) || []).map((n) => n.did)) |
| 385 | 361 | const tracked = new Set(readTracked()) |
| 386 | | const following = (await state.appview.following(me).catch(() => [])) || [] |
| 362 | // The graph lives in favela-web's index, which is now the only one. `.catch(() => [])` is kept |
| 363 | // deliberately: a favela-web that is not up yet must degrade to "nothing to sync", not throw on boot. |
| 364 | const following = (await api().then((a) => a.follows({ did: me })).then((r) => (r && r.dids) || []).catch(() => [])) || [] |
| 387 | 365 | const todo = following.filter((d) => !hosted.has(d) && !tracked.has(d)).slice(0, max) |
| 388 | 366 | for (const did of todo) { |
| 389 | 367 | handlers.viewDid({ did }).then( |
| … | … | @@ -420,24 +398,20 @@ const handlers = { |
| 420 | 398 | // author feed, thread, search — with no special case. Tracked so it survives a restart. |
| 421 | 399 | async viewDid ({ did, track = true, timeout = 45000 } = {}) { |
| 422 | 400 | if (!did || !did.startsWith('did:')) throw new Error('viewDid requires { did }') |
| 423 | | // A DID WE HOST IS ALREADY HERE. Its records are in the local overlay — that is how boot indexes it |
| 424 | | // (_noRepo) — so routing it through topic discovery and head verification is wrong by construction, |
| 425 | | // and it failed exactly that way: tapping our own identity reported "nobody is serving it right now" |
| 426 | | // while its posts were on disk and on screen. Local first; the network is only for what is elsewhere. |
| 427 | | const hosted = ((state.hpp && state.hpp.nodes) || []).some((n) => n.did === did) |
| 428 | | if (hosted) { |
| 429 | | const n = await state.appview.indexPeer({ did, _noRepo: true }) |
| 430 | | return { did, indexed: n && n.indexed != null ? n.indexed : null, rev: null, local: true } |
| 431 | | } |
| 432 | | const src = await p2pSource() |
| 433 | | const peer = await src.peer(did, { timeout }) |
| 434 | | if (!peer.head) throw new Error('no verified head for ' + did + ' — nobody is serving it right now') |
| 435 | | const n = await state.appview.indexPeer(peer) |
| 401 | // ONE ROUTE FOR BOTH CASES, because favela-web already distinguishes them better than this did. |
| 402 | // /api/track is HOSTED-FIRST: a DID hpp holds is baselined from the shared store on disk rather than |
| 403 | // chased over the network — which is the same correction made here once before, when tapping our own |
| 404 | // identity reported "nobody is serving it right now" while its posts were on screen. A DID we do not |
| 405 | // hold gets a verified front, and the route waits for one so an unreachable DID SAYS so. |
| 406 | const a = await api() |
| 407 | const r = await a.track({ did, wait: timeout }) |
| 436 | 408 | if (track) { |
| 437 | 409 | const list = readTracked() |
| 438 | 410 | if (!list.includes(did)) { list.push(did); writeTracked(list) } |
| 439 | 411 | } |
| 440 | | return { did, indexed: n && n.indexed != null ? n.indexed : null, rev: peer.head.rev || null } |
| 412 | // `rev` is the front's head where there is one; a hosted DID has no front and reports null, exactly |
| 413 | // as the local branch did before. |
| 414 | return { did, indexed: r && r.indexed != null ? r.indexed : null, rev: (r && r.rev) || null, local: !!(r && r.hosted) } |
| 441 | 415 | }, |
| 442 | 416 | |
| 443 | 417 | async trackedDids () { |
| … | … | @@ -446,9 +420,12 @@ const handlers = { |
| 446 | 420 | // yet is normal (indexed but no app.bsky.actor.profile record) and must not read as an error. |
| 447 | 421 | const out = [] |
| 448 | 422 | for (const did of tracked) { |
| 449 | | let profile = null |
| 450 | | try { profile = await state.appview.getProfile(did) } catch {} |
| 451 | | out.push({ did, handle: (profile && profile.handle) || null, displayName: (profile && profile.displayName) || null, avatar: (profile && profile.avatar) || null }) |
| 423 | // GET /api/profile resolves the handle too, raced against a deadline — so a row is a render rather |
| 424 | // than a DID the UI then has to look up. `held:false` (we hold no profile record) reads the same as |
| 425 | // a failed request here, which is correct: both mean "no name to show", and neither is an error. |
| 426 | let p = null |
| 427 | try { p = await (await api()).profile({ did }) } catch {} |
| 428 | out.push({ did, handle: (p && p.handle) || null, displayName: (p && p.displayName) || null, avatar: (p && p.avatar) || null }) |
| 452 | 429 | } |
| 453 | 430 | return { dids: out } |
| 454 | 431 | }, |
| … | … | @@ -531,7 +508,20 @@ const handlers = { |
| 531 | 508 | async collections () { |
| 532 | 509 | // The #33 ledger — useful on a phone for the same reason it is on a server: an unclassified lexicon |
| 533 | 510 | // arriving should be visible, not silent. |
| 534 | | return { ledger: await state.appview.collectionLedger() } |
| 511 | // favela-web already exposes this on its control surface (`svc.handle('collections')`) — the ledger |
| 512 | // is a property of the index, and the index is favela-web's. Over its UDS rather than HTTP because |
| 513 | // that is where the verb lives; a diagnostic does not deserve a new public route. |
| 514 | // |
| 515 | // ⚠ THE SHAPE CHANGED, and nothing consumes it: this returned `{ ledger: [...] }` from the worklet's |
| 516 | // own index; it now passes favela-web's payload through — `{ ok, tracked, ignored, unknown }`, which |
| 517 | // is strictly more (it says which lexicons are deliberately ignored and whether we have ever seen |
| 518 | // one, not just which are unclassified). No screen reads this verb today; a future one should read |
| 519 | // `unknown`. |
| 520 | const { Client } = await import('hyper-svc') |
| 521 | const sock = process.env.FAVELA_SOCK |
| 522 | if (!sock) throw new Error('favela-web control socket unknown') |
| 523 | const c = await Client.connectUds(sock) |
| 524 | try { return await c.call('collections', {}) } finally { try { await c.close() } catch {} } |
| 535 | 525 | }, |
| 536 | 526 | |
| 537 | 527 | // ── the ported read surface ──────────────────────────────────────────────────────────────────────── |
| … | … | @@ -564,19 +554,19 @@ const handlers = { |
| 564 | 554 | async follow ({ did } = {}) { |
| 565 | 555 | if (!did || !String(did).startsWith('did:')) throw new Error('follow requires { did }') |
| 566 | 556 | const me = viewerDid() |
| 557 | // ⛔ REFUSE HERE, NOT DOWNSTREAM. Every write op on a node with no identity must say the same thing — |
| 558 | // "pair it" — rather than surfacing whatever the layer below happened to notice first. Delegating |
| 559 | // straight to favela-web returned its 403 "this instance is read-only", which is true of favela-web |
| 560 | // and useless to someone holding an unpaired phone. Same wording as writer(), deliberately. |
| 561 | if (!me) throw new Error('this node has no identity yet — pair it before writing') |
| 567 | 562 | if (did === me) throw new Error('cannot follow yourself') |
| 568 | | const w = await writer() |
| 569 | | const existing = await state.appview.followRkey(me, did) |
| 570 | | if (existing) { |
| 571 | | await w.putRecord('app.bsky.graph.follow', existing, null, { action: 'delete' }) |
| 572 | | try { await state.appview.delFollow({ did: me, rkey: existing }) } catch {} |
| 573 | | return { following: false } |
| 574 | | } |
| 575 | | const rkey = tid() |
| 576 | | const record = { $type: 'app.bsky.graph.follow', subject: did, createdAt: new Date().toISOString() } |
| 577 | | await w.putRecord('app.bsky.graph.follow', rkey, record, { action: 'create' }) |
| 578 | | try { await state.appview.putFollow({ did: me, rkey, record }) } catch {} |
| 579 | | return { following: true, rkey } |
| 563 | // ⛔ THE TOGGLE IS NOT OURS TO REIMPLEMENT. This read the existing rkey out of the worklet's index, |
| 564 | // asked hpp to write, then wrote back into that same index — a second follow implementation racing |
| 565 | // favela-web's, with its own idea of what "already following" means. /api/follow does all three |
| 566 | // steps against the one index, and its writer.state() check refuses a write whose signed-in viewer |
| 567 | // is not the DID this hpp signs for — a guard the local path never had. |
| 568 | const r = await (await api()).follow({ did }) |
| 569 | return { following: !!(r && r.following) } |
| 580 | 570 | }, |
| 581 | 571 | // invite — mint an adopt invite for ANOTHER device. Symmetric by design: keet's `addDevice` is |
| 582 | 572 | // available to every active member and confers nothing on the adder, and coownerInvite no longer |