| … | … | @@ -1,9 +1,9 @@ |
| 1 | 1 | { |
| 2 | 2 | "name": "favela-peer", |
| 3 | "version": "0.0.43", | |
| 3 | "version": "0.0.44", | |
| 4 | 4 | "runtime": "bare", |
| 5 | "rootTreeHash": "91f004fdec4b38dd4c751e1e99702893b60b2ba5010e4b92015a945fb1803f17", | |
| 6 | "publishedAt": "2026-09-08T19:00:06.696Z", | |
| 5 | "rootTreeHash": "c4f89bd25a183aeab4d047c0496d479dbd3c948142e069c62ee78f8df8e9e0cc", | |
| 6 | "publishedAt": "2026-09-08T19:24:14.838Z", | |
| 7 | 7 | "deps": [ |
| 8 | 8 | { |
| 9 | 9 | "name": "@hyperswarm/secret-stream", |
| … | … | @@ -3017,7 +3017,7 @@ |
| 3017 | 3017 | } |
| 3018 | 3018 | ], |
| 3019 | 3019 | "reason": "version-splits only: 0 code-differs", |
| 3020 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3020 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3021 | 3021 | }, |
| 3022 | 3022 | { |
| 3023 | 3023 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3067,7 +3067,7 @@ |
| 3067 | 3067 | } |
| 3068 | 3068 | ], |
| 3069 | 3069 | "reason": "version-splits only: 0 code-differs", |
| 3070 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3070 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3071 | 3071 | }, |
| 3072 | 3072 | { |
| 3073 | 3073 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3082,7 +3082,7 @@ |
| 3082 | 3082 | } |
| 3083 | 3083 | ], |
| 3084 | 3084 | "reason": "version-splits only: 0 code-differs", |
| 3085 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3085 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3086 | 3086 | }, |
| 3087 | 3087 | { |
| 3088 | 3088 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3107,7 +3107,7 @@ |
| 3107 | 3107 | } |
| 3108 | 3108 | ], |
| 3109 | 3109 | "reason": "version-splits only: 0 code-differs", |
| 3110 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3110 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3111 | 3111 | }, |
| 3112 | 3112 | { |
| 3113 | 3113 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3142,7 +3142,7 @@ |
| 3142 | 3142 | } |
| 3143 | 3143 | ], |
| 3144 | 3144 | "reason": "version-splits only: 0 code-differs", |
| 3145 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3145 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3146 | 3146 | }, |
| 3147 | 3147 | { |
| 3148 | 3148 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3157,7 +3157,7 @@ |
| 3157 | 3157 | } |
| 3158 | 3158 | ], |
| 3159 | 3159 | "reason": "version-splits only: 0 code-differs", |
| 3160 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3160 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3161 | 3161 | }, |
| 3162 | 3162 | { |
| 3163 | 3163 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3172,7 +3172,7 @@ |
| 3172 | 3172 | } |
| 3173 | 3173 | ], |
| 3174 | 3174 | "reason": "version-splits only: 0 code-differs", |
| 3175 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3175 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3176 | 3176 | }, |
| 3177 | 3177 | { |
| 3178 | 3178 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3187,7 +3187,7 @@ |
| 3187 | 3187 | } |
| 3188 | 3188 | ], |
| 3189 | 3189 | "reason": "version-splits only: 0 code-differs", |
| 3190 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3190 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3191 | 3191 | }, |
| 3192 | 3192 | { |
| 3193 | 3193 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3202,7 +3202,7 @@ |
| 3202 | 3202 | } |
| 3203 | 3203 | ], |
| 3204 | 3204 | "reason": "version-splits only: 0 code-differs", |
| 3205 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3205 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3206 | 3206 | }, |
| 3207 | 3207 | { |
| 3208 | 3208 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3232,7 +3232,7 @@ |
| 3232 | 3232 | } |
| 3233 | 3233 | ], |
| 3234 | 3234 | "reason": "version-splits only: 0 code-differs", |
| 3235 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3235 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3236 | 3236 | }, |
| 3237 | 3237 | { |
| 3238 | 3238 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3247,7 +3247,7 @@ |
| 3247 | 3247 | } |
| 3248 | 3248 | ], |
| 3249 | 3249 | "reason": "version-splits only: 0 code-differs", |
| 3250 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3250 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3251 | 3251 | }, |
| 3252 | 3252 | { |
| 3253 | 3253 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3277,7 +3277,7 @@ |
| 3277 | 3277 | } |
| 3278 | 3278 | ], |
| 3279 | 3279 | "reason": "version-splits only: 0 code-differs", |
| 3280 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3280 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3281 | 3281 | }, |
| 3282 | 3282 | { |
| 3283 | 3283 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3307,7 +3307,7 @@ |
| 3307 | 3307 | } |
| 3308 | 3308 | ], |
| 3309 | 3309 | "reason": "version-splits only: 0 code-differs", |
| 3310 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3310 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3311 | 3311 | }, |
| 3312 | 3312 | { |
| 3313 | 3313 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3322,7 +3322,7 @@ |
| 3322 | 3322 | } |
| 3323 | 3323 | ], |
| 3324 | 3324 | "reason": "version-splits only: 0 code-differs", |
| 3325 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3325 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3326 | 3326 | }, |
| 3327 | 3327 | { |
| 3328 | 3328 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3337,7 +3337,7 @@ |
| 3337 | 3337 | } |
| 3338 | 3338 | ], |
| 3339 | 3339 | "reason": "version-splits only: 0 code-differs", |
| 3340 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3340 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3341 | 3341 | }, |
| 3342 | 3342 | { |
| 3343 | 3343 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3367,7 +3367,7 @@ |
| 3367 | 3367 | } |
| 3368 | 3368 | ], |
| 3369 | 3369 | "reason": "version-splits only: 0 code-differs", |
| 3370 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3370 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3371 | 3371 | }, |
| 3372 | 3372 | { |
| 3373 | 3373 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3387,7 +3387,7 @@ |
| 3387 | 3387 | } |
| 3388 | 3388 | ], |
| 3389 | 3389 | "reason": "version-splits only: 0 code-differs", |
| 3390 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3390 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3391 | 3391 | }, |
| 3392 | 3392 | { |
| 3393 | 3393 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3412,7 +3412,7 @@ |
| 3412 | 3412 | } |
| 3413 | 3413 | ], |
| 3414 | 3414 | "reason": "version-splits only: 0 code-differs", |
| 3415 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3415 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3416 | 3416 | }, |
| 3417 | 3417 | { |
| 3418 | 3418 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3427,7 +3427,7 @@ |
| 3427 | 3427 | } |
| 3428 | 3428 | ], |
| 3429 | 3429 | "reason": "version-splits only: 0 code-differs", |
| 3430 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3430 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3431 | 3431 | }, |
| 3432 | 3432 | { |
| 3433 | 3433 | "type": "VERSION_CONFLICT", |
| … | … | @@ -3442,7 +3442,7 @@ |
| 3442 | 3442 | } |
| 3443 | 3443 | ], |
| 3444 | 3444 | "reason": "version-splits only: 0 code-differs", |
| 3445 | "acceptedAt": "2026-09-08T19:00:06.709Z" | |
| 3445 | "acceptedAt": "2026-09-08T19:24:14.856Z" | |
| 3446 | 3446 | } |
| 3447 | 3447 | ] |
| 3448 | 3448 | } |
| … | … | @@ -23,6 +23,8 @@ |
| 23 | 23 | // |
| 24 | 24 | // Design: hyper-social/docs/HOLDING-AND-TELLING.md |
| 25 | 25 | |
| 26 | import { embeds } from 'favela-markdown/lib/render.js' | |
| 27 | ||
| 26 | 28 | const DEFAULT_CONCURRENCY = 4 |
| 27 | 29 | |
| 28 | 30 | /** |
| … | … | @@ -33,7 +35,7 @@ const DEFAULT_CONCURRENCY = 4 |
| 33 | 35 | * than what exists. What is unbounded is the file count, and a freshly followed |
| 34 | 36 | * feed is all of them at once. |
| 35 | 37 | */ |
| 36 | export function createMirror ({ feeds, concurrency = DEFAULT_CONCURRENCY, debounceMs = 2000, log = () => {} }) { | |
| 38 | export function createMirror ({ feeds, files = null, concurrency = DEFAULT_CONCURRENCY, debounceMs = 2000, log = () => {} }) { | |
| 37 | 39 | if (typeof feeds !== 'function') throw new Error('createMirror: feeds() is required') |
| 38 | 40 | |
| 39 | 41 | let stopped = false |
| … | … | @@ -43,7 +45,7 @@ export function createMirror ({ feeds, concurrency = DEFAULT_CONCURRENCY, deboun |
| 43 | 45 | // ⛔ Counted per TRIGGER so a dead hook is OBSERVABLE. 0.0.40's watcher never fired and nothing |
| 44 | 46 | // on a running node could say so — the only evidence was how LONG a record took to appear, which |
| 45 | 47 | // is inference. These counters are what turned that into a measurement. |
| 46 | const fired = { append: 0, connection: 0, boot: 0 } | |
| 48 | const fired = { append: 0, connection: 0, boot: 0, files: 0 } | |
| 47 | 49 | const bases = [] |
| 48 | 50 | const hooked = new Map() |
| 49 | 51 | |
| … | … | @@ -70,6 +72,36 @@ export function createMirror ({ feeds, concurrency = DEFAULT_CONCURRENCY, deboun |
| 70 | 72 | } |
| 71 | 73 | } |
| 72 | 74 | |
| 75 | /** | |
| 76 | * ⛔ LEVEL 2 — the files a record NAMES. | |
| 77 | * | |
| 78 | * A record's own bytes are one oid, carried in the log. The pictures and files inside it are more | |
| 79 | * oids, carried in its markdown. Mirroring only the first leaves a post you can read whose image | |
| 80 | * will not load — the same failure as the outage this module exists for, one level down. | |
| 81 | * Following means holding what somebody published, and a picture is part of what they published. | |
| 82 | * | |
| 83 | * ⚠ Both levels are the SAME store. files.js: "There is NO blob store" — a record and an image | |
| 84 | * are both registerFile/fetchFile by content hash in hyper-overlay. There is one store here. | |
| 85 | * | |
| 86 | * `embeds()` is favela-markdown's real parser over the markdown tree, so this sees exactly what a | |
| 87 | * renderer would and never a regex's idea of it. A file we cannot fetch does NOT undo the record: | |
| 88 | * the record is held and readable, and the picture is missing — which is the honest state one | |
| 89 | * level down, and the next pass retries it. | |
| 90 | */ | |
| 91 | async function mirrorFiles (record) { | |
| 92 | if (!files || typeof files.get !== 'function') return | |
| 93 | const body = record && record.body | |
| 94 | if (!body) return | |
| 95 | let refs | |
| 96 | try { refs = embeds(body) } catch { return } | |
| 97 | for (const e of refs) { | |
| 98 | if (stopped) return | |
| 99 | try { | |
| 100 | if (await files.get(e.oid)) fired.files++ | |
| 101 | } catch { /* missing, not fatal — the record stands */ } | |
| 102 | } | |
| 103 | } | |
| 104 | ||
| 73 | 105 | async function mirrorFeed (key, feed) { |
| 74 | 106 | // ⛔ `pointers()` and not `list()` or `unresolved()`. Both of those RESOLVE every record, which |
| 75 | 107 | // means both of them fetch — unbounded, all at once. Enumerating without fetching is what lets |
| … | … | @@ -113,7 +145,7 @@ export function createMirror ({ feeds, concurrency = DEFAULT_CONCURRENCY, deboun |
| 113 | 145 | // is gone: the record stays unresolved, the surface says so, and the next pass retries. |
| 114 | 146 | const got = await feed.store.ensureLocal(r.collection, r.name, r.pointer && r.pointer.$oid) |
| 115 | 147 | if (got == null) failed++ |
| 116 | else fetched++ | |
| 148 | else { fetched++; await mirrorFiles(got) } | |
| 117 | 149 | } catch { |
| 118 | 150 | failed++ |
| 119 | 151 | } |
| … | … | @@ -225,6 +225,10 @@ export class FavelaNode { |
| 225 | 225 | // `following()` rather than every readable feed. `peer.timeline` draws the same line for the |
| 226 | 226 | // same reason. See hyper-social/docs/HOLDING-AND-TELLING.md. |
| 227 | 227 | this._mirror = createMirror({ |
| 228 | // ⛔ LEVEL 2. A record's bytes are one oid in the log; the files it NAMES are more oids inside | |
| 229 | // its markdown, in the SAME overlay. Without this a mirrored post reads fine and its picture | |
| 230 | // will not load — the outage this exists for, one level down. | |
| 231 | files: this.files, | |
| 228 | 232 | feeds: () => { |
| 229 | 233 | const out = [] |
| 230 | 234 | for (const [name, feed] of this.peer.feeds) { |
| … | … | @@ -1,6 +1,6 @@ |
| 1 | 1 | { |
| 2 | 2 | "name": "favela-peer", |
| 3 | "version": "0.0.44", | |
| 3 | "version": "0.0.45", | |
| 4 | 4 | "description": "The favela daemon — feeds and documents of markdown records over Hypercore. No atproto, no ActivityPub.", |
| 5 | 5 | "type": "module", |
| 6 | 6 | "scripts": { |
| … | … | @@ -266,3 +266,65 @@ test('★★ a peer connecting retries what could not be fetched', async (t) => |
| 266 | 266 | t.is(m.stats().connection, 1, 'and the connection trigger is what did it') |
| 267 | 267 | }) |
| 268 | 268 | |
| 269 | ||
| 270 | // ⛔ LEVEL 2. A record's own bytes are one oid in the log; the pictures and files it names are more | |
| 271 | // oids, inside its markdown. Mirroring only the first leaves a post you can read with an image that | |
| 272 | // will not load — the same failure as §0's outage, one level down. Following means holding what | |
| 273 | // somebody published, and a picture is part of what they published. | |
| 274 | // | |
| 275 | // ⚠ Both levels are the SAME hyper-overlay: files.js says "There is NO blob store", and a record | |
| 276 | // and an image are both registerFile/fetchFile by content hash. There is one store here, not two. | |
| 277 | test('★★ it mirrors the files a record names, not just the record', async (t) => { | |
| 278 | const gotOids = [] | |
| 279 | const body = 'look\n\n + ')\n\n' + | |
| 280 | '[the pdf](favela://blob/' + 'b'.repeat(32) + ')\n' | |
| 281 | const feed = { | |
| 282 | async update () {}, | |
| 283 | async pointers () { return [{ key: 'post/p1', collection: 'post', name: 'p1', pointer: { $oid: 'rec' } }] }, | |
| 284 | store: { | |
| 285 | async get () { return null }, | |
| 286 | async ensureLocal () { return { frontmatter: {}, body } } | |
| 287 | } | |
| 288 | } | |
| 289 | const files = { async get (oid) { gotOids.push(oid); return { path: '/x', mimeType: 'image/png', size: 1 } } } | |
| 290 | ||
| 291 | const m = createMirror({ feeds: () => [{ key: 'f1', feed }], files, log: () => {} }) | |
| 292 | await m.pass() | |
| 293 | ||
| 294 | t.alike(gotOids.sort(), ['a'.repeat(32), 'b'.repeat(32)], 'the image AND the linked file') | |
| 295 | t.is(m.stats().files, 2, 'counted separately from records') | |
| 296 | }) | |
| 297 | ||
| 298 | test('★ a record naming no files costs no extra fetches', async (t) => { | |
| 299 | let calls = 0 | |
| 300 | const feed = { | |
| 301 | async update () {}, | |
| 302 | async pointers () { return [{ key: 'post/p1', collection: 'post', name: 'p1', pointer: { $oid: 'r' } }] }, | |
| 303 | store: { async get () { return null }, async ensureLocal () { return { frontmatter: {}, body: 'just words' } } } | |
| 304 | } | |
| 305 | const m = createMirror({ feeds: () => [{ key: 'f1', feed }], files: { async get () { calls++; return null } }, log: () => {} }) | |
| 306 | await m.pass() | |
| 307 | t.is(calls, 0) | |
| 308 | }) | |
| 309 | ||
| 310 | // ⛔ A file we cannot fetch must not fail the record that named it. The record is already held and | |
| 311 | // readable; the picture is simply missing, which is the honest UNRESOLVED state one level down. | |
| 312 | test('⛔ an unfetchable file does not undo the record', async (t) => { | |
| 313 | const feed = { | |
| 314 | async update () {}, | |
| 315 | async pointers () { return [{ key: 'post/p1', collection: 'post', name: 'p1', pointer: { $oid: 'r' } }] }, | |
| 316 | store: { | |
| 317 | async get () { return null }, | |
| 318 | async ensureLocal () { return { frontmatter: {}, body: ' + ')' } } | |
| 319 | } | |
| 320 | } | |
| 321 | const m = createMirror({ | |
| 322 | feeds: () => [{ key: 'f1', feed }], | |
| 323 | files: { async get () { throw new Error('no holder online') } }, | |
| 324 | log: () => {} | |
| 325 | }) | |
| 326 | ||
| 327 | const r = await m.pass() | |
| 328 | t.is(r.fetched, 1, 'the record still counts as mirrored') | |
| 329 | t.is(m.stats().files, 0, 'and the file did not') | |
| 330 | }) |