| … | … | @@ -54,9 +54,10 @@ async function post(text) { |
| 54 | 54 | const contentHash = ANProto.hashData(yaml) |
| 55 | 55 | // → "pZGm1Av0IEBKARczz7exkNYsZb8LzaMrV7J32a2fFG4=" |
| 56 | 56 | |
| 57 | | // 3. Store content in local cache |
| 58 | | await contentStore.put(contentHash, yaml) |
| 59 | | // → Writes to {storage}/content.json |
| 57 | // 3. Store content in Hyperdrive by hash |
| 58 | const hashPath = `/${contentHash.substring(0, 2)}/${contentHash}` |
| 59 | await contentDrive.put(hashPath, Buffer.from(yaml)) |
| 60 | // → Writes to cores/content-drive/{hash} |
| 60 | 61 | |
| 61 | 62 | // 4. Sign the hash with ANProto |
| 62 | 63 | const signature = ANProto.signHash(contentHash, anprotoKeypair) |
| … | … | @@ -83,15 +84,19 @@ async function getTimeline(limit) { |
| 83 | 84 | // 2. Verify signature and extract hash |
| 84 | 85 | const { timestamp, hash } = ANProto.verifySignature(signature) |
| 85 | 86 | |
| 86 | | // 3. Try to get content from local cache |
| 87 | | let content = await contentStore.get(hash) |
| 87 | // 3. Try to get content from Hyperdrive |
| 88 | const hashPath = `/${hash.substring(0, 2)}/${hash}` |
| 89 | let content = await contentDrive.get(hashPath) |
| 88 | 90 | |
| 89 | 91 | // 4. If not cached, request from peers |
| 90 | 92 | if (!content) { |
| 91 | 93 | content = await contentSync.fetch(hash) |
| 92 | | await contentStore.put(hash, content) |
| 94 | // Cache in Hyperdrive |
| 95 | await contentDrive.put(hashPath, content) |
| 93 | 96 | } |
| 94 | 97 | |
| 98 | content = content.toString() |
| 99 | |
| 95 | 100 | // 5. Parse YAML and extract metadata |
| 96 | 101 | const { metadata, text } = YAMLHelper.parse(content) |
| 97 | 102 | |
| … | … | @@ -115,38 +120,37 @@ async function getTimeline(limit) { |
| 115 | 120 | ### Storage Architecture (Refactored) |
| 116 | 121 | |
| 117 | 122 | ``` |
| 118 | | {storage-path}/ |
| 119 | | ├── anproto-keypair.json # ANProto ed25519 keypair (132 chars) |
| 120 | | │ { |
| 121 | | │ "keypair": "BSY7/er4VJIu08o39NaRAiPY/MAvd7oQhlGCRDABjYU=tQa03kqUWG..." |
| 122 | | │ } |
| 123 | {storage-path}/cores/ # Corestore (Hypercore feeds) |
| 124 | │ |
| 125 | ├── my-feed/ # Personal feed (writable) |
| 126 | │ └── data # Contains ONLY ANProto signatures! |
| 127 | │ # Each block: 132 bytes |
| 123 | 128 | │ |
| 124 | | ├── content.json # Content-by-hash cache |
| 125 | | │ { |
| 126 | | │ "pZGm1Av0...": "---\nname: fleeky\nimage: hash...\n---\nHello!", |
| 127 | | │ "anotherHash...": "---\nname: fleeky\n---\nAnother post", |
| 128 | | │ "imageHash...": "base64-encoded-image-data..." |
| 129 | | │ } |
| 129 | ├── {followed-feed-1}/ # Followed user's feed (read-only) |
| 130 | │ └── data # Signatures only, sparse download |
| 130 | 131 | │ |
| 131 | | ├── cores/ # Corestore (Hypercore feeds) |
| 132 | | │ ├── identity/ # IdentityKey seed (for display name) |
| 133 | | │ │ └── (kept for compatibility) |
| 134 | | │ │ |
| 135 | | │ ├── my-feed/ # Personal feed (writable) |
| 136 | | │ │ └── data # Contains ONLY ANProto signatures! |
| 137 | | │ │ # Each block: 132 bytes |
| 138 | | │ │ |
| 139 | | │ ├── {followed-feed-1}/ # Followed user's feed (read-only) |
| 140 | | │ │ └── data # Signatures only, sparse download |
| 141 | | │ │ |
| 142 | | │ └── {followed-feed-N}/ # More followed feeds |
| 132 | ├── {followed-feed-N}/ # More followed feeds |
| 133 | │ |
| 134 | ├── content-drive/ # Hyperdrive: Content by hash |
| 135 | │ └── {hash-prefix}/ # Organized by first 2 chars |
| 136 | │ └── {full-hash} # Each file = YAML content or image |
| 137 | │ # Example: |
| 138 | │ # pZ/pZGm1Av0... = "---\nname: fleeky\n---\nHello!" |
| 139 | │ # aB/aBcd2Ef3... = <base64-image-data> |
| 143 | 140 | │ |
| 144 | 141 | └── metadata/ # Hyperbee for user metadata |
| 145 | | ├── profile # { username, bio, avatar } |
| 142 | ├── anproto-keypair # ANProto ed25519 keypair (132 chars) |
| 143 | ├── profile # { username, bio, avatarHash } |
| 146 | 144 | ├── follow:{pubkey} # Follow list |
| 147 | 145 | └── config # App settings |
| 148 | 146 | ``` |
| 149 | 147 | |
| 148 | **Key Changes from Current**: |
| 149 | - ❌ Removed: `identity/` core (IdentityKey) |
| 150 | - ✅ Added: `content-drive/` (Hyperdrive for ALL content) |
| 151 | - ✅ Added: `anproto-keypair` in metadata Hyperbee |
| 152 | - ✅ Changed: my-feed stores signatures, not full messages |
| 153 | |
| 150 | 154 | ### Network Architecture |
| 151 | 155 | |
| 152 | 156 | ``` |
| … | … | @@ -230,34 +234,70 @@ class YAMLHelper { |
| 230 | 234 | } |
| 231 | 235 | ``` |
| 232 | 236 | |
| 233 | | ### 3. Content Store (`lib/content-store.js`) |
| 237 | ### 3. Content Drive (Hyperdrive Wrapper) |
| 234 | 238 | |
| 235 | | **Copy from**: `lib/core/content-store.js` |
| 239 | **NEW MODULE**: `lib/content-drive.js` |
| 240 | |
| 241 | **Purpose**: Wrap Hyperdrive for content-by-hash storage |
| 236 | 242 | |
| 237 | 243 | **API**: |
| 238 | 244 | ```javascript |
| 239 | | class ContentStore { |
| 240 | | constructor(storagePath) |
| 245 | class ContentDrive { |
| 246 | constructor(corestore, options) |
| 241 | 247 | |
| 242 | | async init() // Load content.json |
| 243 | | async get(hash) → content | null |
| 248 | async ready() // Initialize Hyperdrive |
| 244 | 249 | async put(hash, content) → void |
| 250 | async get(hash) → Buffer | null |
| 245 | 251 | async has(hash) → boolean |
| 246 | 252 | async delete(hash) → void |
| 247 | | async clear() → void |
| 248 | | async size() → number |
| 253 | async list() → AsyncIterator<{ hash, size }> |
| 254 | async close() → void |
| 249 | 255 | } |
| 250 | 256 | ``` |
| 251 | 257 | |
| 252 | | **File Format** (`content.json`): |
| 253 | | ```json |
| 254 | | { |
| 255 | | "hashABC...": "yaml content here", |
| 256 | | "hashDEF...": "another yaml content", |
| 257 | | "hashGHI...": "base64-image-data..." |
| 258 | **Implementation**: |
| 259 | ```javascript |
| 260 | import Hyperdrive from 'hyperdrive' |
| 261 | |
| 262 | class ContentDrive { |
| 263 | constructor(corestore, options = {}) { |
| 264 | this.drive = new Hyperdrive(corestore, { name: 'content-drive' }) |
| 265 | } |
| 266 | |
| 267 | async ready() { |
| 268 | await this.drive.ready() |
| 269 | } |
| 270 | |
| 271 | async put(hash, content) { |
| 272 | const path = `/${hash.substring(0, 2)}/${hash}` |
| 273 | await this.drive.put(path, content) |
| 274 | } |
| 275 | |
| 276 | async get(hash) { |
| 277 | const path = `/${hash.substring(0, 2)}/${hash}` |
| 278 | return await this.drive.get(path) |
| 279 | } |
| 280 | |
| 281 | async has(hash) { |
| 282 | const content = await this.get(hash) |
| 283 | return content !== null |
| 284 | } |
| 285 | |
| 286 | // ... other methods |
| 258 | 287 | } |
| 259 | 288 | ``` |
| 260 | 289 | |
| 290 | **Storage Format** (Hyperdrive): |
| 291 | ``` |
| 292 | content-drive/ |
| 293 | ├── pZ/ |
| 294 | │ └── pZGm1Av0... (YAML content as Buffer) |
| 295 | ├── aB/ |
| 296 | │ └── aBcd2Ef3... (Image data as Buffer) |
| 297 | └── xY/ |
| 298 | └── xYz789... (More content) |
| 299 | ``` |
| 300 | |
| 261 | 301 | ### 4. Content Sync Manager (`lib/content-sync.js`) |
| 262 | 302 | |
| 263 | 303 | **Copy from**: `lib/core/content-sync.js` |
| … | … | @@ -317,48 +357,63 @@ Message Types: |
| 317 | 357 | - ✅ Create refactor plan |
| 318 | 358 | - ⏳ Get user approval on approach |
| 319 | 359 | |
| 320 | | ### Phase 1: Add ANProto Layer (Non-Breaking) |
| 321 | | **Goal**: Add ANProto alongside existing code, no breaking changes |
| 360 | ### Phase 1: Add ANProto Layer (Breaking - Fresh Start) |
| 361 | **Goal**: Replace IdentityKey with ANProto, fresh start for users |
| 322 | 362 | |
| 323 | 363 | **Tasks**: |
| 324 | | 1. Copy `anproto.js` to pear-app |
| 325 | | 2. Add ANProto keypair generation/loading |
| 326 | | - Store in `anproto-keypair.json` |
| 327 | | - Keep IdentityKey for backwards compat |
| 328 | | 3. Add ANProto signing to post() |
| 329 | | - Store signature in message metadata |
| 330 | | - Keep full JSON in Hypercore (for now) |
| 331 | | 4. Add ANProto verification to timeline reading |
| 332 | | - Verify signatures when reading |
| 333 | | - Log warnings if verification fails |
| 364 | 1. Copy `anproto.js` to pear-app/lib/ |
| 365 | 2. Remove IdentityKey dependency |
| 366 | - Remove `import IdentityKey from 'keet-identity-key'` |
| 367 | - Remove `identity/` core usage |
| 368 | - Remove IdentityKey from package.json |
| 369 | 3. Add ANProto keypair generation |
| 370 | - Generate on `init` |
| 371 | - Store in Hyperbee metadata: `anproto-keypair` |
| 372 | - Load on startup |
| 373 | 4. Update `whoami` command to show ANProto publicKey (base64) |
| 374 | 5. Add ANProto signing to post() |
| 375 | - Hash content with ANProto |
| 376 | - Sign hash with ANProto |
| 377 | - Store signature in message metadata (for now) |
| 378 | - Keep full JSON in Hypercore (temporary) |
| 379 | 6. Add ANProto verification to timeline |
| 380 | - Extract signature from message |
| 381 | - Verify with ANProto |
| 382 | - Log warnings if fails |
| 334 | 383 | |
| 335 | 384 | **Testing**: |
| 336 | | - Existing posts still work |
| 337 | | - New posts have ANProto signatures |
| 385 | - Init creates ANProto keypair |
| 386 | - Keypair stored in Hyperbee |
| 387 | - Whoami shows base64 publicKey |
| 388 | - Posts include ANProto signatures |
| 338 | 389 | - Signatures verify correctly |
| 339 | 390 | |
| 340 | | **Rollback**: Remove anproto.js, revert changes |
| 391 | **Note**: This is a BREAKING CHANGE - users must create new identities |
| 392 | **Migration**: Fresh start (no data migration) |
| 341 | 393 | |
| 342 | | ### Phase 2: Add Content Storage (Non-Breaking) |
| 343 | | **Goal**: Add content.json cache, still store full messages in Hypercore |
| 394 | ### Phase 2: Add Content Storage (Hyperdrive) |
| 395 | **Goal**: Add Hyperdrive for content-by-hash, still store full messages in Hypercore |
| 344 | 396 | |
| 345 | 397 | **Tasks**: |
| 346 | | 1. Copy `content-store.js` to pear-app |
| 347 | | 2. Initialize ContentStore on startup |
| 348 | | 3. Store content in content.json when posting |
| 398 | 1. Create `lib/content-drive.js` (Hyperdrive wrapper) |
| 399 | 2. Initialize ContentDrive on startup |
| 400 | - Create in Corestore: `{ name: 'content-drive' }` |
| 401 | - Ready after store initialization |
| 402 | 3. Store content in Hyperdrive when posting |
| 349 | 403 | - Hash content |
| 350 | | - Store in content.json |
| 351 | | - Keep storing full message in Hypercore |
| 352 | | 4. Read from content.json when available |
| 353 | | - Try content cache first |
| 404 | - Store in Hyperdrive: `/{prefix}/{hash}` |
| 405 | - Keep storing full message in Hypercore (for now) |
| 406 | 4. Read from Hyperdrive when available |
| 407 | - Try Hyperdrive first |
| 354 | 408 | - Fall back to Hypercore |
| 355 | 409 | |
| 356 | 410 | **Testing**: |
| 357 | 411 | - Posts work as before |
| 358 | | - Content cached in content.json |
| 359 | | - Timeline reads from cache |
| 412 | - Content cached in Hyperdrive |
| 413 | - Timeline reads from Hyperdrive |
| 414 | - Hyperdrive replicates via Hyperswarm |
| 360 | 415 | |
| 361 | | **Rollback**: Remove content-store.js, revert changes |
| 416 | **Benefit**: Content is now P2P-replicable (unlike content.json) |
| 362 | 417 | |
| 363 | 418 | ### Phase 3: Add YAML Format (Non-Breaking) |
| 364 | 419 | **Goal**: Convert messages to YAML, dual-format reader |
| … | … | @@ -588,27 +643,125 @@ pear-app/test/ |
| 588 | 643 | |
| 589 | 644 | --- |
| 590 | 645 | |
| 591 | | ## Open Questions |
| 646 | ## Decisions Made |
| 647 | |
| 648 | **Updated: 2026-01-21** |
| 649 | |
| 650 | ### 1. Content Storage: Hyperdrive (DECIDED ✓) |
| 651 | |
| 652 | **Decision**: Use Hyperdrive for ALL content (text + images) |
| 653 | |
| 654 | ``` |
| 655 | {storage-path}/cores/ |
| 656 | ├── my-feed/ # Hypercore: ANProto signatures only (132 bytes) |
| 657 | ├── content-drive/ # Hyperdrive: ALL content by hash |
| 658 | │ ├── pZ/ # First 2 chars of hash |
| 659 | │ │ └── pZGm1Av0IEBKARczz7exkNYsZb8LzaMrV7J32a2fFG4= |
| 660 | │ └── aB/ |
| 661 | │ └── aBcd2Ef3... |
| 662 | └── metadata/ # Hyperbee: username, follows, anproto-keypair |
| 663 | ``` |
| 664 | |
| 665 | **Rationale**: |
| 666 | - ✅ Pear app already uses Hyperdrive for images |
| 667 | - ✅ Native P2P replication |
| 668 | - ✅ Sparse downloads (fetch only needed content) |
| 669 | - ✅ Efficient for all sizes (text and images) |
| 670 | - ✅ Consistent: one storage mechanism for all content |
| 592 | 671 | |
| 593 | | 1. **Storage Location**: Should anproto-keypair.json be separate or in Hyperbee? |
| 594 | | - Separate: Easier to export/import |
| 595 | | - Hyperbee: More integrated with other metadata |
| 672 | **Alternative considered**: content.json (original lib/core design) |
| 673 | **Why rejected**: Not P2P-native, poor for large files, loads everything into memory |
| 596 | 674 | |
| 597 | | 2. **Image Storage**: Should large images use Hyperdrive or content.json? |
| 598 | | - content.json: Simpler, consistent |
| 599 | | - Hyperdrive: Better for large files, sparse downloads |
| 675 | ### 2. Identity: ANProto Only (DECIDED ✓) |
| 600 | 676 | |
| 601 | | 3. **Migration**: Force migration or support dual-format forever? |
| 602 | | - Force: Cleaner codebase |
| 603 | | - Dual: Better UX, no breaking change |
| 677 | **Decision**: Remove IdentityKey, use ONLY ANProto keypair |
| 604 | 678 | |
| 605 | | 4. **Identity**: Keep both IdentityKey and ANProto keypair? |
| 606 | | - Both: Backwards compatible |
| 607 | | - ANProto only: Simpler, cleaner |
| 679 | ```javascript |
| 680 | // Store ANProto keypair in Hyperbee metadata |
| 681 | await metaBee.put('anproto-keypair', keypair) // 132-char string |
| 682 | |
| 683 | // Remove: |
| 684 | // - cores/identity/ (IdentityKey seed) |
| 685 | // - import IdentityKey from 'keet-identity-key' |
| 686 | ``` |
| 687 | |
| 688 | **Rationale**: |
| 689 | - ✅ Simpler: one identity system |
| 690 | - ✅ More portable: pure ed25519, not Keet-specific |
| 691 | - ✅ Consistent: ANProto publicKey used everywhere |
| 692 | - ✅ Less storage: no identity core needed |
| 693 | |
| 694 | **Consequences**: |
| 695 | - ❌ No Keet integration (we're not using it anyway) |
| 696 | - ✅ Breaking change acceptable (fresh start migration) |
| 697 | |
| 698 | ### 3. Migration Strategy: Fresh Start (DECIDED ✓) |
| 699 | |
| 700 | **Decision**: Ship refactored version as new release, users create new identities |
| 701 | |
| 702 | **Rationale**: |
| 703 | - ✅ Clean slate: no backwards compatibility burden |
| 704 | - ✅ Simpler implementation: no dual-format reader |
| 705 | - ✅ Better architecture: start with correct design |
| 706 | - ✅ Alpha phase: acceptable to ask users to restart |
| 707 | |
| 708 | **User Experience**: |
| 709 | - Old version continues to work (frozen) |
| 710 | - New version is separate release |
| 711 | - Users choose when to migrate |
| 712 | - Export/import tools for manual data porting (if needed later) |
| 713 | |
| 714 | ### 4. Keypair Storage: Hyperbee (DECIDED ✓) |
| 715 | |
| 716 | **Decision**: Store ANProto keypair in Hyperbee metadata, not separate file |
| 717 | |
| 718 | ```javascript |
| 719 | // Store |
| 720 | await metaBee.put('anproto-keypair', keypair) |
| 721 | |
| 722 | // Load |
| 723 | const keypair = await metaBee.get('anproto-keypair') |
| 724 | ``` |
| 725 | |
| 726 | **Rationale**: |
| 727 | - ✅ Integrated with other metadata (username, follows) |
| 728 | - ✅ Replicated with Hypercore |
| 729 | - ✅ Backed up automatically with Corestore |
| 730 | - ✅ No separate file to manage |
| 731 | |
| 732 | **Alternative considered**: anproto-keypair.json separate file |
| 733 | **Why rejected**: Separate file requires manual backup, not integrated with Corestore |
| 734 | |
| 735 | ### 5. Content Sync: Multi-Transport (DECIDED ✓) |
| 736 | |
| 737 | **Decision**: Use BOTH Hyperswarm and relay for content fetching |
| 738 | |
| 739 | **Implementation**: |
| 740 | ```javascript |
| 741 | // 1. Try local Hyperdrive cache |
| 742 | let content = await contentDrive.get(hashPath) |
| 743 | |
| 744 | // 2. If not cached, try Hyperswarm peers (Protomux) |
| 745 | if (!content) { |
| 746 | content = await contentSync.fetchViaHyperswarm(hash) |
| 747 | } |
| 748 | |
| 749 | // 3. If still missing, try relay WebSocket (APDS) |
| 750 | if (!content) { |
| 751 | content = await contentSync.fetchViaRelay(hash) |
| 752 | } |
| 753 | |
| 754 | // 4. Cache result |
| 755 | if (content) { |
| 756 | await contentDrive.put(hashPath, content) |
| 757 | } |
| 758 | ``` |
| 608 | 759 | |
| 609 | | 5. **Content Sync**: Should relay serve content-by-hash for web users? |
| 610 | | - Yes: Better for web users (no P2P content transport) |
| 611 | | - No: Keep relay simple (signatures only) |
| 760 | **Rationale**: |
| 761 | - ✅ Relay already serves content-by-hash (APDS protocol) |
| 762 | - ✅ Fallback mechanism for content discovery |
| 763 | - ✅ Web users benefit from relay gateway |
| 764 | - ✅ Native users get direct P2P via Hyperswarm |
| 612 | 765 | |
| 613 | 766 | --- |
| 614 | 767 | |