OverviewFilesCommitsIssuesDocsReleasesPeers

atproto-store commits / 4238893

stores: alwaysDuplicate:false — make redundant puts no-op (kill the balloon class)
All three Hyperbee-backed stores (RepoStore, BlobStore, AccountRegistry) were constructed with Hyperbee's alwaysDuplicate at its TRUE default, which DISABLES its byte-identical value-dedup short-circuit. So a re-put of an unchanged (key,value) appended a fresh B-tree path to the append-only core instead of being a no-op read. This is the primitive-level enabler beneath the storage leak: a projection re-putting the whole record set on a 2s timer ballooned a RepoStore core to ~1.3M mostly-dead blocks; a replicating consumer then re-wrote ~1.2GB at ~100MB/s. Set alwaysDuplicate:false on all three. Redundant re-puts (from any caller — serve-membrane projection, favela indexPeer, blob re-uploads) now no-op at the storage primitive; genuinely-changed records still append. Verified: 20 identical re-puts leave core.length unchanged; a new record still grows it. Suite green 25/25. Defense-in-depth under the per-caller fixes (projectRepo gate, lease cadence, etc.). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
4238893bb0dda1b395a72ca7a1e1536a726e0b45
by plankatron <fleeky@gmail.com> · 2026-07-17T22:41:45.000Z

Changed files (4) +11 -4

lib/account-registry.js +2 -1
……@@ -19,7 +19,8 @@ export class AccountRegistry {
1919 async ready () {
2020 const core = this._store.get({ name: this._name })
2121 await core.ready()
22 this.bee = new Hyperbee(core, { keyEncoding: 'utf-8', valueEncoding: 'json' })
22 // alwaysDuplicate:false → re-putting an unchanged account record is a no-op, not an append. See repo-store.js.
23 this.bee = new Hyperbee(core, { keyEncoding: 'utf-8', valueEncoding: 'json', alwaysDuplicate: false })
2324 await this.bee.ready()
2425 return this
2526 }
lib/blob-store.js +3 -1
……@@ -26,7 +26,9 @@ export class BlobStore {
2626 async ready () {
2727 const core = this._store.get({ name: this._name })
2828 await core.ready()
29 this.bee = new Hyperbee(core, { keyEncoding: 'utf-8', valueEncoding: 'binary' })
29 // alwaysDuplicate:false → re-uploading identical bytes at the same CID is a no-op, not a re-append of
30 // the full blob value (content-addressed key is stable, so the value is byte-identical). See repo-store.js.
31 this.bee = new Hyperbee(core, { keyEncoding: 'utf-8', valueEncoding: 'binary', alwaysDuplicate: false })
3032 await this.bee.ready()
3133 return this
3234 }
lib/repo-store.js +5 -1
……@@ -38,7 +38,11 @@ export class RepoStore {
3838 // By KEY = replicate another instance's repo (read-only over the swarm); by NAME = our own.
3939 this.core = this._key ? this._store.get({ key: this._key }) : this._store.get({ name: this._name })
4040 await this.core.ready()
41 this.bee = new Hyperbee(this.core, { keyEncoding: 'utf-8', valueEncoding: 'binary' })
41 // alwaysDuplicate:false enables Hyperbee's byte-identical value-dedup — a re-put of an unchanged
42 // (key,value) becomes a no-op READ instead of appending a fresh B-tree path. Without it (the default
43 // TRUE), redundant puts (e.g. a projection re-putting the whole record set on a timer) balloon this
44 // append-only core to millions of dead blocks. Defense-in-depth under every caller.
45 this.bee = new Hyperbee(this.core, { keyEncoding: 'utf-8', valueEncoding: 'binary', alwaysDuplicate: false })
4246 await this.bee.ready()
4347 return this
4448 }
package.json +1 -1
……@@ -1,6 +1,6 @@
11{
22 "name": "atproto-store",
3 "version": "0.0.1",
3 "version": "0.0.2",
44 "description": "Holepunch-native atproto repo store — records in a classic Hyperbee (collection/rkey -> DAG-CBOR). The P2P home; the atproto-repo module is the bridge serializer over it.",
55 "type": "module",
66 "main": "lib/repo-store.js",