| … | … | @@ -0,0 +1,336 @@ |
| 1 | #!/usr/bin/env node |
| 2 | /** |
| 3 | * Comprehensive Relay Sync Test |
| 4 | * |
| 5 | * Tests the full pipeline: |
| 6 | * 1. Pull existing posts from relay |
| 7 | * 2. Check which local posts relay doesn't have (diff) |
| 8 | * 3. Push only new posts |
| 9 | * 4. Verify deduplication works |
| 10 | * 5. Test content-by-hash retrieval |
| 11 | */ |
| 12 | |
| 13 | import https from 'https' |
| 14 | import { ANProto } from './lib/anproto.js' |
| 15 | import { YAMLHelper } from './lib/yaml-helper.js' |
| 16 | import Corestore from 'corestore' |
| 17 | import Hyperbee from 'hyperbee' |
| 18 | import b4a from 'b4a' |
| 19 | import { RelayClient } from './lib/relay-client.js' |
| 20 | |
| 21 | const RELAY_URL = 'https://7rnx.net/relay' |
| 22 | const STORAGE_PATH = process.env.PEAR_STORAGE || './wirecore-storage' |
| 23 | |
| 24 | // Color codes for output |
| 25 | const colors = { |
| 26 | reset: '\x1b[0m', |
| 27 | green: '\x1b[32m', |
| 28 | yellow: '\x1b[33m', |
| 29 | blue: '\x1b[34m', |
| 30 | red: '\x1b[31m', |
| 31 | gray: '\x1b[90m' |
| 32 | } |
| 33 | |
| 34 | function log(emoji, message, color = 'reset') { |
| 35 | console.log(`${colors[color]}${emoji} ${message}${colors.reset}`) |
| 36 | } |
| 37 | |
| 38 | class RelaySyncTest { |
| 39 | constructor() { |
| 40 | this.store = null |
| 41 | this.metaBee = null |
| 42 | this.contentCache = null |
| 43 | this.myFeed = null |
| 44 | this.anprotoKeypair = null |
| 45 | this.relayClient = new RelayClient(RELAY_URL) |
| 46 | this.stats = { |
| 47 | relayTotal: 0, |
| 48 | relayOurs: 0, |
| 49 | localTotal: 0, |
| 50 | localNotInRelay: 0, |
| 51 | duplicates: 0, |
| 52 | pushed: 0, |
| 53 | pulled: 0 |
| 54 | } |
| 55 | } |
| 56 | |
| 57 | async init() { |
| 58 | log('🔧', 'Initializing storage...', 'blue') |
| 59 | |
| 60 | // Open Corestore |
| 61 | this.store = new Corestore(STORAGE_PATH) |
| 62 | await this.store.ready() |
| 63 | |
| 64 | // Load metadata |
| 65 | this.metaBee = new Hyperbee(this.store.get({ name: 'metadata' }), { |
| 66 | keyEncoding: 'utf-8', |
| 67 | valueEncoding: 'json' |
| 68 | }) |
| 69 | await this.metaBee.ready() |
| 70 | |
| 71 | // Load content cache |
| 72 | this.contentCache = new Hyperbee(this.store.get({ name: 'content-cache' }), { |
| 73 | keyEncoding: 'utf-8', |
| 74 | valueEncoding: 'utf-8' |
| 75 | }) |
| 76 | await this.contentCache.ready() |
| 77 | |
| 78 | // Load ANProto keypair |
| 79 | const keypairNode = await this.metaBee.get('anproto-keypair') |
| 80 | if (!keypairNode) { |
| 81 | throw new Error('Not initialized. Run: pear run . cli init') |
| 82 | } |
| 83 | |
| 84 | const keypairStr = keypairNode.value |
| 85 | this.anprotoKeypair = ANProto.loadKeypair(keypairStr) |
| 86 | |
| 87 | // Load own feed |
| 88 | this.myFeed = this.store.get({ |
| 89 | publicKey: this.anprotoKeypair.publicKey, |
| 90 | secretKey: this.anprotoKeypair.secretKey |
| 91 | }) |
| 92 | await this.myFeed.ready() |
| 93 | |
| 94 | const pubkey = b4a.toString(this.anprotoKeypair.publicKey, 'base64') |
| 95 | log('✅', `Initialized as: ${pubkey.substring(0, 20)}...`, 'green') |
| 96 | } |
| 97 | |
| 98 | async pullAllFromRelay() { |
| 99 | log('📥', 'Step 1: Pulling all posts from relay...', 'blue') |
| 100 | |
| 101 | const posts = await this.relayClient.pull() |
| 102 | this.stats.relayTotal = posts.length |
| 103 | |
| 104 | log('✅', `Relay has ${this.stats.relayTotal} total posts`, 'green') |
| 105 | |
| 106 | // Count how many are ours |
| 107 | const ourPubkey = b4a.toString(this.anprotoKeypair.publicKey, 'base64') |
| 108 | this.stats.relayOurs = posts.filter(p => p.author === ourPubkey).length |
| 109 | |
| 110 | log('📊', ` Our posts on relay: ${this.stats.relayOurs}`, 'gray') |
| 111 | |
| 112 | return posts |
| 113 | } |
| 114 | |
| 115 | async getLocalPosts() { |
| 116 | log('📖', 'Step 2: Reading local feed...', 'blue') |
| 117 | |
| 118 | const localPosts = [] |
| 119 | |
| 120 | for (let i = 0; i < this.myFeed.length; i++) { |
| 121 | try { |
| 122 | const block = await this.myFeed.get(i) |
| 123 | const entry = JSON.parse(block.toString()) |
| 124 | |
| 125 | if (entry.yaml) { |
| 126 | const contentHash = entry.contentHash || ANProto.hash(entry.yaml) |
| 127 | localPosts.push({ |
| 128 | index: i, |
| 129 | yaml: entry.yaml, |
| 130 | contentHash, |
| 131 | timestamp: entry.timestamp |
| 132 | }) |
| 133 | } |
| 134 | } catch (err) { |
| 135 | log('⚠️', ` Skipping block ${i}: ${err.message}`, 'yellow') |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | this.stats.localTotal = localPosts.length |
| 140 | log('✅', `Local feed has ${this.stats.localTotal} posts`, 'green') |
| 141 | |
| 142 | return localPosts |
| 143 | } |
| 144 | |
| 145 | async diffLocalVsRelay(localPosts, relayPosts) { |
| 146 | log('🔍', 'Step 3: Computing diff (what relay is missing)...', 'blue') |
| 147 | |
| 148 | // Build hash set of relay content |
| 149 | const relayHashes = new Set() |
| 150 | for (const post of relayPosts) { |
| 151 | if (post.text) { |
| 152 | const hash = ANProto.hash(post.text) |
| 153 | relayHashes.add(hash) |
| 154 | } |
| 155 | } |
| 156 | |
| 157 | log('📊', ` Relay content hashes: ${relayHashes.size}`, 'gray') |
| 158 | |
| 159 | // Find local posts not in relay |
| 160 | const missing = [] |
| 161 | const duplicates = [] |
| 162 | |
| 163 | for (const post of localPosts) { |
| 164 | if (relayHashes.has(post.contentHash)) { |
| 165 | duplicates.push(post) |
| 166 | } else { |
| 167 | missing.push(post) |
| 168 | } |
| 169 | } |
| 170 | |
| 171 | this.stats.localNotInRelay = missing.length |
| 172 | this.stats.duplicates = duplicates.length |
| 173 | |
| 174 | log('✅', `Diff complete:`, 'green') |
| 175 | log(' ', ` Duplicates (already on relay): ${this.stats.duplicates}`, 'gray') |
| 176 | log(' ', ` Missing (need to push): ${this.stats.localNotInRelay}`, 'yellow') |
| 177 | |
| 178 | return { missing, duplicates } |
| 179 | } |
| 180 | |
| 181 | async pushMissingPosts(missingPosts) { |
| 182 | log('📤', `Step 4: Pushing ${missingPosts.length} missing posts to relay...`, 'blue') |
| 183 | |
| 184 | if (missingPosts.length === 0) { |
| 185 | log('✅', 'Nothing to push - all posts already on relay!', 'green') |
| 186 | return |
| 187 | } |
| 188 | |
| 189 | let pushed = 0 |
| 190 | let errors = 0 |
| 191 | |
| 192 | for (const post of missingPosts) { |
| 193 | try { |
| 194 | // Create signature |
| 195 | const signature = ANProto.sign(post.contentHash, this.anprotoKeypair) |
| 196 | const pubkey = b4a.toString(this.anprotoKeypair.publicKey, 'base64') |
| 197 | |
| 198 | // Push to relay |
| 199 | await this.relayClient.push(pubkey, post.yaml, signature, null) |
| 200 | pushed++ |
| 201 | log('✅', ` Pushed post ${post.index}: ${post.yaml.substring(0, 40)}...`, 'gray') |
| 202 | } catch (err) { |
| 203 | errors++ |
| 204 | log('❌', ` Failed to push post ${post.index}: ${err.message}`, 'red') |
| 205 | } |
| 206 | } |
| 207 | |
| 208 | this.stats.pushed = pushed |
| 209 | log('✅', `Push complete: ${pushed} pushed, ${errors} errors`, 'green') |
| 210 | } |
| 211 | |
| 212 | async verifyRelayUpdated() { |
| 213 | log('🔍', 'Step 5: Verifying relay was updated...', 'blue') |
| 214 | |
| 215 | // Wait a bit for relay to process |
| 216 | await new Promise(resolve => setTimeout(resolve, 2000)) |
| 217 | |
| 218 | // Pull again |
| 219 | const posts = await this.relayClient.pull() |
| 220 | const newTotal = posts.length |
| 221 | |
| 222 | const ourPubkey = b4a.toString(this.anprotoKeypair.publicKey, 'base64') |
| 223 | const ourCount = posts.filter(p => p.author === ourPubkey).length |
| 224 | |
| 225 | const increase = newTotal - this.stats.relayTotal |
| 226 | const ourIncrease = ourCount - this.stats.relayOurs |
| 227 | |
| 228 | log('✅', `Verification:`, 'green') |
| 229 | log(' ', ` Relay total: ${this.stats.relayTotal} → ${newTotal} (+${increase})`, 'gray') |
| 230 | log(' ', ` Our posts: ${this.stats.relayOurs} → ${ourCount} (+${ourIncrease})`, 'gray') |
| 231 | |
| 232 | if (ourIncrease === this.stats.pushed) { |
| 233 | log('✅', ' Perfect! All pushed posts appeared on relay', 'green') |
| 234 | } else if (ourIncrease > 0) { |
| 235 | log('⚠️', ` Partial success: expected +${this.stats.pushed}, got +${ourIncrease}`, 'yellow') |
| 236 | } else { |
| 237 | log('❌', ' No new posts on relay - push may have failed', 'red') |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | async testContentByHashRetrieval() { |
| 242 | log('🧪', 'Step 6: Testing content-by-hash retrieval...', 'blue') |
| 243 | |
| 244 | // Pick a random local post |
| 245 | if (this.myFeed.length === 0) { |
| 246 | log('⚠️', ' No local posts to test', 'yellow') |
| 247 | return |
| 248 | } |
| 249 | |
| 250 | const randomIndex = Math.floor(Math.random() * this.myFeed.length) |
| 251 | const block = await this.myFeed.get(randomIndex) |
| 252 | const entry = JSON.parse(block.toString()) |
| 253 | |
| 254 | if (!entry.yaml) { |
| 255 | log('⚠️', ' Selected post has no YAML', 'yellow') |
| 256 | return |
| 257 | } |
| 258 | |
| 259 | const contentHash = entry.contentHash || ANProto.hash(entry.yaml) |
| 260 | |
| 261 | // Check contentCache |
| 262 | const cached = await this.contentCache.get(contentHash) |
| 263 | |
| 264 | if (cached) { |
| 265 | log('✅', ` Content found in cache: ${contentHash.substring(0, 20)}...`, 'green') |
| 266 | log(' ', ` Preview: ${cached.value.substring(0, 50)}...`, 'gray') |
| 267 | } else { |
| 268 | log('❌', ` Content NOT in cache: ${contentHash.substring(0, 20)}...`, 'red') |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | printSummary() { |
| 273 | console.log('\n' + '='.repeat(60)) |
| 274 | log('📊', 'SYNC TEST SUMMARY', 'blue') |
| 275 | console.log('='.repeat(60)) |
| 276 | |
| 277 | console.log(` |
| 278 | ${colors.gray}Relay State:${colors.reset} |
| 279 | Total posts on relay: ${this.stats.relayTotal} |
| 280 | Our posts on relay: ${this.stats.relayOurs} |
| 281 | |
| 282 | ${colors.gray}Local State:${colors.reset} |
| 283 | Total local posts: ${this.stats.localTotal} |
| 284 | |
| 285 | ${colors.gray}Sync Results:${colors.reset} |
| 286 | Already on relay (dupes): ${this.stats.duplicates} |
| 287 | Missing from relay: ${this.stats.localNotInRelay} |
| 288 | Successfully pushed: ${this.stats.pushed} |
| 289 | |
| 290 | ${colors.gray}Efficiency:${colors.reset} |
| 291 | Deduplication rate: ${this.stats.localTotal > 0 ? Math.round(this.stats.duplicates / this.stats.localTotal * 100) : 0}% |
| 292 | Push success rate: ${this.stats.localNotInRelay > 0 ? Math.round(this.stats.pushed / this.stats.localNotInRelay * 100) : 0}% |
| 293 | `) |
| 294 | |
| 295 | console.log('='.repeat(60)) |
| 296 | } |
| 297 | |
| 298 | async cleanup() { |
| 299 | if (this.store) { |
| 300 | await this.store.close() |
| 301 | } |
| 302 | } |
| 303 | |
| 304 | async run() { |
| 305 | try { |
| 306 | await this.init() |
| 307 | |
| 308 | const relayPosts = await this.pullAllFromRelay() |
| 309 | const localPosts = await this.getLocalPosts() |
| 310 | const { missing, duplicates } = await this.diffLocalVsRelay(localPosts, relayPosts) |
| 311 | |
| 312 | await this.pushMissingPosts(missing) |
| 313 | await this.verifyRelayUpdated() |
| 314 | await this.testContentByHashRetrieval() |
| 315 | |
| 316 | this.printSummary() |
| 317 | |
| 318 | log('✅', 'All tests complete!', 'green') |
| 319 | } catch (err) { |
| 320 | log('❌', `Test failed: ${err.message}`, 'red') |
| 321 | console.error(err.stack) |
| 322 | process.exit(1) |
| 323 | } finally { |
| 324 | await this.cleanup() |
| 325 | } |
| 326 | } |
| 327 | } |
| 328 | |
| 329 | // Run the test |
| 330 | const test = new RelaySyncTest() |
| 331 | test.run().then(() => { |
| 332 | process.exit(0) |
| 333 | }).catch(err => { |
| 334 | console.error(err) |
| 335 | process.exit(1) |
| 336 | }) |