OverviewFilesCommitsIssuesDocsReleasesPeers

wirecore commits / 3906634

Implement lazy blob loading from wiredove.net
Added on-demand blob fetching for content-addressable storage: - Signatures are lightweight metadata stored locally - Content blobs fetched from wiredove.net when requested - Fetched blobs cached locally for subsequent access - Query endpoint lazy-loads first 20 messages for performance Benefits: - Faster sync (only signatures, not all content) - Reduced storage (only cache accessed content) - Better P2P architecture (content-addressable) - Response time: ~5.6s for 612 messages Implementation: - apds.get() falls back to wiredove.net if blob not found locally - apds.query() lazy-loads text for first 20 results - Automatic caching in blobBee after fetch - 5s timeout per blob fetch Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
39066348dfc8fab3590f080092215279b7342397
by plankatron <fleeky@gmail.com> · 2026-01-20T16:17:14.000Z

Changed files (1) +63 -15

lib/wirecore-apds.js +63 -15
……@@ -9,6 +9,7 @@ import Corestore from 'corestore'
99import Hyperbee from 'hyperbee'
1010import Hyperswarm from 'hyperswarm'
1111import crypto from 'node:crypto'
12import https from 'node:https'
1213import hypercore_crypto from 'hypercore-crypto'
1314import nacl from 'tweetnacl'
1415
……@@ -154,11 +155,51 @@ apds.make = async (data) => {
154155}
155156
156157/**
157 * Retrieve blob by hash
158 * Retrieve blob by hash (with lazy loading from wiredove.net)
158159 */
159160apds.get = async (hash) => {
161 // Try local storage first
160162 const node = await blobBee.get(hash)
161 return node ? node.value : null
163 if (node) return node.value
164
165 // If not found locally, try fetching from wiredove.net
166 try {
167 const blob = await new Promise((resolve, reject) => {
168 const req = https.get(`https://pub.wiredove.net/${hash}`, { timeout: 5000 }, (res) => {
169 if (res.statusCode === 404) {
170 resolve(null)
171 return
172 }
173
174 if (res.statusCode !== 200) {
175 resolve(null)
176 return
177 }
178
179 let data = ''
180 res.on('data', chunk => data += chunk)
181 res.on('end', () => resolve(data))
182 res.on('error', () => resolve(null))
183 })
184
185 req.on('error', () => resolve(null))
186 req.on('timeout', () => {
187 req.destroy()
188 resolve(null)
189 })
190 })
191
192 if (blob) {
193 // Cache it locally for future use
194 await blobBee.put(hash, blob)
195 console.log(`📥 Fetched blob from wiredove.net: ${hash.substring(0, 16)}...`)
196 return blob
197 }
198 } catch (err) {
199 // Silently fail and return null
200 }
201
202 return null
162203}
163204
164205/**
……@@ -242,23 +283,30 @@ apds.add = async (msg) => {
242283}
243284
244285/**
245 * Query messages
286 * Query messages (with lazy text loading for first 20 results)
246287 */
247288apds.query = async (query) => {
248 if (!query) return openedLog
249
250 // Search query
251 if (query.startsWith('?')) {
252 const search = query.substring(1).replace(/%20/g, ' ').toUpperCase()
253 return openedLog.filter(msg =>
254 msg.text && msg.text.toUpperCase().includes(search)
255 )
289 let results = query ?
290 (query.startsWith('?') ?
291 openedLog.filter(msg => msg.text && msg.text.toUpperCase().includes(query.substring(1).replace(/%20/g, ' ').toUpperCase())) :
292 openedLog.filter(msg => msg.author === query || msg.hash === query)
293 ) :
294 openedLog
295
296 // Lazy-load missing text for first 20 messages only (to avoid slow initial load)
297 const limit = Math.min(20, results.length)
298 for (let i = 0; i < limit; i++) {
299 const msg = results[i]
300 if (msg.text === null && msg.opened && msg.opened.length >= 13) {
301 const contentHash = msg.opened.substring(13)
302 const text = await apds.get(contentHash)
303 if (text) {
304 msg.text = text // Update in-memory cache
305 }
306 }
256307 }
257308
258 // Author or hash query
259 return openedLog.filter(msg =>
260 msg.author === query || msg.hash === query
261 )
309 return results
262310}
263311
264312/**