| … | … | @@ -13,6 +13,9 @@ import https from 'node:https' |
| 13 | 13 | import hypercore_crypto from 'hypercore-crypto' |
| 14 | 14 | import nacl from 'tweetnacl' |
| 15 | 15 | |
| 16 | // Increase concurrent HTTP connections limit for blob fetching |
| 17 | https.globalAgent.maxSockets = 50 |
| 18 | |
| 16 | 19 | let store = null |
| 17 | 20 | let metaBee = null // For key-value metadata |
| 18 | 21 | let blobBee = null // For content-addressable blobs |
| … | … | @@ -24,6 +27,7 @@ const MAX_CACHED_MESSAGES = 10000 |
| 24 | 27 | let hashLog = [] |
| 25 | 28 | let openedLog = [] |
| 26 | 29 | let hashSet = new Set() // O(1) duplicate checking |
| 30 | const inFlightFetches = new Map() // Track pending blob fetches |
| 27 | 31 | |
| 28 | 32 | export const apds = {} |
| 29 | 33 | |
| … | … | @@ -162,44 +166,57 @@ apds.get = async (hash) => { |
| 162 | 166 | const node = await blobBee.get(hash) |
| 163 | 167 | if (node) return node.value |
| 164 | 168 | |
| 165 | | // If not found locally, try fetching from wiredove.net |
| 166 | | try { |
| 167 | | const blob = await new Promise((resolve, reject) => { |
| 168 | | const req = https.get(`https://pub.wiredove.net/${hash}`, { timeout: 5000 }, (res) => { |
| 169 | | if (res.statusCode === 404) { |
| 170 | | resolve(null) |
| 171 | | return |
| 172 | | } |
| 169 | // Check if already fetching this blob |
| 170 | if (inFlightFetches.has(hash)) { |
| 171 | return await inFlightFetches.get(hash) |
| 172 | } |
| 173 | 173 | |
| 174 | | if (res.statusCode !== 200) { |
| 174 | // Start new fetch and track it |
| 175 | const fetchPromise = (async () => { |
| 176 | try { |
| 177 | const blob = await new Promise((resolve, reject) => { |
| 178 | const req = https.get(`https://pub.wiredove.net/${hash}`, { timeout: 5000 }, (res) => { |
| 179 | if (res.statusCode === 404) { |
| 180 | resolve(null) |
| 181 | return |
| 182 | } |
| 183 | |
| 184 | if (res.statusCode !== 200) { |
| 185 | resolve(null) |
| 186 | return |
| 187 | } |
| 188 | |
| 189 | let data = '' |
| 190 | res.on('data', chunk => data += chunk) |
| 191 | res.on('end', () => resolve(data)) |
| 192 | res.on('error', () => resolve(null)) |
| 193 | }) |
| 194 | |
| 195 | req.on('error', () => resolve(null)) |
| 196 | req.on('timeout', () => { |
| 197 | req.destroy() |
| 175 | 198 | resolve(null) |
| 176 | | return |
| 177 | | } |
| 178 | | |
| 179 | | let data = '' |
| 180 | | res.on('data', chunk => data += chunk) |
| 181 | | res.on('end', () => resolve(data)) |
| 182 | | res.on('error', () => resolve(null)) |
| 183 | | }) |
| 184 | | |
| 185 | | req.on('error', () => resolve(null)) |
| 186 | | req.on('timeout', () => { |
| 187 | | req.destroy() |
| 188 | | resolve(null) |
| 199 | }) |
| 189 | 200 | }) |
| 190 | | }) |
| 191 | 201 | |
| 192 | | if (blob) { |
| 193 | | // Cache it locally for future use |
| 194 | | await blobBee.put(hash, blob) |
| 195 | | console.log(`📥 Fetched blob from wiredove.net: ${hash.substring(0, 16)}...`) |
| 196 | | return blob |
| 202 | if (blob) { |
| 203 | // Cache it locally for future use |
| 204 | await blobBee.put(hash, blob) |
| 205 | console.log(`📥 Fetched blob from wiredove.net: ${hash.substring(0, 16)}...`) |
| 206 | return blob |
| 207 | } |
| 208 | } catch (err) { |
| 209 | // Silently fail and return null |
| 210 | } finally { |
| 211 | // Remove from in-flight tracking |
| 212 | inFlightFetches.delete(hash) |
| 197 | 213 | } |
| 198 | | } catch (err) { |
| 199 | | // Silently fail and return null |
| 200 | | } |
| 201 | 214 | |
| 202 | | return null |
| 215 | return null |
| 216 | })() |
| 217 | |
| 218 | inFlightFetches.set(hash, fetchPromise) |
| 219 | return await fetchPromise |
| 203 | 220 | } |
| 204 | 221 | |
| 205 | 222 | /** |
| … | … | @@ -283,7 +300,7 @@ apds.add = async (msg) => { |
| 283 | 300 | } |
| 284 | 301 | |
| 285 | 302 | /** |
| 286 | | * Query messages (with lazy text loading for first 20 results) |
| 303 | * Query messages (with parallel lazy text loading for first 20 results) |
| 287 | 304 | */ |
| 288 | 305 | apds.query = async (query) => { |
| 289 | 306 | let results = query ? |
| … | … | @@ -293,19 +310,26 @@ apds.query = async (query) => { |
| 293 | 310 | ) : |
| 294 | 311 | openedLog |
| 295 | 312 | |
| 296 | | // Lazy-load missing text for first 20 messages only (to avoid slow initial load) |
| 313 | // Lazy-load missing text for first 20 messages in parallel |
| 297 | 314 | const limit = Math.min(20, results.length) |
| 315 | const fetchPromises = [] |
| 316 | |
| 298 | 317 | for (let i = 0; i < limit; i++) { |
| 299 | 318 | const msg = results[i] |
| 300 | 319 | if (msg.text === null && msg.opened && msg.opened.length >= 13) { |
| 301 | 320 | const contentHash = msg.opened.substring(13) |
| 302 | | const text = await apds.get(contentHash) |
| 303 | | if (text) { |
| 304 | | msg.text = text // Update in-memory cache |
| 305 | | } |
| 321 | // Fetch in parallel |
| 322 | fetchPromises.push( |
| 323 | apds.get(contentHash).then(text => { |
| 324 | if (text) msg.text = text // Update in-memory cache |
| 325 | }) |
| 326 | ) |
| 306 | 327 | } |
| 307 | 328 | } |
| 308 | 329 | |
| 330 | // Wait for all fetches to complete in parallel |
| 331 | await Promise.all(fetchPromises) |
| 332 | |
| 309 | 333 | return results |
| 310 | 334 | } |
| 311 | 335 | |