OverviewFilesCommitsIssuesDocsReleasesPeers

hyper-overlay commits / 6651efa

test: conformance vectors for the FastCDC chunker
Second half of the port groundwork, and the half that matters more. A message divergence fails LOUDLY — the frame does not decode. A chunking divergence fails SILENTLY: both implementations produce valid chunks, verify their own hashes, and simply never share one with each other. Dedup drops to zero and nothing errors. 49 assertions: 16 chunk cases, 5 hashChunk cases, 9 tier-selection cases, plus the constants and mask tables. FIVE THINGS THE VECTORS PIN, each a plausible way to get it wrong: 1. THE CHUNK HASH IS NOT PLAIN BLAKE2B. hashChunk is hypercore-crypto's data(): blake2b-256(0x00 || uint64_LE(byteLength) || data). Hashing the bytes alone gives different content hashes for identical bytes — total interop loss, no error anywhere. 2. THE ROLLING HASH IS A VARIANT: hash = ((hash << 1) + GEAR[byte]) >>> 0, shift LEFT where canonical FastCDC shifts right; not reset between mask phases; the first minSize bytes are never hashed (i starts AT minSize). 3. THE CUT IS EXCLUSIVE. On a match the loop breaks without incrementing, so the triggering byte belongs to the NEXT chunk. An off-by-one still yields plausible-looking chunks. 4. THE FINAL-CHUNK TEST IS `remaining <= minSize`, not `<`. 5. TIER MASKS DEGENERATE: large and small are equal for all four tiers, because every avgSize is a power of two. Recorded rather than assumed — a port may implement one mask, but only because this holds, and if a future tier breaks it the port and the assertion must change together. The same 1 MB input forced through tiers 0-3 gives 59 / 13 / 3 / 1 chunks. A port that ignores the tier parameter collapses those to one answer, which is the cheapest possible way to catch it. Inputs are a deterministic LCG spec rather than megabytes of hex, so a port reproduces multi-MB cases from the recorded seed. Generation asserts the chunks TILE the input exactly, and verification re-checks tiling independently — a port could match every recorded boundary and still emit chunks that do not reconstruct the file. Negative control run before committing: adding 1 to a single recorded chunk length takes it to FAIL with the exact offending boundary printed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
6651efafeecae32708b9df96c0d1587364cc6530
by plankatron <fleeky@gmail.com> · 2026-08-10T23:01:46.000Z

Changed files (3) +1478 -0

test/conformance/chunker-vectors.json +1211 -0
……@@ -0,0 +1,1211 @@
1{
2 "generator": "hyper-overlay/test/conformance/generate-chunker.mjs",
3 "overlayVersion": "0.3.12",
4 "prng": "lcg32: s = (s * 1664525 + 1013904223) mod 2^32; byte = (s >>> 24) & 0xff; one byte per step, s starts at seed",
5 "hashConstruction": "blake2b-256(0x00 || uint64_LE(byteLength) || data) — hypercore-crypto crypto.data, NOT plain blake2b",
6 "constants": {
7 "MIN_SIZE": 4096,
8 "AVG_SIZE": 16384,
9 "MAX_SIZE": 65536,
10 "TIERS": [
11 {
12 "minSize": 4096,
13 "avgSize": 16384,
14 "maxSize": 65536
15 },
16 {
17 "minSize": 16384,
18 "avgSize": 65536,
19 "maxSize": 262144
20 },
21 {
22 "minSize": 65536,
23 "avgSize": 262144,
24 "maxSize": 1048576
25 },
26 {
27 "minSize": 262144,
28 "avgSize": 1048576,
29 "maxSize": 4194304
30 }
31 ]
32 },
33 "masks": [
34 {
35 "tier": 0,
36 "avgSize": 16384,
37 "large": 16383,
38 "small": 16383,
39 "equal": true
40 },
41 {
42 "tier": 1,
43 "avgSize": 65536,
44 "large": 65535,
45 "small": 65535,
46 "equal": true
47 },
48 {
49 "tier": 2,
50 "avgSize": 262144,
51 "large": 262143,
52 "small": 262143,
53 "equal": true
54 },
55 {
56 "tier": 3,
57 "avgSize": 1048576,
58 "large": 1048575,
59 "small": 1048575,
60 "equal": true
61 }
62 ],
63 "tierSelection": [
64 {
65 "size": 0,
66 "tier": 0,
67 "params": {
68 "minSize": 4096,
69 "avgSize": 16384,
70 "maxSize": 65536
71 }
72 },
73 {
74 "size": 1,
75 "tier": 0,
76 "params": {
77 "minSize": 4096,
78 "avgSize": 16384,
79 "maxSize": 65536
80 }
81 },
82 {
83 "size": 1048575,
84 "tier": 0,
85 "params": {
86 "minSize": 4096,
87 "avgSize": 16384,
88 "maxSize": 65536
89 }
90 },
91 {
92 "size": 1048576,
93 "tier": 1,
94 "params": {
95 "minSize": 16384,
96 "avgSize": 65536,
97 "maxSize": 262144
98 }
99 },
100 {
101 "size": 104857599,
102 "tier": 1,
103 "params": {
104 "minSize": 16384,
105 "avgSize": 65536,
106 "maxSize": 262144
107 }
108 },
109 {
110 "size": 104857600,
111 "tier": 2,
112 "params": {
113 "minSize": 65536,
114 "avgSize": 262144,
115 "maxSize": 1048576
116 }
117 },
118 {
119 "size": 10737418239,
120 "tier": 2,
121 "params": {
122 "minSize": 65536,
123 "avgSize": 262144,
124 "maxSize": 1048576
125 }
126 },
127 {
128 "size": 10737418240,
129 "tier": 3,
130 "params": {
131 "minSize": 262144,
132 "avgSize": 1048576,
133 "maxSize": 4194304
134 }
135 },
136 {
137 "size": 99999999999,
138 "tier": 3,
139 "params": {
140 "minSize": 262144,
141 "avgSize": 1048576,
142 "maxSize": 4194304
143 }
144 }
145 ],
146 "hashChunk": [
147 {
148 "label": "empty",
149 "hex": "",
150 "hash": "5187b7a8021bf4f2c004ea3a54cfece1754f11c7624d2363c7f4cf4fddd1441e"
151 },
152 {
153 "label": "single zero byte",
154 "hex": "00",
155 "hash": "db10035e7a49836258fdb2613413948afa33c8f7db510f0860d7eb493d89da99"
156 },
157 {
158 "label": "abc",
159 "hex": "616263",
160 "hash": "cf7315a0fc0e5d6ef3292bb080af8126e66fc8e73e2d2158204c8b158eb24759"
161 },
162 {
163 "label": "32 zero bytes",
164 "hex": "0000000000000000000000000000000000000000000000000000000000000000",
165 "hash": "6ad4193c45e8d6ef1ac3b945417fd9c5ce12df85c92aea496621bb98099bfe77"
166 },
167 {
168 "label": "255 bytes 0x00..0xfe",
169 "hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfe",
170 "hash": "04cc042a877def2bbce35a68188b69bd443c77ff8a36d293add1481e1dc35718"
171 }
172 ],
173 "chunkCases": [
174 {
175 "label": "empty",
176 "note": "returns NO chunks, not one empty chunk",
177 "input": {
178 "kind": "hex",
179 "hex": ""
180 },
181 "tier": null,
182 "autoTier": 0,
183 "chunkCount": 0,
184 "chunks": []
185 },
186 {
187 "label": "single byte",
188 "note": null,
189 "input": {
190 "kind": "hex",
191 "hex": "00"
192 },
193 "tier": null,
194 "autoTier": 0,
195 "chunkCount": 1,
196 "chunks": [
197 {
198 "offset": 0,
199 "length": 1,
200 "hash": "db10035e7a49836258fdb2613413948afa33c8f7db510f0860d7eb493d89da99"
201 }
202 ]
203 },
204 {
205 "label": "minSize-1",
206 "note": null,
207 "input": {
208 "kind": "lcg32",
209 "seed": 1,
210 "length": 4095
211 },
212 "tier": null,
213 "autoTier": 0,
214 "chunkCount": 1,
215 "chunks": [
216 {
217 "offset": 0,
218 "length": 4095,
219 "hash": "4afc6ac85f9c914720b4be315447ee4239f8378720bbdca7083ebaa95e5a6dea"
220 }
221 ]
222 },
223 {
224 "label": "exactly minSize",
225 "note": "final-chunk test is `remaining <= minSize`, so this is ONE chunk with no search",
226 "input": {
227 "kind": "lcg32",
228 "seed": 2,
229 "length": 4096
230 },
231 "tier": null,
232 "autoTier": 0,
233 "chunkCount": 1,
234 "chunks": [
235 {
236 "offset": 0,
237 "length": 4096,
238 "hash": "511963a22481c56d2c16fcf1561bdac964af547fb19feff9726ca48927a8f473"
239 }
240 ]
241 },
242 {
243 "label": "minSize+1",
244 "note": "first size that enters the cut-point search",
245 "input": {
246 "kind": "lcg32",
247 "seed": 3,
248 "length": 4097
249 },
250 "tier": null,
251 "autoTier": 0,
252 "chunkCount": 1,
253 "chunks": [
254 {
255 "offset": 0,
256 "length": 4097,
257 "hash": "76cbb41dececa7f39067ab24ebe70645cd1598cc1d04a82826a4b3e4cc594d33"
258 }
259 ]
260 },
261 {
262 "label": "avgSize",
263 "note": null,
264 "input": {
265 "kind": "lcg32",
266 "seed": 4,
267 "length": 16384
268 },
269 "tier": null,
270 "autoTier": 0,
271 "chunkCount": 2,
272 "chunks": [
273 {
274 "offset": 0,
275 "length": 15862,
276 "hash": "d2fd846187d556a83093bb5822cc65097724663a8423be81455aa1099ba40823"
277 },
278 {
279 "offset": 15862,
280 "length": 522,
281 "hash": "df20a1837741a9ad6904440ce26572f62246713420b20fb4a13ab80fd5896776"
282 }
283 ]
284 },
285 {
286 "label": "between avg and max",
287 "note": null,
288 "input": {
289 "kind": "lcg32",
290 "seed": 5,
291 "length": 40960
292 },
293 "tier": null,
294 "autoTier": 0,
295 "chunkCount": 4,
296 "chunks": [
297 {
298 "offset": 0,
299 "length": 13980,
300 "hash": "699e2cda5f0184ecba24f28b7b2ed15f059423f26bd7652899af9edbbe7c8301"
301 },
302 {
303 "offset": 13980,
304 "length": 14116,
305 "hash": "d0f67fac680fc20def0cd6348c56227080c26da040afae60328497451f7d9e3d"
306 },
307 {
308 "offset": 28096,
309 "length": 11864,
310 "hash": "ff1eaedaba1c3302698d695dd09add0fc8db9a19cf980f4433de296413e198eb"
311 },
312 {
313 "offset": 39960,
314 "length": 1000,
315 "hash": "5b6bb68e04da9f9a9e124e065dd218507701cbcecb3609892e0258c533d2dd81"
316 }
317 ]
318 },
319 {
320 "label": "exactly maxSize",
321 "note": null,
322 "input": {
323 "kind": "lcg32",
324 "seed": 6,
325 "length": 65536
326 },
327 "tier": null,
328 "autoTier": 0,
329 "chunkCount": 3,
330 "chunks": [
331 {
332 "offset": 0,
333 "length": 15305,
334 "hash": "9af77bf4332cc48f2278db9590b4934da53946696ea585f874ef0dd7b27883b7"
335 },
336 {
337 "offset": 15305,
338 "length": 48382,
339 "hash": "44ea2a52c74e3b129d2b65d90ff018b954d798723686a89d7fe5128a3ccb172c"
340 },
341 {
342 "offset": 63687,
343 "length": 1849,
344 "hash": "657a27ff2e563df1067414be98732ed1dbbf1d1c67de8b3cbf4fc3968144425f"
345 }
346 ]
347 },
348 {
349 "label": "several chunks",
350 "note": null,
351 "input": {
352 "kind": "lcg32",
353 "seed": 7,
354 "length": 263378
355 },
356 "tier": null,
357 "autoTier": 0,
358 "chunkCount": 15,
359 "chunks": [
360 {
361 "offset": 0,
362 "length": 9045,
363 "hash": "778171efc5b101504f85c3f6a8c074070b418a36392695688c7fce9806976901"
364 },
365 {
366 "offset": 9045,
367 "length": 33713,
368 "hash": "56a81dd833fcbc197fd7d4b0efafba6ab2b17a84254cc8dbbceaa4ee63c0486f"
369 },
370 {
371 "offset": 42758,
372 "length": 26147,
373 "hash": "e77cc7e187b00f34e6cdc5620b68b43dce5d75922b66a1990fa214f594ab1e84"
374 },
375 {
376 "offset": 68905,
377 "length": 8029,
378 "hash": "28f58c92138fc4959f494255cdf0aeb9da3c9d8e089c5dbc659643517af6e34a"
379 },
380 {
381 "offset": 76934,
382 "length": 4474,
383 "hash": "a1bd1ab6eb7bdadb413d3f434f512301a32c85c512de578a4923963b20f102fe"
384 },
385 {
386 "offset": 81408,
387 "length": 17939,
388 "hash": "35d70452aad0900b7f96eb80c2395ea6fa86418e9e69f1aede14d9730391d7c0"
389 },
390 {
391 "offset": 99347,
392 "length": 8404,
393 "hash": "83988c77fe048119cb6f7c68b157f495027f58c4b56513bcc733bbd2e6e2ead9"
394 },
395 {
396 "offset": 107751,
397 "length": 23001,
398 "hash": "1456fb11244fe41fe4437c53b0c4ed439f8a6da36d4e631dc828e45696b44397"
399 },
400 {
401 "offset": 130752,
402 "length": 12490,
403 "hash": "03de9d66082a1d05792a06b7b98956379f71195b4e6472574cfbae462653fe9b"
404 },
405 {
406 "offset": 143242,
407 "length": 54324,
408 "hash": "44f10b42e64a14be32d5fafccbc06225a209d7b03a079aba05f1a5684ea27ec5"
409 },
410 {
411 "offset": 197566,
412 "length": 25222,
413 "hash": "8965d89e8c2ebf2767c39dc028c6ac2116d50ecb674c15ec5ce925564038bfb2"
414 },
415 {
416 "offset": 222788,
417 "length": 11124,
418 "hash": "b9444ae9a8ee6595ce35941f4cb483d5d5c5fa054a37f27e058e0e06fc47ab11"
419 },
420 {
421 "offset": 233912,
422 "length": 14403,
423 "hash": "d229d009c2a21866c9f5b99d66e11b3cce74cda1b9133ba1f87fab0893a1a1b2"
424 },
425 {
426 "offset": 248315,
427 "length": 11054,
428 "hash": "0f47104a324cdad9a57ef644268e37f5fa4257790121989c8a5e8ac438b2bfff"
429 },
430 {
431 "offset": 259369,
432 "length": 4009,
433 "hash": "eb9881861ccde16e52a6c45bff67c97dd6f40d5101a1680d22df06664f44c766"
434 }
435 ]
436 },
437 {
438 "label": "all zero bytes",
439 "note": "constant input drives the gear hash deterministically; exercises the maxSize cutoff",
440 "input": {
441 "kind": "const",
442 "byte": 0,
443 "length": 131079
444 },
445 "tier": null,
446 "autoTier": 0,
447 "chunkCount": 3,
448 "chunks": [
449 {
450 "offset": 0,
451 "length": 65536,
452 "hash": "2b5ed8227e13b5d8d5398336f0ef980add6bac7092d436c612a98aefc0a1caa9"
453 },
454 {
455 "offset": 65536,
456 "length": 65536,
457 "hash": "2b5ed8227e13b5d8d5398336f0ef980add6bac7092d436c612a98aefc0a1caa9"
458 },
459 {
460 "offset": 131072,
461 "length": 7,
462 "hash": "8cff55fa2f70f3a551d2537200e02721b604d7d8b368f0a56d800b84a144f914"
463 }
464 ]
465 },
466 {
467 "label": "all 0xff bytes",
468 "note": null,
469 "input": {
470 "kind": "const",
471 "byte": 255,
472 "length": 69632
473 },
474 "tier": null,
475 "autoTier": 0,
476 "chunkCount": 2,
477 "chunks": [
478 {
479 "offset": 0,
480 "length": 65536,
481 "hash": "4c5967b81482236ecb70d6590a0aab89bdf390bf1f5ddb69ff3584d37457d4ad"
482 },
483 {
484 "offset": 65536,
485 "length": 4096,
486 "hash": "9501c2198579f3c074c9df7fc55e149b90086bf5680053cb567cbe555e48430c"
487 }
488 ]
489 },
490 {
491 "label": "forced tier 0",
492 "note": null,
493 "input": {
494 "kind": "lcg32",
495 "seed": 99,
496 "length": 1048576
497 },
498 "tier": 0,
499 "autoTier": null,
500 "chunkCount": 59,
501 "chunks": [
502 {
503 "offset": 0,
504 "length": 39526,
505 "hash": "ef561491c8e9b6a14cd375e05724bcbcf4cda5ba3472ea97da2cb254f2ad48bd"
506 },
507 {
508 "offset": 39526,
509 "length": 7126,
510 "hash": "3271311fc64c296703291d7707f6734e16e835c58d1f67a81daa94a820e557ba"
511 },
512 {
513 "offset": 46652,
514 "length": 14559,
515 "hash": "fcd14d005f61c05f4f453b9586115ca5b45614fcc35b58c326b5cff8e5fb7241"
516 },
517 {
518 "offset": 61211,
519 "length": 8451,
520 "hash": "850a5876aab9877b0cb786819b5a6dcb86eb7f6ab57f41681bc381a7f965f829"
521 },
522 {
523 "offset": 69662,
524 "length": 11716,
525 "hash": "67608a811d38514d58bebad3f8a5bbbc093efdba1a1eac6535b548013ae25b8a"
526 },
527 {
528 "offset": 81378,
529 "length": 18785,
530 "hash": "bef4f2cddf820ebc9abba1aaa11a6f50c72f0bed5006bc1e95e3f4ee4331dd70"
531 },
532 {
533 "offset": 100163,
534 "length": 12875,
535 "hash": "6f3077ebc9b5518670a9d8a81418ddabce07abf63a4665495c45588298f23f24"
536 },
537 {
538 "offset": 113038,
539 "length": 4433,
540 "hash": "cbdcf10429c878e4c1dfdb901c802f757edf349f84f342f44a2aab2db9729401"
541 },
542 {
543 "offset": 117471,
544 "length": 8602,
545 "hash": "1130c0fd1e273d6487ff8a975dd03b9cc838b82c43c3ce871437eb62dcc52522"
546 },
547 {
548 "offset": 126073,
549 "length": 13002,
550 "hash": "56aa0d9028312967341b7d5e1e56096a53e1e4b23ceb03c8795250615870fc15"
551 },
552 {
553 "offset": 139075,
554 "length": 65536,
555 "hash": "538d5c02a5f29d115b49207319351af73fbcb8ba5dff28983016022ae56f2a80"
556 },
557 {
558 "offset": 204611,
559 "length": 21007,
560 "hash": "ceac34581465653061ddfcbfbbf63f07fd054ccc41063c605a996adbb027a33b"
561 },
562 {
563 "offset": 225618,
564 "length": 6498,
565 "hash": "2059563b18298f668f27fe8eb3f1cb53ee5bd15901decc5eb51c0be805c9a9f0"
566 },
567 {
568 "offset": 232116,
569 "length": 17715,
570 "hash": "f4eb5e6307f04a66d0ae3e3b09955003055f2b9c6b6ab11c68a16ddddbfbf594"
571 },
572 {
573 "offset": 249831,
574 "length": 6281,
575 "hash": "7260edbc78417466cbb6bd806f8266c7602a9fbd45dacc28daa775be1da969be"
576 },
577 {
578 "offset": 256112,
579 "length": 7895,
580 "hash": "406e97cbe60534187ca318885aa30fede583d1d6da3c40d577a4bb5978cd8d95"
581 },
582 {
583 "offset": 264007,
584 "length": 7634,
585 "hash": "31307e9fd1075f88e3b51a7d827ebd21a14c3044c6175658eab3c0769eb5aa6d"
586 },
587 {
588 "offset": 271641,
589 "length": 10244,
590 "hash": "c7294c9c2cea81a0271e48dfac4cb2a173e83f23264a4b515837c315fb5ffbfc"
591 },
592 {
593 "offset": 281885,
594 "length": 4736,
595 "hash": "87f4ac71de61fb4768d5c9437875e5c9b5eb3afcd559a6baab1b06def21e13c4"
596 },
597 {
598 "offset": 286621,
599 "length": 13117,
600 "hash": "ba42834e47b72c9e36d404fef98e24f6d66586fb05fced0e094d4faeade04746"
601 },
602 {
603 "offset": 299738,
604 "length": 17020,
605 "hash": "14a69a995b96d99abe565e8bb349968418a0d808872ef9ba0ce39ef88f8c5eb5"
606 },
607 {
608 "offset": 316758,
609 "length": 16375,
610 "hash": "0b0b4e62e6027b4645ed6cf0450e8ffe7870571ae076e309e5a1579f3b8e24ab"
611 },
612 {
613 "offset": 333133,
614 "length": 4931,
615 "hash": "2a4a815ac7cc2dc5c75033f77f0784ec023b91ed31b975457b6bb8a2117e4e36"
616 },
617 {
618 "offset": 338064,
619 "length": 34767,
620 "hash": "9ca3e925e6d4dc9fc3f3348fedb3a62ad5e7d3ba10cf43695fefd4922e78e196"
621 },
622 {
623 "offset": 372831,
624 "length": 20721,
625 "hash": "72112426d2d650488baab4b7687b22adca99fc92a544f4e2982e842522dd5db2"
626 },
627 {
628 "offset": 393552,
629 "length": 53466,
630 "hash": "0269b05d89ad3fcfcbbe02a782a06ef46518e0c781a672bbd2dda6947b0dd25b"
631 },
632 {
633 "offset": 447018,
634 "length": 18588,
635 "hash": "0405e37dbcbcaa76a644315016b6631f20b59a43b4a3ed92b5a0d9bdca268fe8"
636 },
637 {
638 "offset": 465606,
639 "length": 24355,
640 "hash": "f7ea646c92c48271cbf5e647fcbff13f6a26e6253010e312366fc74b9c6e0cca"
641 },
642 {
643 "offset": 489961,
644 "length": 39193,
645 "hash": "e112d666f3fe317936c73b74e79a6439ff5e385fd925f40cb097fc1729674cb1"
646 },
647 {
648 "offset": 529154,
649 "length": 13972,
650 "hash": "d6e07bcf7eeba36e60d3cafdaa02d919f9b28742a873aaf83a027718b9fc24d8"
651 },
652 {
653 "offset": 543126,
654 "length": 22650,
655 "hash": "78fd26726c3416ea5c091097160c9d45a7976d3052f0ab0c93a455acd6d14e7c"
656 },
657 {
658 "offset": 565776,
659 "length": 4243,
660 "hash": "0e17a48a017ef5afbb2add1a776e7393cf92385191f4d3c9e7350f09fced60ba"
661 },
662 {
663 "offset": 570019,
664 "length": 4114,
665 "hash": "f405a6549f04521fcc85691c903df5a8a328cbe5bd3b02306046478bd91e0f4e"
666 },
667 {
668 "offset": 574133,
669 "length": 31306,
670 "hash": "559db6483e331cd7a0c981e856026ab0d8e827631340d3f04eb52d804a1cdeab"
671 },
672 {
673 "offset": 605439,
674 "length": 11917,
675 "hash": "a156c72c47a73243dc7d8d3a72ac926ded0f2687557e22525191ec340172ce87"
676 },
677 {
678 "offset": 617356,
679 "length": 31811,
680 "hash": "998eec3955a4b22fe5f4d141f5c33c036c787f336e826458f092c6b4982fecd9"
681 },
682 {
683 "offset": 649167,
684 "length": 12678,
685 "hash": "637550bd9a67ad0764d118635ea15e7f1f24cc77aafc3d6bba9062c155f61e72"
686 },
687 {
688 "offset": 661845,
689 "length": 31062,
690 "hash": "ad9a7570304ba8470092c21a12e7edab706a3091b9d131b21846cf525a4c32e3"
691 },
692 {
693 "offset": 692907,
694 "length": 40288,
695 "hash": "5cf4019329bb740a0998ffa0c9e01246d268e8fe7a94e9d7df0b3ad6aab8ee04"
696 },
697 {
698 "offset": 733195,
699 "length": 9181,
700 "hash": "76360c4aad038f6bdf504978ff853ee2a39e7979b65cb1f37b3ac73599268cb7"
701 },
702 {
703 "offset": 742376,
704 "length": 42173,
705 "hash": "aa0c9d49ec491e759f8c6c3fc0e3d8e8c75e965a2ae6990b20d80c48360077b6"
706 },
707 {
708 "offset": 784549,
709 "length": 6205,
710 "hash": "67628eccb64c2291356a2683d90ed675b4499c8e4c5b3d9287127c0f7346d947"
711 },
712 {
713 "offset": 790754,
714 "length": 5421,
715 "hash": "12ef9c2b4e5a46eb712ffd932f7b3b7402f07e3534cbe7a677fa1a883fb5a9e7"
716 },
717 {
718 "offset": 796175,
719 "length": 6674,
720 "hash": "7f4982045e5a6f156bdb6cb3be9c1008eb0ddf8411bd7616f7d2611e12c697e0"
721 },
722 {
723 "offset": 802849,
724 "length": 5054,
725 "hash": "b48ee8fdff22f0afc26fa087fbe45f3e26ceee64c6ab31d16886813d131cdd32"
726 },
727 {
728 "offset": 807903,
729 "length": 42095,
730 "hash": "07a68b0db9dcd2b4d42b6620a07eac8bf114d0d2eb2c2eee19a78009fa68661a"
731 },
732 {
733 "offset": 849998,
734 "length": 9846,
735 "hash": "393fb651b14ab4b1d8492aa597eb10e922bc418ae49833aa8054c86d5442fe4c"
736 },
737 {
738 "offset": 859844,
739 "length": 8302,
740 "hash": "ec9852082564bb49a3d675c8a141b28f12f20f589b288b10f65f9e45fc263cef"
741 },
742 {
743 "offset": 868146,
744 "length": 4421,
745 "hash": "72bb8020b80cea198915cc1ca3e8b6ccc166fd64c31baa5d1d76ee14f7ab516c"
746 },
747 {
748 "offset": 872567,
749 "length": 5400,
750 "hash": "78700538f91a5a95ad9ebe178f4d193314f16983f4e0f6d9723b6648a7b834ac"
751 },
752 {
753 "offset": 877967,
754 "length": 4397,
755 "hash": "67207a393beae4239ad5d565170bfc8ec09626cf5540610b40b60814ddc27a15"
756 },
757 {
758 "offset": 882364,
759 "length": 14318,
760 "hash": "9026ad7a9b6086630a3961c97bc0ad66f27d80e14881d94345662028e68e254b"
761 },
762 {
763 "offset": 896682,
764 "length": 23632,
765 "hash": "ed8b7e3145ffdb41ce5786c3bc690032c63c1dfb876a4bbc9843a05c6ad0e238"
766 },
767 {
768 "offset": 920314,
769 "length": 10784,
770 "hash": "0e9ec2dfde77d4d5028cc30fe9bc8c5a0131e0360553df74ebc02c854f90952a"
771 },
772 {
773 "offset": 931098,
774 "length": 8612,
775 "hash": "b3795c13c77767087f178c696c787d943cd133cef5661d09e44c0f85ba4625e4"
776 },
777 {
778 "offset": 939710,
779 "length": 50157,
780 "hash": "c7879d4e74ee0c76517a48f450aab6ecedf8d8dcfc0b01293b41bb4bd51d6019"
781 },
782 {
783 "offset": 989867,
784 "length": 27156,
785 "hash": "6e1451acc87e28e0e35e5f658ac4c667c3302d5b1780c873819a0b00340a8cd1"
786 },
787 {
788 "offset": 1017023,
789 "length": 22108,
790 "hash": "7d7902a44072cef5ccdfe68fb48d3f60632992c01a3bcb0974f3c8e8dd99c913"
791 },
792 {
793 "offset": 1039131,
794 "length": 9445,
795 "hash": "d46a8e5fac9c85841826bee6b5baa1c972f1016732a0930a260b6724eb8b8572"
796 }
797 ]
798 },
799 {
800 "label": "forced tier 1",
801 "note": null,
802 "input": {
803 "kind": "lcg32",
804 "seed": 99,
805 "length": 1048576
806 },
807 "tier": 1,
808 "autoTier": null,
809 "chunkCount": 13,
810 "chunks": [
811 {
812 "offset": 0,
813 "length": 139075,
814 "hash": "5b01cf7d92eb41e19139c5facdd6428379872c406e8b1e1b8771ba386e7cb600"
815 },
816 {
817 "offset": 139075,
818 "length": 117037,
819 "hash": "29c06c724e948bc88e6d0014283c6bfcd2b1ce582894fd3c55e277a61cc61e6b"
820 },
821 {
822 "offset": 256112,
823 "length": 30509,
824 "hash": "c905fc9ff80745e24f5f8623c9240894e15089f28f50868a7de681e335ac07fb"
825 },
826 {
827 "offset": 286621,
828 "length": 32976,
829 "hash": "b5e1de522f1ff5b835f93c7639e4826eee93082f2b43e309fc742c921601dd9e"
830 },
831 {
832 "offset": 319597,
833 "length": 209557,
834 "hash": "e420e5835bba01a38e3a5a991f1d5cb1a1319fe56a4b8fa0e1fd0b978fc0585f"
835 },
836 {
837 "offset": 529154,
838 "length": 76285,
839 "hash": "e4c53a6a4553b5f6c20b8da526916843f03def1dcd5a308be4e243fc6ba0393a"
840 },
841 {
842 "offset": 605439,
843 "length": 56406,
844 "hash": "ec96ead3d0c26084359c9246751e1bec47ee65ea5f8979bac4bd1cac8cbb121b"
845 },
846 {
847 "offset": 661845,
848 "length": 122704,
849 "hash": "51ca5b73dc16501c200dc5f08c8ff43e74d1d7a23614ef54816eb445183d20b4"
850 },
851 {
852 "offset": 784549,
853 "length": 88018,
854 "hash": "7a69b1bc86d2f4d4a8c2e3cd27837d8bb5c11a8b2e7481ab10d1b5adec83e02b"
855 },
856 {
857 "offset": 872567,
858 "length": 24115,
859 "hash": "5159405308ba3bfc15892097db0896778202a4b995d72c9f9cc6f99ea6ba1555"
860 },
861 {
862 "offset": 896682,
863 "length": 43028,
864 "hash": "0271f1e888a7eb1910fd761adb699492209f6555f12e1beebb64303f618a1afe"
865 },
866 {
867 "offset": 939710,
868 "length": 77313,
869 "hash": "5af4a09d47e7d8268c94b9f1b86ece92761f560801882b062856da1739ef7197"
870 },
871 {
872 "offset": 1017023,
873 "length": 31553,
874 "hash": "6f4cdb40523a36cc72996b1a8935a566ab7db8a1c4814fce0f35f6770ba97579"
875 }
876 ]
877 },
878 {
879 "label": "forced tier 2",
880 "note": null,
881 "input": {
882 "kind": "lcg32",
883 "seed": 99,
884 "length": 1048576
885 },
886 "tier": 2,
887 "autoTier": null,
888 "chunkCount": 3,
889 "chunks": [
890 {
891 "offset": 0,
892 "length": 271641,
893 "hash": "ab293c640c9ca74efc66991d33cd23d0f65f0ed2419b4f615350d9120abb0460"
894 },
895 {
896 "offset": 271641,
897 "length": 606326,
898 "hash": "3b4bda3366031a130640282ef750a1394be503d9bc0fe67ec1ee0caf251dd5c2"
899 },
900 {
901 "offset": 877967,
902 "length": 170609,
903 "hash": "602ea8fea1556cedc5dfbc3cba85c13b956102b6f371f85d0416f14f365d80fc"
904 }
905 ]
906 },
907 {
908 "label": "forced tier 3",
909 "note": null,
910 "input": {
911 "kind": "lcg32",
912 "seed": 99,
913 "length": 1048576
914 },
915 "tier": 3,
916 "autoTier": null,
917 "chunkCount": 1,
918 "chunks": [
919 {
920 "offset": 0,
921 "length": 1048576,
922 "hash": "93e8c7f4950ca17893496c2810dd7e618e386a5f7b15a29c251097be7213f087"
923 }
924 ]
925 },
926 {
927 "label": "auto tier under 1MB",
928 "note": null,
929 "input": {
930 "kind": "lcg32",
931 "seed": 11,
932 "length": 1048575
933 },
934 "tier": null,
935 "autoTier": 0,
936 "chunkCount": 54,
937 "chunks": [
938 {
939 "offset": 0,
940 "length": 12474,
941 "hash": "ea65470611d3bb3668b2e47e78ef60ee731105d8dc008798730f2ef98e3f735a"
942 },
943 {
944 "offset": 12474,
945 "length": 6091,
946 "hash": "55ee82c5014901b4fcf1ac6754513cee7d48bdb1f10565d5bb1bc756377791ae"
947 },
948 {
949 "offset": 18565,
950 "length": 4776,
951 "hash": "c3846c51a9f54472d4e9cf32757c5c075b11b6f90b68aa3cff75f9a2003fa665"
952 },
953 {
954 "offset": 23341,
955 "length": 13272,
956 "hash": "f882acdac339e2f4950532272172d6f2e962d7bcf931547431807ffbea9cb001"
957 },
958 {
959 "offset": 36613,
960 "length": 8608,
961 "hash": "03efbea973ac5522d511b003db3c93f75ebba79844b5fb396f53d014b2e92296"
962 },
963 {
964 "offset": 45221,
965 "length": 30084,
966 "hash": "e32ae3c04f7086a3c89867cb9766b1825a1700a09205e4c9fcd69cd7be6ca966"
967 },
968 {
969 "offset": 75305,
970 "length": 27253,
971 "hash": "6606e4414ff3944f4406dd5b8a0e901b92c0bb764b281c97cbcd66ff570faf5b"
972 },
973 {
974 "offset": 102558,
975 "length": 20418,
976 "hash": "c08b7db859f228895268087d498b9d9b180bb032a51cd5978da3046e7b5d389e"
977 },
978 {
979 "offset": 122976,
980 "length": 65536,
981 "hash": "0bc2726ec593304fc27b7c6ab68d50dfe21472870210d9fb163da41bebb044f4"
982 },
983 {
984 "offset": 188512,
985 "length": 47503,
986 "hash": "186d41cef4c0f6fbf4ffa5bb3d984adb129f8f630dcaa543969e217d71d44cd8"
987 },
988 {
989 "offset": 236015,
990 "length": 12057,
991 "hash": "82981293b5af1b603970677da3221fafc1bd7ee1c11ea3be3b25005af5372971"
992 },
993 {
994 "offset": 248072,
995 "length": 5288,
996 "hash": "58efd3abd0efe922e009e324d13103409a994bfd7accbafa0a7d5c1d06e16311"
997 },
998 {
999 "offset": 253360,
1000 "length": 17170,
1001 "hash": "6305b82c90e5605aabfd57bb9619b256295cfdf97da939611b3f0b42a515fd15"
1002 },
1003 {
1004 "offset": 270530,
1005 "length": 11572,
1006 "hash": "b9fdac474632f7fe2d0e6846ad1653c8d0351cc3990e7d31782d334c8f57ae80"
1007 },
1008 {
1009 "offset": 282102,
1010 "length": 24367,
1011 "hash": "338010bfcbbcaee46f4eec8a1a77347ec539835366a984c80bed0d53f0f9da42"
1012 },
1013 {
1014 "offset": 306469,
1015 "length": 18029,
1016 "hash": "6b8664f4991d1b0dc899a6c758a5a3612ff1fadc370a76211ee051717644a400"
1017 },
1018 {
1019 "offset": 324498,
1020 "length": 7396,
1021 "hash": "f9cb6ff58ad1876e5e7414185ee6dc75019b4230db2ca2cb911aa0acac02b7c5"
1022 },
1023 {
1024 "offset": 331894,
1025 "length": 42802,
1026 "hash": "ce65e282a7cf21a1deeb2c2d82375f6dad14b080dae3c83b319ec69a8009acde"
1027 },
1028 {
1029 "offset": 374696,
1030 "length": 12591,
1031 "hash": "ab5d358ede3a465e0579fc3bcd1c2b42efeb228d5dfce13c6f76c83b88a9dfa7"
1032 },
1033 {
1034 "offset": 387287,
1035 "length": 43772,
1036 "hash": "b0e6f2575f0c7996278712dbdf7d100f7eb0980dd433869ece81ac519f18974b"
1037 },
1038 {
1039 "offset": 431059,
1040 "length": 35690,
1041 "hash": "cb475e0e7d8df261ee434d562844f95e32104902587fd4958c5623500113f0b6"
1042 },
1043 {
1044 "offset": 466749,
1045 "length": 40878,
1046 "hash": "48674502addabe0ac2d0ca24732043f1c13d5f74b54cb0d1c66cb6b9d87e5d51"
1047 },
1048 {
1049 "offset": 507627,
1050 "length": 14002,
1051 "hash": "9dc8583c1bdcda6ac8259f3dc611c1757c91a33a017bff39b88c51bb35f622bb"
1052 },
1053 {
1054 "offset": 521629,
1055 "length": 8720,
1056 "hash": "b96a1206d9caf35ff81224dcd21e83c9667210850e74eb53091dd69d1f9672bb"
1057 },
1058 {
1059 "offset": 530349,
1060 "length": 6123,
1061 "hash": "81d7d1f82ca10ac719d6e57b8142020d5cfbe3fa303bc10d76823bddf06caa77"
1062 },
1063 {
1064 "offset": 536472,
1065 "length": 12079,
1066 "hash": "85c2571a0acabdaac64fda6c4d65698d7a2484bcceaa372c40802eb11cd83135"
1067 },
1068 {
1069 "offset": 548551,
1070 "length": 5687,
1071 "hash": "1d9b1578283165274da26aeca7691cbe6af40eb3882cc3f4731cd6de91405fa4"
1072 },
1073 {
1074 "offset": 554238,
1075 "length": 9650,
1076 "hash": "d073843fbc04fe154c1802027a0c5bd171a5bf3c984e388d504b2163bec1be2a"
1077 },
1078 {
1079 "offset": 563888,
1080 "length": 65536,
1081 "hash": "52084179c8bb35aa3997cadb369a062bc4717982682a4ba99b3d357f99ee7081"
1082 },
1083 {
1084 "offset": 629424,
1085 "length": 9106,
1086 "hash": "3b46a8ee8d9caea69b73a986de4f79081db8fb8b3e1adfd228a645ec0979374a"
1087 },
1088 {
1089 "offset": 638530,
1090 "length": 8888,
1091 "hash": "645725e5be88936e7cc714483aff948bda74a80688271dcdc6e430e34d9c7ae4"
1092 },
1093 {
1094 "offset": 647418,
1095 "length": 9437,
1096 "hash": "798af4f12aee2b551b1c9ad4dc28842c98ca9d5eb4dc31052ecf203b7814896e"
1097 },
1098 {
1099 "offset": 656855,
1100 "length": 17092,
1101 "hash": "e48e4d3df098ebab23aba03b0d9398d2266c9f5620cc9d6dda46abe043a5bf93"
1102 },
1103 {
1104 "offset": 673947,
1105 "length": 8019,
1106 "hash": "18de0e57e1ec2e9c426648018b337a642808a8ad51fd7da443e11b92abed6342"
1107 },
1108 {
1109 "offset": 681966,
1110 "length": 25281,
1111 "hash": "417fc09a2485ac5cb88c9a88a7b78449678e384fab406d6d1380a78dec23c162"
1112 },
1113 {
1114 "offset": 707247,
1115 "length": 5844,
1116 "hash": "47e294300eba49b9ef1d48331bf9f72d1d334bc10262a0518f1a4ae2bc3860e5"
1117 },
1118 {
1119 "offset": 713091,
1120 "length": 12211,
1121 "hash": "06b1a634bdedfd16e22239cb5ea0974de5b8b293feedf42a246995ae8c3302de"
1122 },
1123 {
1124 "offset": 725302,
1125 "length": 13447,
1126 "hash": "fa7a12a267bf6cfe02408069aeb58e2c284abf6feac3c537617eda3943d28ab2"
1127 },
1128 {
1129 "offset": 738749,
1130 "length": 23385,
1131 "hash": "afc553cc66af4af92a89e42f52296329b8c5d3a37bea894e100e9bef2fc26e3d"
1132 },
1133 {
1134 "offset": 762134,
1135 "length": 24901,
1136 "hash": "95f197b9b8522414600f655501920fcf549ce38228deb34076584b5163e53237"
1137 },
1138 {
1139 "offset": 787035,
1140 "length": 34115,
1141 "hash": "4f0dbb76b71311c748843fb5a45ab4b262b234099576feb74d6cb0413d49ab39"
1142 },
1143 {
1144 "offset": 821150,
1145 "length": 14855,
1146 "hash": "17412af0f205177d6599022150b3ba946bfee49fdb526c8ec50f7b405f365f70"
1147 },
1148 {
1149 "offset": 836005,
1150 "length": 10213,
1151 "hash": "6f80c0cbffd5d9df281143be656b5da59cbf31c95b15f8e8596470d628dbbcaf"
1152 },
1153 {
1154 "offset": 846218,
1155 "length": 19616,
1156 "hash": "928fab62d41d262ba4217914174e3452eba0fab9b0367e566fb4f7925783c23d"
1157 },
1158 {
1159 "offset": 865834,
1160 "length": 8045,
1161 "hash": "71d12b421af44263760a0aca1a2de871f1a1b3e7598deb3d301abc3fc0bfc8fb"
1162 },
1163 {
1164 "offset": 873879,
1165 "length": 13180,
1166 "hash": "90babee84aedbdd491a5bb4cdc6a15b14a6c7d84c6d366ebef57c5b67cc7d743"
1167 },
1168 {
1169 "offset": 887059,
1170 "length": 16431,
1171 "hash": "44188c2ff329230a20f573a5ffeb38363d464bf9536cf80446b420ff3a8cd0bf"
1172 },
1173 {
1174 "offset": 903490,
1175 "length": 19531,
1176 "hash": "11c4d76a2c8c55d02ed5115fe780dcc28a48d4199c3fbb741125f00a276c6770"
1177 },
1178 {
1179 "offset": 923021,
1180 "length": 22212,
1181 "hash": "6371b1ac824bfe591502084bac4708c259383e0d9196ddd95e108c49753e305b"
1182 },
1183 {
1184 "offset": 945233,
1185 "length": 20647,
1186 "hash": "be12936c5c2d59f244dea3b1e1a18f0ac344a99dd0f59306acf3f13c84b0a63d"
1187 },
1188 {
1189 "offset": 965880,
1190 "length": 23943,
1191 "hash": "17612d0f3844b4a2e8792c4af2d36b110cccf353912183ad42021aa909223f15"
1192 },
1193 {
1194 "offset": 989823,
1195 "length": 26068,
1196 "hash": "cd4e6bf246730f86129cf40e4d9e4cf806c889524685808b390ac6031e50a4a4"
1197 },
1198 {
1199 "offset": 1015891,
1200 "length": 5091,
1201 "hash": "93a0e7b74a6834ed431fd39b0aef228a066e524273681347ebc80ec9c3ae2233"
1202 },
1203 {
1204 "offset": 1020982,
1205 "length": 27593,
1206 "hash": "b4690ae1ffa5dde3b48c1a9dfcdbe9d97b3266b15d209b6defd7545f677f52fe"
1207 }
1208 ]
1209 }
1210 ]
1211}
test/conformance/generate-chunker.mjs +160 -0
……@@ -0,0 +1,160 @@
1/**
2 * Conformance vectors for the FastCDC chunker.
3 *
4 * WHY THIS MATTERS MORE THAN THE MESSAGE VECTORS. A message divergence fails
5 * loudly — the frame does not decode. A CHUNKING divergence fails SILENTLY: both
6 * implementations work, produce valid chunks, verify their own hashes, and simply
7 * never share a single chunk with each other. Deduplication quietly drops to
8 * zero and nothing errors.
9 *
10 * Run: node test/conformance/generate-chunker.mjs > test/conformance/chunker-vectors.json
11 * Check: node test/conformance/verify-chunker.mjs
12 *
13 * THE FIVE THINGS A PORT MUST MATCH EXACTLY
14 *
15 * 1. THE CHUNK HASH IS NOT PLAIN BLAKE2B. hashChunk is hypercore-crypto's
16 * `data()`: blake2b-256(0x00 || uint64_LE(byteLength) || data). A port that
17 * hashes the bytes alone gets different content hashes for identical bytes and
18 * shares nothing. Pinned by the hashChunk cases below.
19 *
20 * 2. THE ROLLING HASH IS A VARIANT. `hash = ((hash << 1) + GEAR[byte]) >>> 0` —
21 * shift LEFT, 32-bit unsigned. Canonical FastCDC shifts right. The hash is NOT
22 * reset between the two mask phases, and the first minSize bytes are never
23 * hashed at all (i starts at minSize).
24 *
25 * 3. THE CUT IS EXCLUSIVE. On a mask match the loop breaks WITHOUT incrementing,
26 * so the chunk is data[offset, offset+i) and the byte that triggered the cut
27 * belongs to the NEXT chunk. A one-off here still produces plausible chunks.
28 *
29 * 4. THE FINAL-CHUNK TEST IS `remaining <= minSize`, not `<`. Exactly-minSize
30 * remaining is a final chunk, not a search.
31 *
32 * 5. TIER MASKS DEGENERATE. large = (1<<ceil(log2(avg)))-1, small =
33 * (1<<floor(log2(avg)))-1 — and every tier's avgSize is a power of two, so
34 * large === small for all four tiers and the two-phase normalisation collapses
35 * to one mask. That is asserted below so a port neither "fixes" it nor relies
36 * on it accidentally.
37 *
38 * INPUTS are given either as literal hex (small cases) or as a deterministic PRNG
39 * spec, so a port reproduces multi-MB inputs without this file carrying them:
40 *
41 * lcg32: s = (s * 1664525 + 1013904223) mod 2^32 ; byte = (s >>> 24) & 0xff
42 *
43 * with `s` starting at `seed`, one byte per iteration.
44 */
45
46import { readFileSync } from 'fs'
47import { fileURLToPath } from 'url'
48import { dirname, join } from 'path'
49import {
50 chunk, hashChunk, selectTier, getTierParams, TIERS, MIN_SIZE, AVG_SIZE, MAX_SIZE
51} from '../../lib/chunker.js'
52
53const __dir = dirname(fileURLToPath(import.meta.url))
54const pkg = JSON.parse(readFileSync(join(__dir, '../../package.json'), 'utf8'))
55
56function lcg32 (seed, length) {
57 const out = Buffer.alloc(length)
58 let s = seed >>> 0
59 for (let i = 0; i < length; i++) {
60 s = (Math.imul(s, 1664525) + 1013904223) >>> 0
61 out[i] = (s >>> 24) & 0xff
62 }
63 return out
64}
65
66const cases = []
67
68function addChunkCase (label, input, opts = {}) {
69 const data = input.buf
70 const res = chunk(data, opts.tier !== undefined ? { tier: opts.tier } : undefined)
71 // Chunks must tile the input exactly — a generator that emits a gap or an
72 // overlap would bake a broken expectation into every port that follows it.
73 let acc = 0
74 for (const ch of res) {
75 if (ch.offset !== acc) throw new Error(`${label}: chunk offset gap at ${ch.offset}, expected ${acc}`)
76 acc += ch.length
77 }
78 if (acc !== data.length) throw new Error(`${label}: chunks cover ${acc} of ${data.length} bytes`)
79
80 cases.push({
81 label,
82 note: opts.note || null,
83 input: input.spec,
84 tier: opts.tier !== undefined ? opts.tier : null,
85 autoTier: opts.tier === undefined ? selectTier(data.length) : null,
86 chunkCount: res.length,
87 chunks: res.map(ch => ({ offset: ch.offset, length: ch.length, hash: ch.hash }))
88 })
89}
90
91const hexInput = (hex) => ({ buf: Buffer.from(hex, 'hex'), spec: { kind: 'hex', hex } })
92const prngInput = (seed, length) => ({
93 buf: lcg32(seed, length),
94 spec: { kind: 'lcg32', seed, length }
95})
96
97// ── degenerate + boundary sizes ──────────────────────────────────────────────
98addChunkCase('empty', hexInput(''), { note: 'returns NO chunks, not one empty chunk' })
99addChunkCase('single byte', hexInput('00'))
100addChunkCase('minSize-1', prngInput(1, MIN_SIZE - 1))
101addChunkCase('exactly minSize', prngInput(2, MIN_SIZE),
102 { note: 'final-chunk test is `remaining <= minSize`, so this is ONE chunk with no search' })
103addChunkCase('minSize+1', prngInput(3, MIN_SIZE + 1),
104 { note: 'first size that enters the cut-point search' })
105
106// ── crossing avg and max ─────────────────────────────────────────────────────
107addChunkCase('avgSize', prngInput(4, AVG_SIZE))
108addChunkCase('between avg and max', prngInput(5, (AVG_SIZE + MAX_SIZE) >> 1))
109addChunkCase('exactly maxSize', prngInput(6, MAX_SIZE))
110addChunkCase('several chunks', prngInput(7, MAX_SIZE * 4 + 1234))
111
112// ── degenerate content: the mask may never match ─────────────────────────────
113addChunkCase('all zero bytes', { buf: Buffer.alloc(MAX_SIZE * 2 + 7, 0x00), spec: { kind: 'const', byte: 0, length: MAX_SIZE * 2 + 7 } },
114 { note: 'constant input drives the gear hash deterministically; exercises the maxSize cutoff' })
115addChunkCase('all 0xff bytes', { buf: Buffer.alloc(MAX_SIZE + MIN_SIZE, 0xff), spec: { kind: 'const', byte: 255, length: MAX_SIZE + MIN_SIZE } })
116
117// ── forced tiers on identical input ──────────────────────────────────────────
118// The SAME bytes chunked under each tier must give different boundaries. If a
119// port ignores the tier parameter these four collapse to one answer.
120for (let t = 0; t < TIERS.length; t++) {
121 addChunkCase(`forced tier ${t}`, prngInput(99, 1048576), { tier: t })
122}
123
124// ── auto tier selection across a threshold ───────────────────────────────────
125addChunkCase('auto tier under 1MB', prngInput(11, 1048575))
126
127// ── hashChunk: pins the leaf construction ────────────────────────────────────
128const hashCases = [
129 { label: 'empty', hex: '' },
130 { label: 'single zero byte', hex: '00' },
131 { label: 'abc', hex: Buffer.from('abc').toString('hex') },
132 { label: '32 zero bytes', hex: '00'.repeat(32) },
133 { label: '255 bytes 0x00..0xfe', hex: Buffer.from(Array.from({ length: 255 }, (_, i) => i)).toString('hex') }
134].map(h => ({
135 ...h,
136 hash: hashChunk(Buffer.from(h.hex, 'hex'))
137}))
138
139// ── selectTier / getTierParams ───────────────────────────────────────────────
140const tierCases = [0, 1, 1048575, 1048576, 104857599, 104857600, 10737418239, 10737418240, 99999999999]
141 .map(size => ({ size, tier: selectTier(size), params: getTierParams(selectTier(size)) }))
142
143// ── mask degeneracy, asserted rather than assumed ────────────────────────────
144const maskCases = TIERS.map((t, i) => {
145 const large = (1 << Math.ceil(Math.log2(t.avgSize))) - 1
146 const small = (1 << Math.floor(Math.log2(t.avgSize))) - 1
147 return { tier: i, avgSize: t.avgSize, large, small, equal: large === small }
148})
149
150process.stdout.write(JSON.stringify({
151 generator: 'hyper-overlay/test/conformance/generate-chunker.mjs',
152 overlayVersion: pkg.version,
153 prng: 'lcg32: s = (s * 1664525 + 1013904223) mod 2^32; byte = (s >>> 24) & 0xff; one byte per step, s starts at seed',
154 hashConstruction: 'blake2b-256(0x00 || uint64_LE(byteLength) || data) — hypercore-crypto crypto.data, NOT plain blake2b',
155 constants: { MIN_SIZE, AVG_SIZE, MAX_SIZE, TIERS },
156 masks: maskCases,
157 tierSelection: tierCases,
158 hashChunk: hashCases,
159 chunkCases: cases
160}, null, 2) + '\n')
test/conformance/verify-chunker.mjs +107 -0
……@@ -0,0 +1,107 @@
1/**
2 * Verifies the FastCDC chunker fixtures against THIS build of lib/chunker.js.
3 *
4 * Doubles as the specification a native port must satisfy: every assertion here
5 * is one the C++ implementation has to pass with the same fixture file.
6 *
7 * A failure here is NOT a test to regenerate past. Chunk boundaries are content
8 * addresses — changing them means every peer on the old version shares nothing
9 * with every peer on the new one, and existing overlay content is orphaned. That
10 * is a migration, not a test update.
11 *
12 * Run: node test/conformance/verify-chunker.mjs
13 */
14
15import { readFileSync } from 'fs'
16import { fileURLToPath } from 'url'
17import { dirname, join } from 'path'
18import {
19 chunk, hashChunk, selectTier, getTierParams, TIERS, MIN_SIZE, AVG_SIZE, MAX_SIZE
20} from '../../lib/chunker.js'
21
22const __dir = dirname(fileURLToPath(import.meta.url))
23const v = JSON.parse(readFileSync(join(__dir, 'chunker-vectors.json'), 'utf8'))
24
25let pass = 0, fail = 0
26const failures = []
27const ok = (cond, what) => { if (cond) pass++; else { fail++; failures.push(what) } }
28
29function lcg32 (seed, length) {
30 const out = Buffer.alloc(length)
31 let s = seed >>> 0
32 for (let i = 0; i < length; i++) {
33 s = (Math.imul(s, 1664525) + 1013904223) >>> 0
34 out[i] = (s >>> 24) & 0xff
35 }
36 return out
37}
38
39function materialise (spec) {
40 if (spec.kind === 'hex') return Buffer.from(spec.hex, 'hex')
41 if (spec.kind === 'lcg32') return lcg32(spec.seed, spec.length)
42 if (spec.kind === 'const') return Buffer.alloc(spec.length, spec.byte)
43 throw new Error('unknown input spec: ' + spec.kind)
44}
45
46// ── constants ────────────────────────────────────────────────────────────────
47ok(v.constants.MIN_SIZE === MIN_SIZE && v.constants.AVG_SIZE === AVG_SIZE &&
48 v.constants.MAX_SIZE === MAX_SIZE, 'default chunk-size constants changed')
49ok(JSON.stringify(v.constants.TIERS) === JSON.stringify(TIERS), 'TIERS table changed')
50
51// ── mask degeneracy ──────────────────────────────────────────────────────────
52// Recorded rather than assumed: every tier's avgSize is a power of two, so the
53// two-phase normalisation collapses to a single mask. A port may implement one
54// mask, but only because THIS holds — if a future tier breaks it, both the port
55// and this assertion must change together.
56for (const m of v.masks) {
57 const t = TIERS[m.tier]
58 const large = (1 << Math.ceil(Math.log2(t.avgSize))) - 1
59 const small = (1 << Math.floor(Math.log2(t.avgSize))) - 1
60 ok(large === m.large && small === m.small, `tier ${m.tier}: mask values changed`)
61 ok((large === small) === m.equal, `tier ${m.tier}: mask degeneracy changed`)
62}
63
64// ── tier selection ───────────────────────────────────────────────────────────
65for (const t of v.tierSelection) {
66 ok(selectTier(t.size) === t.tier, `selectTier(${t.size}) => ${selectTier(t.size)}, want ${t.tier}`)
67 ok(JSON.stringify(getTierParams(t.tier)) === JSON.stringify(t.params),
68 `getTierParams(${t.tier}) changed`)
69}
70
71// ── hashChunk: the leaf construction ─────────────────────────────────────────
72for (const h of v.hashChunk) {
73 const got = hashChunk(Buffer.from(h.hex, 'hex'))
74 ok(got === h.hash, `hashChunk[${h.label}] => ${got}, want ${h.hash}`)
75}
76
77// ── chunk boundaries ─────────────────────────────────────────────────────────
78for (const t of v.chunkCases) {
79 const data = materialise(t.input)
80 const res = chunk(data, t.tier !== null ? { tier: t.tier } : undefined)
81 if (res.length !== t.chunkCount) {
82 fail++
83 failures.push(`${t.label}: ${res.length} chunks, want ${t.chunkCount}`)
84 continue
85 }
86 let bad = null
87 for (let i = 0; i < res.length; i++) {
88 const a = res[i], b = t.chunks[i]
89 if (a.offset !== b.offset || a.length !== b.length || a.hash !== b.hash) {
90 bad = `chunk ${i}: got {off:${a.offset},len:${a.length},hash:${a.hash.slice(0, 16)}} ` +
91 `want {off:${b.offset},len:${b.length},hash:${b.hash.slice(0, 16)}}`
92 break
93 }
94 }
95 // Tiling is re-checked here, not just at generation: a port could match every
96 // recorded boundary and still emit chunks that do not reconstruct the file.
97 const total = res.reduce((n, c) => n + c.length, 0)
98 if (!bad && total !== data.length) bad = `chunks cover ${total} of ${data.length} bytes`
99 ok(!bad, `${t.label}: ${bad}`)
100}
101
102console.log(`hyper-overlay chunker conformance @ ${v.overlayVersion}`)
103console.log(` ${pass} assertions pass, ${fail} fail`)
104console.log(` ${v.chunkCases.length} chunk cases, ${v.hashChunk.length} hash cases, ${v.tierSelection.length} tier cases`)
105for (const f of failures) console.log(` FAIL ${f}`)
106console.log(fail === 0 ? 'PASS' : 'FAIL')
107process.exit(fail === 0 ? 0 : 1)