2 * Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
8 * You may select, at your option, one of the above-listed licenses.
12 /*-************************************
14 **************************************/
15 #ifdef _MSC_VER /* Visual Studio */
16 # define _CRT_SECURE_NO_WARNINGS /* fgets */
17 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
18 # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */
22 /*-************************************
24 **************************************/
25 #include <stdlib.h> /* free */
26 #include <stdio.h> /* fgets, sscanf */
27 #include <string.h> /* strcmp */
29 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */
31 #include "zstd.h" /* ZSTD_VERSION_STRING */
32 #include "zstd_errors.h" /* ZSTD_getErrorCode */
33 #include "zstdmt_compress.h"
34 #define ZDICT_STATIC_LINKING_ONLY
35 #include "zdict.h" /* ZDICT_trainFromBuffer */
36 #include "datagen.h" /* RDG_genBuffer */
38 #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */
39 #include "xxhash.h" /* XXH64 */
41 #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */
44 /*-************************************
46 **************************************/
51 static const int FUZ_compressibility_default
= 50;
52 static const int nbTestsDefault
= 30000;
55 /*-************************************
57 **************************************/
58 #define DISPLAY(...) fprintf(stderr, __VA_ARGS__)
59 #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); }
60 static U32 g_displayLevel
= 2;
62 static const U64 g_refreshRate
= SEC_TO_MICRO
/ 6;
63 static UTIL_time_t g_displayClock
= UTIL_TIME_INITIALIZER
;
65 #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \
66 if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \
67 { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \
68 if (g_displayLevel>=4) fflush(stderr); } }
71 /*-*******************************************************
73 *********************************************************/
76 /* Declaring the function is it isn't unused */
77 void FUZ_bug976(void);
79 { /* these constants shall not depend on MIN() macro */
80 assert(ZSTD_HASHLOG_MAX
< 31);
81 assert(ZSTD_CHAINLOG_MAX
< 31);
85 /*-*******************************************************
87 *********************************************************/
88 #define MIN(a,b) ((a)<(b)?(a):(b))
89 #define MAX(a,b) ((a)>(b)?(a):(b))
91 #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r)))
92 static U32
FUZ_rand(U32
* src
)
94 static const U32 prime1
= 2654435761U;
95 static const U32 prime2
= 2246822519U;
99 rand32
= FUZ_rotl32(rand32
, 13);
104 static U32
FUZ_highbit32(U32 v32
)
107 if (v32
==0) return 0;
108 while (v32
) v32
>>= 1, nbBits
++;
113 /*=============================================
115 =============================================*/
116 #define CHECK_Z(f) { \
117 size_t const err = f; \
118 if (ZSTD_isError(err)) { \
119 DISPLAY("Error => %s : %s ", \
120 #f, ZSTD_getErrorName(err)); \
124 #define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error
125 #define CHECK(fn) { CHECK_V(err, fn); }
126 #define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; }
128 #define CHECK_OP(op, lhs, rhs) { \
129 if (!((lhs) op (rhs))) { \
130 DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \
131 goto _output_error; \
134 #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs)
135 #define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs)
138 /*=============================================
140 =============================================*/
141 #if defined(__APPLE__) && defined(__MACH__)
143 #include <malloc/malloc.h> /* malloc_size */
146 unsigned long long totalMalloc
;
147 size_t currentMalloc
;
153 static const mallocCounter_t INIT_MALLOC_COUNTER
= { 0, 0, 0, 0, 0 };
155 static void* FUZ_mallocDebug(void* counter
, size_t size
)
157 mallocCounter_t
* const mcPtr
= (mallocCounter_t
*)counter
;
158 void* const ptr
= malloc(size
);
159 if (ptr
==NULL
) return NULL
;
160 DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n",
161 (unsigned)(size
>> 10), (unsigned)(malloc_size(ptr
) >> 10)); /* OS-X specific */
162 mcPtr
->totalMalloc
+= size
;
163 mcPtr
->currentMalloc
+= size
;
164 if (mcPtr
->currentMalloc
> mcPtr
->peakMalloc
)
165 mcPtr
->peakMalloc
= mcPtr
->currentMalloc
;
166 mcPtr
->nbMalloc
+= 1;
170 static void FUZ_freeDebug(void* counter
, void* address
)
172 mallocCounter_t
* const mcPtr
= (mallocCounter_t
*)counter
;
173 DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address
) >> 10));
175 mcPtr
->currentMalloc
-= malloc_size(address
); /* OS-X specific */
179 static void FUZ_displayMallocStats(mallocCounter_t count
)
181 DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n",
182 (unsigned)(count
.peakMalloc
>> 10),
184 (unsigned)(count
.totalMalloc
>> 10));
187 static int FUZ_mallocTests_internal(unsigned seed
, double compressibility
, unsigned part
,
188 void* inBuffer
, size_t inSize
, void* outBuffer
, size_t outSize
)
190 /* test only played in verbose mode, as they are long */
191 if (g_displayLevel
<3) return 0;
193 /* Create compressible noise */
194 if (!inBuffer
|| !outBuffer
) {
195 DISPLAY("Not enough memory, aborting\n");
198 RDG_genBuffer(inBuffer
, inSize
, compressibility
, 0. /*auto*/, seed
);
200 /* simple compression tests */
202 { int compressionLevel
;
203 for (compressionLevel
=1; compressionLevel
<=6; compressionLevel
++) {
204 mallocCounter_t malcount
= INIT_MALLOC_COUNTER
;
205 ZSTD_customMem
const cMem
= { FUZ_mallocDebug
, FUZ_freeDebug
, &malcount
};
206 ZSTD_CCtx
* const cctx
= ZSTD_createCCtx_advanced(cMem
);
207 CHECK_Z( ZSTD_compressCCtx(cctx
, outBuffer
, outSize
, inBuffer
, inSize
, compressionLevel
) );
209 DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel
);
210 FUZ_displayMallocStats(malcount
);
213 /* streaming compression tests */
215 { int compressionLevel
;
216 for (compressionLevel
=1; compressionLevel
<=6; compressionLevel
++) {
217 mallocCounter_t malcount
= INIT_MALLOC_COUNTER
;
218 ZSTD_customMem
const cMem
= { FUZ_mallocDebug
, FUZ_freeDebug
, &malcount
};
219 ZSTD_CCtx
* const cstream
= ZSTD_createCStream_advanced(cMem
);
220 ZSTD_outBuffer out
= { outBuffer
, outSize
, 0 };
221 ZSTD_inBuffer in
= { inBuffer
, inSize
, 0 };
222 CHECK_Z( ZSTD_initCStream(cstream
, compressionLevel
) );
223 CHECK_Z( ZSTD_compressStream(cstream
, &out
, &in
) );
224 CHECK_Z( ZSTD_endStream(cstream
, &out
) );
225 ZSTD_freeCStream(cstream
);
226 DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel
);
227 FUZ_displayMallocStats(malcount
);
230 /* advanced MT API test */
232 { unsigned nbThreads
;
233 for (nbThreads
=1; nbThreads
<=4; nbThreads
++) {
234 int compressionLevel
;
235 for (compressionLevel
=1; compressionLevel
<=6; compressionLevel
++) {
236 mallocCounter_t malcount
= INIT_MALLOC_COUNTER
;
237 ZSTD_customMem
const cMem
= { FUZ_mallocDebug
, FUZ_freeDebug
, &malcount
};
238 ZSTD_CCtx
* const cctx
= ZSTD_createCCtx_advanced(cMem
);
239 CHECK_Z( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_compressionLevel
, compressionLevel
) );
240 CHECK_Z( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_nbWorkers
, nbThreads
) );
241 CHECK_Z( ZSTD_compress2(cctx
, outBuffer
, outSize
, inBuffer
, inSize
) );
243 DISPLAYLEVEL(3, "compress_generic,-T%u,end level %i : ",
244 nbThreads
, compressionLevel
);
245 FUZ_displayMallocStats(malcount
);
248 /* advanced MT streaming API test */
250 { unsigned nbThreads
;
251 for (nbThreads
=1; nbThreads
<=4; nbThreads
++) {
252 int compressionLevel
;
253 for (compressionLevel
=1; compressionLevel
<=6; compressionLevel
++) {
254 mallocCounter_t malcount
= INIT_MALLOC_COUNTER
;
255 ZSTD_customMem
const cMem
= { FUZ_mallocDebug
, FUZ_freeDebug
, &malcount
};
256 ZSTD_CCtx
* const cctx
= ZSTD_createCCtx_advanced(cMem
);
257 ZSTD_outBuffer out
= { outBuffer
, outSize
, 0 };
258 ZSTD_inBuffer in
= { inBuffer
, inSize
, 0 };
259 CHECK_Z( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_compressionLevel
, compressionLevel
) );
260 CHECK_Z( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_nbWorkers
, nbThreads
) );
261 CHECK_Z( ZSTD_compressStream2(cctx
, &out
, &in
, ZSTD_e_continue
) );
262 while ( ZSTD_compressStream2(cctx
, &out
, &in
, ZSTD_e_end
) ) {}
264 DISPLAYLEVEL(3, "compress_generic,-T%u,continue level %i : ",
265 nbThreads
, compressionLevel
);
266 FUZ_displayMallocStats(malcount
);
272 static int FUZ_mallocTests(unsigned seed
, double compressibility
, unsigned part
)
274 size_t const inSize
= 64 MB
+ 16 MB
+ 4 MB
+ 1 MB
+ 256 KB
+ 64 KB
; /* 85.3 MB */
275 size_t const outSize
= ZSTD_compressBound(inSize
);
276 void* const inBuffer
= malloc(inSize
);
277 void* const outBuffer
= malloc(outSize
);
280 /* Create compressible noise */
281 if (!inBuffer
|| !outBuffer
) {
282 DISPLAY("Not enough memory, aborting \n");
286 result
= FUZ_mallocTests_internal(seed
, compressibility
, part
,
287 inBuffer
, inSize
, outBuffer
, outSize
);
296 static int FUZ_mallocTests(unsigned seed
, double compressibility
, unsigned part
)
298 (void)seed
; (void)compressibility
; (void)part
;
304 /*=============================================
306 =============================================*/
308 static int basicUnitTests(U32 seed
, double compressibility
)
310 size_t const CNBuffSize
= 5 MB
;
311 void* const CNBuffer
= malloc(CNBuffSize
);
312 size_t const compressedBufferSize
= ZSTD_compressBound(CNBuffSize
);
313 void* const compressedBuffer
= malloc(compressedBufferSize
);
314 void* const decodedBuffer
= malloc(CNBuffSize
);
319 /* Create compressible noise */
320 if (!CNBuffer
|| !compressedBuffer
|| !decodedBuffer
) {
321 DISPLAY("Not enough memory, aborting\n");
325 RDG_genBuffer(CNBuffer
, CNBuffSize
, compressibility
, 0., seed
);
328 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb
++);
329 { const char* errorString
= ZSTD_getErrorName(0);
330 DISPLAYLEVEL(3, "OK : %s \n", errorString
);
333 DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb
++);
334 { const char* errorString
= ZSTD_getErrorName(499);
335 DISPLAYLEVEL(3, "OK : %s \n", errorString
);
338 DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb
++);
339 { int const mcl
= ZSTD_minCLevel();
340 DISPLAYLEVEL(3, "%i (OK) \n", mcl
);
343 DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb
++, (unsigned)CNBuffSize
);
344 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
345 if (cctx
==NULL
) goto _output_error
;
346 CHECKPLUS(r
, ZSTD_compressCCtx(cctx
,
347 compressedBuffer
, compressedBufferSize
,
348 CNBuffer
, CNBuffSize
, 1),
350 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize
, (double)cSize
/CNBuffSize
*100);
352 DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb
++);
353 { size_t const cctxSize
= ZSTD_sizeof_CCtx(cctx
);
354 DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize
);
359 DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb
++);
361 char const skippable8
[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff";
362 size_t const size
= ZSTD_decompress(NULL
, 0, skippable8
, 8);
363 if (!ZSTD_isError(size
)) goto _output_error
;
365 DISPLAYLEVEL(3, "OK \n");
368 DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb
++);
369 { unsigned long long const rSize
= ZSTD_getFrameContentSize(compressedBuffer
, cSize
);
370 if (rSize
!= CNBuffSize
) goto _output_error
;
372 DISPLAYLEVEL(3, "OK \n");
374 DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb
++);
375 { unsigned long long const rSize
= ZSTD_findDecompressedSize(compressedBuffer
, cSize
);
376 if (rSize
!= CNBuffSize
) goto _output_error
;
378 DISPLAYLEVEL(3, "OK \n");
380 DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb
++);
382 unsigned long long bound
= ZSTD_decompressBound(compressedBuffer
, cSize
);
383 if (bound
!= CNBuffSize
) goto _output_error
;
385 DISPLAYLEVEL(3, "OK \n");
387 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb
++);
389 unsigned long long bound
= ZSTD_decompressBound(compressedBuffer
, cSize
- 1);
390 if (bound
!= ZSTD_CONTENTSIZE_ERROR
) goto _output_error
;
392 DISPLAYLEVEL(3, "OK \n");
394 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb
++, (unsigned)CNBuffSize
);
395 { size_t const r
= ZSTD_decompress(decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
);
396 if (r
!= CNBuffSize
) goto _output_error
; }
397 DISPLAYLEVEL(3, "OK \n");
399 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb
++);
401 for (u
=0; u
<CNBuffSize
; u
++) {
402 if (((BYTE
*)decodedBuffer
)[u
] != ((BYTE
*)CNBuffer
)[u
]) goto _output_error
;;
404 DISPLAYLEVEL(3, "OK \n");
407 DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb
++);
408 { ZSTD_DCtx
* const dctx
= ZSTD_createDCtx(); assert(dctx
!= NULL
);
409 { size_t const r
= ZSTD_decompress_usingDict(dctx
,
410 decodedBuffer
, CNBuffSize
,
411 compressedBuffer
, cSize
,
413 if (r
!= CNBuffSize
) goto _output_error
;
417 DISPLAYLEVEL(3, "OK \n");
419 DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb
++);
420 { ZSTD_DCtx
* const dctx
= ZSTD_createDCtx(); assert(dctx
!= NULL
);
421 { size_t const r
= ZSTD_decompress_usingDDict(dctx
,
422 decodedBuffer
, CNBuffSize
,
423 compressedBuffer
, cSize
,
425 if (r
!= CNBuffSize
) goto _output_error
;
429 DISPLAYLEVEL(3, "OK \n");
431 DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb
++);
432 { size_t const r
= ZSTD_decompress(decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
-1);
433 if (!ZSTD_isError(r
)) goto _output_error
;
434 if (ZSTD_getErrorCode((size_t)r
) != ZSTD_error_srcSize_wrong
) goto _output_error
; }
435 DISPLAYLEVEL(3, "OK \n");
437 DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb
++);
438 { size_t const r
= ZSTD_decompress(decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
+1);
439 if (!ZSTD_isError(r
)) goto _output_error
;
440 if (ZSTD_getErrorCode(r
) != ZSTD_error_srcSize_wrong
) goto _output_error
; }
441 DISPLAYLEVEL(3, "OK \n");
443 DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb
++);
444 { size_t const r
= ZSTD_decompress(decodedBuffer
, CNBuffSize
, compressedBuffer
, compressedBufferSize
);
445 if (!ZSTD_isError(r
)) goto _output_error
;
446 if (ZSTD_getErrorCode(r
) != ZSTD_error_srcSize_wrong
) goto _output_error
; }
447 DISPLAYLEVEL(3, "OK \n");
449 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb
++);
450 { /* create compressed buffer with content size missing */
451 ZSTD_CCtx
* cctx
= ZSTD_createCCtx();
452 CHECK_Z( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_contentSizeFlag
, 0) );
453 CHECKPLUS(r
, ZSTD_compress2(cctx
,
454 compressedBuffer
, compressedBufferSize
,
455 CNBuffer
, CNBuffSize
),
459 { /* ensure frame content size is missing */
460 ZSTD_frameHeader zfh
;
461 size_t const ret
= ZSTD_getFrameHeader(&zfh
, compressedBuffer
, compressedBufferSize
);
462 if (ret
!= 0 || zfh
.frameContentSize
!= ZSTD_CONTENTSIZE_UNKNOWN
) goto _output_error
;
464 { /* ensure CNBuffSize <= decompressBound */
465 unsigned long long const bound
= ZSTD_decompressBound(compressedBuffer
, compressedBufferSize
);
466 if (CNBuffSize
> bound
) goto _output_error
;
468 DISPLAYLEVEL(3, "OK \n");
470 DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb
++);
471 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
472 size_t const r
= ZSTD_compressCCtx(cctx
, compressedBuffer
, compressedBufferSize
, NULL
, 0, 19);
473 if (ZSTD_isError(r
)) goto _output_error
;
474 if (ZSTD_sizeof_CCtx(cctx
) > (1U << 20)) goto _output_error
;
478 DISPLAYLEVEL(3, "OK \n");
480 DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb
++);
481 { size_t const r
= ZSTD_decompress(NULL
, 0, compressedBuffer
, cSize
);
482 if (ZSTD_isError(r
)) goto _output_error
;
483 if (r
!= 0) goto _output_error
;
485 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
486 ZSTD_outBuffer output
;
487 if (cctx
==NULL
) goto _output_error
;
488 output
.dst
= compressedBuffer
;
489 output
.size
= compressedBufferSize
;
491 CHECK_Z( ZSTD_initCStream(cctx
, 1) ); /* content size unknown */
492 CHECK_Z( ZSTD_flushStream(cctx
, &output
) ); /* ensure no possibility to "concatenate" and determine the content size */
493 CHECK_Z( ZSTD_endStream(cctx
, &output
) );
495 /* single scan decompression */
496 { size_t const r
= ZSTD_decompress(NULL
, 0, compressedBuffer
, output
.pos
);
497 if (ZSTD_isError(r
)) goto _output_error
;
498 if (r
!= 0) goto _output_error
;
500 /* streaming decompression */
501 { ZSTD_DCtx
* const dstream
= ZSTD_createDStream();
502 ZSTD_inBuffer dinput
;
503 ZSTD_outBuffer doutput
;
505 if (dstream
==NULL
) goto _output_error
;
506 dinput
.src
= compressedBuffer
;
512 CHECK_Z ( ZSTD_initDStream(dstream
) );
513 for (ipos
=1; ipos
<=output
.pos
; ipos
++) {
515 CHECK_Z ( ZSTD_decompressStream(dstream
, &doutput
, &dinput
) );
517 if (doutput
.pos
!= 0) goto _output_error
;
518 ZSTD_freeDStream(dstream
);
521 DISPLAYLEVEL(3, "OK \n");
523 DISPLAYLEVEL(3, "test%3d : re-use CCtx with expanding block size : ", testNb
++);
524 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
525 ZSTD_parameters
const params
= ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN
, 0);
526 assert(params
.fParams
.contentSizeFlag
== 1); /* block size will be adapted if pledgedSrcSize is enabled */
527 CHECK_Z( ZSTD_compressBegin_advanced(cctx
, NULL
, 0, params
, 1 /*pledgedSrcSize*/) );
528 CHECK_Z( ZSTD_compressEnd(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, 1) ); /* creates a block size of 1 */
530 CHECK_Z( ZSTD_compressBegin_advanced(cctx
, NULL
, 0, params
, ZSTD_CONTENTSIZE_UNKNOWN
) ); /* re-use same parameters */
531 { size_t const inSize
= 2* 128 KB
;
532 size_t const outSize
= ZSTD_compressBound(inSize
);
533 CHECK_Z( ZSTD_compressEnd(cctx
, compressedBuffer
, outSize
, CNBuffer
, inSize
) );
534 /* will fail if blockSize is not resized */
538 DISPLAYLEVEL(3, "OK \n");
540 DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb
++);
541 { size_t const sampleSize
= 30;
544 ((char*)CNBuffer
)[i
] = (char)i
; /* ensure no match during initial section */
545 memcpy((char*)CNBuffer
+ 20, CNBuffer
, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */
546 for (i
=1; i
<=19; i
++) {
547 ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
549 DISPLAYLEVEL(5, "l%i ", i
);
550 size1
= ZSTD_compressCCtx(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, sampleSize
, i
);
553 size2
= ZSTD_compressCCtx(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, sampleSize
, i
);
555 CHECK_EQ(size1
, size2
);
557 CHECK_Z( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_compressionLevel
, i
) );
558 size2
= ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, sampleSize
);
560 CHECK_EQ(size1
, size2
);
562 size2
= ZSTD_compress2(cctx
, compressedBuffer
, ZSTD_compressBound(sampleSize
) - 1, CNBuffer
, sampleSize
); /* force streaming, as output buffer is not large enough to guarantee success */
564 CHECK_EQ(size1
, size2
);
570 inb
.size
= sampleSize
;
571 outb
.dst
= compressedBuffer
;
573 outb
.size
= ZSTD_compressBound(sampleSize
) - 1; /* force streaming, as output buffer is not large enough to guarantee success */
574 CHECK_Z( ZSTD_compressStream2(cctx
, &outb
, &inb
, ZSTD_e_end
) );
575 assert(inb
.pos
== inb
.size
);
576 CHECK_EQ(size1
, outb
.pos
);
582 DISPLAYLEVEL(3, "OK \n");
584 DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb
++);
585 { size_t const sampleSize
= 1024;
586 ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
592 outb
.dst
= compressedBuffer
;
594 outb
.size
= compressedBufferSize
;
595 CHECK_Z( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_compressionLevel
, ZSTD_maxCLevel()) );
597 inb
.size
= sampleSize
; /* start with something, so that context is already used */
598 CHECK_Z( ZSTD_compressStream2(cctx
, &outb
, &inb
, ZSTD_e_end
) ); /* will break internal assert if stats_init is not disabled */
599 assert(inb
.pos
== inb
.size
);
600 outb
.pos
= 0; /* cancel output */
602 CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx
, sampleSize
) );
603 inb
.size
= 4; /* too small size : compression will be skipped */
605 CHECK_Z( ZSTD_compressStream2(cctx
, &outb
, &inb
, ZSTD_e_flush
) );
606 assert(inb
.pos
== inb
.size
);
608 inb
.size
+= 5; /* too small size : compression will be skipped */
609 CHECK_Z( ZSTD_compressStream2(cctx
, &outb
, &inb
, ZSTD_e_flush
) );
610 assert(inb
.pos
== inb
.size
);
612 inb
.size
+= 11; /* small enough to attempt compression */
613 CHECK_Z( ZSTD_compressStream2(cctx
, &outb
, &inb
, ZSTD_e_flush
) );
614 assert(inb
.pos
== inb
.size
);
616 assert(inb
.pos
< sampleSize
);
617 inb
.size
= sampleSize
; /* large enough to trigger stats_init, but no longer at beginning */
618 CHECK_Z( ZSTD_compressStream2(cctx
, &outb
, &inb
, ZSTD_e_end
) ); /* will break internal assert if stats_init is not disabled */
619 assert(inb
.pos
== inb
.size
);
622 DISPLAYLEVEL(3, "OK \n");
624 DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb
++);
625 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
626 ZSTD_outBuffer out
= {NULL
, 0, 0};
627 ZSTD_inBuffer in
= {NULL
, 0, 0};
630 CHECK_Z(ZSTD_CCtx_getParameter(cctx
, ZSTD_c_compressionLevel
, &value
));
632 CHECK_Z(ZSTD_CCtx_getParameter(cctx
, ZSTD_c_hashLog
, &value
));
634 CHECK_Z(ZSTD_CCtx_setParameter(cctx
, ZSTD_c_hashLog
, ZSTD_HASHLOG_MIN
));
635 CHECK_Z(ZSTD_CCtx_getParameter(cctx
, ZSTD_c_compressionLevel
, &value
));
637 CHECK_Z(ZSTD_CCtx_getParameter(cctx
, ZSTD_c_hashLog
, &value
));
638 CHECK_EQ(value
, ZSTD_HASHLOG_MIN
);
639 CHECK_Z(ZSTD_CCtx_setParameter(cctx
, ZSTD_c_compressionLevel
, 7));
640 CHECK_Z(ZSTD_CCtx_getParameter(cctx
, ZSTD_c_compressionLevel
, &value
));
642 CHECK_Z(ZSTD_CCtx_getParameter(cctx
, ZSTD_c_hashLog
, &value
));
643 CHECK_EQ(value
, ZSTD_HASHLOG_MIN
);
644 /* Start a compression job */
645 ZSTD_compressStream2(cctx
, &out
, &in
, ZSTD_e_continue
);
646 CHECK_Z(ZSTD_CCtx_getParameter(cctx
, ZSTD_c_compressionLevel
, &value
));
648 CHECK_Z(ZSTD_CCtx_getParameter(cctx
, ZSTD_c_hashLog
, &value
));
649 CHECK_EQ(value
, ZSTD_HASHLOG_MIN
);
651 ZSTD_CCtx_reset(cctx
, ZSTD_reset_session_only
);
652 CHECK_Z(ZSTD_CCtx_getParameter(cctx
, ZSTD_c_compressionLevel
, &value
));
654 CHECK_Z(ZSTD_CCtx_getParameter(cctx
, ZSTD_c_hashLog
, &value
));
655 CHECK_EQ(value
, ZSTD_HASHLOG_MIN
);
656 /* Reset the parameters */
657 ZSTD_CCtx_reset(cctx
, ZSTD_reset_parameters
);
658 CHECK_Z(ZSTD_CCtx_getParameter(cctx
, ZSTD_c_compressionLevel
, &value
));
660 CHECK_Z(ZSTD_CCtx_getParameter(cctx
, ZSTD_c_hashLog
, &value
));
665 DISPLAYLEVEL(3, "OK \n");
667 /* this test is really too long, and should be made faster */
668 DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb
++);
669 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
670 ZSTD_parameters params
= ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN
, 0);
671 size_t const nbCompressions
= ((1U << 31) / CNBuffSize
) + 2; /* ensure U32 overflow protection is triggered */
673 assert(cctx
!= NULL
);
674 params
.fParams
.contentSizeFlag
= 0;
675 params
.cParams
.windowLog
= ZSTD_WINDOWLOG_MAX
;
676 for (cnb
= 0; cnb
< nbCompressions
; ++cnb
) {
677 DISPLAYLEVEL(6, "run %zu / %zu \n", cnb
, nbCompressions
);
678 CHECK_Z( ZSTD_compressBegin_advanced(cctx
, NULL
, 0, params
, ZSTD_CONTENTSIZE_UNKNOWN
) ); /* re-use same parameters */
679 CHECK_Z( ZSTD_compressEnd(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, CNBuffSize
) );
683 DISPLAYLEVEL(3, "OK \n");
685 DISPLAYLEVEL(3, "test%3d : size down context : ", testNb
++);
686 { ZSTD_CCtx
* const largeCCtx
= ZSTD_createCCtx();
687 assert(largeCCtx
!= NULL
);
688 CHECK_Z( ZSTD_compressBegin(largeCCtx
, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */
689 CHECK_Z( ZSTD_compressEnd(largeCCtx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, 1) );
690 { size_t const largeCCtxSize
= ZSTD_sizeof_CCtx(largeCCtx
); /* size of context must be measured after compression */
691 { ZSTD_CCtx
* const smallCCtx
= ZSTD_createCCtx();
692 assert(smallCCtx
!= NULL
);
693 CHECK_Z(ZSTD_compressCCtx(smallCCtx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, 1, 1));
694 { size_t const smallCCtxSize
= ZSTD_sizeof_CCtx(smallCCtx
);
695 DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ",
696 largeCCtxSize
>>10, smallCCtxSize
>>10);
697 assert(largeCCtxSize
> 32* smallCCtxSize
); /* note : "too large" definition is handled within zstd_compress.c .
698 * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */
700 ZSTD_freeCCtx(smallCCtx
);
702 { U32
const maxNbAttempts
= 1100; /* nb of usages before triggering size down is handled within zstd_compress.c.
703 * currently defined as 128x, but could be adjusted in the future.
704 * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */
706 for (u
=0; u
<maxNbAttempts
; u
++) {
707 CHECK_Z(ZSTD_compressCCtx(largeCCtx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, 1, 1));
708 if (ZSTD_sizeof_CCtx(largeCCtx
) < largeCCtxSize
) break; /* sized down */
710 DISPLAYLEVEL(5, "size down after %u attempts : ", u
);
711 if (u
==maxNbAttempts
) goto _output_error
; /* no sizedown happened */
714 ZSTD_freeCCtx(largeCCtx
);
716 DISPLAYLEVEL(3, "OK \n");
718 /* Static CCtx tests */
719 #define STATIC_CCTX_LEVEL 3
720 DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u :", testNb
++, STATIC_CCTX_LEVEL
);
721 { size_t const staticCCtxSize
= ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL
);
722 void* const staticCCtxBuffer
= malloc(staticCCtxSize
);
723 size_t const staticDCtxSize
= ZSTD_estimateDCtxSize();
724 void* const staticDCtxBuffer
= malloc(staticDCtxSize
);
725 if (staticCCtxBuffer
==NULL
|| staticDCtxBuffer
==NULL
) {
726 free(staticCCtxBuffer
);
727 free(staticDCtxBuffer
);
728 DISPLAY("Not enough memory, aborting\n");
732 { ZSTD_CCtx
* staticCCtx
= ZSTD_initStaticCCtx(staticCCtxBuffer
, staticCCtxSize
);
733 ZSTD_DCtx
* staticDCtx
= ZSTD_initStaticDCtx(staticDCtxBuffer
, staticDCtxSize
);
734 if ((staticCCtx
==NULL
) || (staticDCtx
==NULL
)) goto _output_error
;
735 DISPLAYLEVEL(3, "OK \n");
737 DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb
++, STATIC_CCTX_LEVEL
);
738 { size_t const r
= ZSTD_compressBegin(staticCCtx
, STATIC_CCTX_LEVEL
);
739 if (ZSTD_isError(r
)) goto _output_error
; }
740 DISPLAYLEVEL(3, "OK \n");
742 DISPLAYLEVEL(3, "test%3i : simple compression test with static CCtx : ", testNb
++);
743 CHECKPLUS(r
, ZSTD_compressCCtx(staticCCtx
,
744 compressedBuffer
, compressedBufferSize
,
745 CNBuffer
, CNBuffSize
, STATIC_CCTX_LEVEL
),
747 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n",
748 (unsigned)cSize
, (double)cSize
/CNBuffSize
*100);
750 DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb
++);
751 { size_t const r
= ZSTD_decompressDCtx(staticDCtx
,
752 decodedBuffer
, CNBuffSize
,
753 compressedBuffer
, cSize
);
754 if (r
!= CNBuffSize
) goto _output_error
; }
755 DISPLAYLEVEL(3, "OK \n");
757 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb
++);
759 for (u
=0; u
<CNBuffSize
; u
++) {
760 if (((BYTE
*)decodedBuffer
)[u
] != ((BYTE
*)CNBuffer
)[u
])
763 DISPLAYLEVEL(3, "OK \n");
765 DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb
++);
766 { size_t const r
= ZSTD_compressBegin(staticCCtx
, ZSTD_maxCLevel());
767 if (!ZSTD_isError(r
)) goto _output_error
; }
768 DISPLAYLEVEL(3, "OK \n");
770 DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb
++, 1);
771 { size_t const r
= ZSTD_compressBegin(staticCCtx
, 1);
772 if (ZSTD_isError(r
)) goto _output_error
; }
773 DISPLAYLEVEL(3, "OK \n");
775 DISPLAYLEVEL(3, "test%3i : init CStream for small level %u : ", testNb
++, 1);
776 { size_t const r
= ZSTD_initCStream(staticCCtx
, 1);
777 if (ZSTD_isError(r
)) goto _output_error
; }
778 DISPLAYLEVEL(3, "OK \n");
780 DISPLAYLEVEL(3, "test%3i : init CStream with dictionary (should fail) : ", testNb
++);
781 { size_t const r
= ZSTD_initCStream_usingDict(staticCCtx
, CNBuffer
, 64 KB
, 1);
782 if (!ZSTD_isError(r
)) goto _output_error
; }
783 DISPLAYLEVEL(3, "OK \n");
785 DISPLAYLEVEL(3, "test%3i : init DStream (should fail) : ", testNb
++);
786 { size_t const r
= ZSTD_initDStream(staticDCtx
);
787 if (ZSTD_isError(r
)) goto _output_error
; }
788 { ZSTD_outBuffer output
= { decodedBuffer
, CNBuffSize
, 0 };
789 ZSTD_inBuffer input
= { compressedBuffer
, ZSTD_FRAMEHEADERSIZE_MAX
+1, 0 };
790 size_t const r
= ZSTD_decompressStream(staticDCtx
, &output
, &input
);
791 if (!ZSTD_isError(r
)) goto _output_error
;
793 DISPLAYLEVEL(3, "OK \n");
795 free(staticCCtxBuffer
);
796 free(staticDCtxBuffer
);
799 DISPLAYLEVEL(3, "test%3i : Static negative levels : ", testNb
++);
800 { size_t const cctxSizeN1
= ZSTD_estimateCCtxSize(-1);
801 size_t const cctxSizeP1
= ZSTD_estimateCCtxSize(1);
802 size_t const cstreamSizeN1
= ZSTD_estimateCStreamSize(-1);
803 size_t const cstreamSizeP1
= ZSTD_estimateCStreamSize(1);
805 if (!(0 < cctxSizeN1
&& cctxSizeN1
<= cctxSizeP1
)) goto _output_error
;
806 if (!(0 < cstreamSizeN1
&& cstreamSizeN1
<= cstreamSizeP1
)) goto _output_error
;
808 DISPLAYLEVEL(3, "OK \n");
811 /* ZSTDMT simple MT compression test */
812 DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb
++);
813 { ZSTDMT_CCtx
* mtctx
= ZSTDMT_createCCtx(2);
815 DISPLAY("mtctx : mot enough memory, aborting \n");
819 DISPLAYLEVEL(3, "OK \n");
821 DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb
++, (unsigned)CNBuffSize
);
822 CHECKPLUS(r
, ZSTDMT_compressCCtx(mtctx
,
823 compressedBuffer
, compressedBufferSize
,
824 CNBuffer
, CNBuffSize
,
827 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize
, (double)cSize
/CNBuffSize
*100);
829 DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb
++);
830 { unsigned long long const rSize
= ZSTD_getFrameContentSize(compressedBuffer
, cSize
);
831 if (rSize
!= CNBuffSize
) {
832 DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize
, (unsigned)CNBuffSize
);
835 DISPLAYLEVEL(3, "OK \n");
837 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb
++, (unsigned)CNBuffSize
);
838 { size_t const r
= ZSTD_decompress(decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
);
839 if (r
!= CNBuffSize
) goto _output_error
; }
840 DISPLAYLEVEL(3, "OK \n");
842 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb
++);
844 for (u
=0; u
<CNBuffSize
; u
++) {
845 if (((BYTE
*)decodedBuffer
)[u
] != ((BYTE
*)CNBuffer
)[u
]) goto _output_error
;;
847 DISPLAYLEVEL(3, "OK \n");
849 DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb
++);
850 { ZSTD_parameters params
= ZSTD_getParams(1, CNBuffSize
, 0);
851 params
.fParams
.checksumFlag
= 1;
852 params
.fParams
.contentSizeFlag
= 1;
853 CHECKPLUS(r
, ZSTDMT_compress_advanced(mtctx
,
854 compressedBuffer
, compressedBufferSize
,
855 CNBuffer
, CNBuffSize
,
856 NULL
, params
, 3 /*overlapRLog*/),
859 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize
, (double)cSize
/CNBuffSize
*100);
861 DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb
++, (unsigned)CNBuffSize
);
862 { size_t const r
= ZSTD_decompress(decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
);
863 if (r
!= CNBuffSize
) goto _output_error
; }
864 DISPLAYLEVEL(3, "OK \n");
866 ZSTDMT_freeCCtx(mtctx
);
869 DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb
++)
870 { ZSTD_CCtx
* cctx
= ZSTD_createCCtx();
871 size_t cSize1
, cSize2
;
872 CHECK( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_compressionLevel
, 1) );
873 CHECK( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_nbWorkers
, 2) );
874 cSize1
= ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, CNBuffSize
);
876 CHECK( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_literalCompressionMode
, ZSTD_lcm_uncompressed
) );
877 cSize2
= ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, CNBuffSize
);
879 CHECK_LT(cSize1
, cSize2
);
882 DISPLAYLEVEL(3, "OK \n");
884 DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb
++)
885 { ZSTD_CCtx
* cctx
= ZSTD_createCCtx();
886 /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so
887 * ZSTDMT is forced to not take the shortcut.
889 CHECK( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_compressionLevel
, 1) );
890 CHECK( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_nbWorkers
, 1) );
891 CHECK( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_rsyncable
, 1) );
892 CHECK( ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
- 1, CNBuffer
, CNBuffSize
) );
895 DISPLAYLEVEL(3, "OK \n");
897 DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb
++)
898 { ZSTD_CCtx_params
* params
= ZSTD_createCCtxParams();
900 /* Check that the overlap log and job size are unset. */
901 CHECK( ZSTD_CCtxParams_getParameter(params
, ZSTD_c_overlapLog
, &value
) );
903 CHECK( ZSTD_CCtxParams_getParameter(params
, ZSTD_c_jobSize
, &value
) );
905 /* Set and check the overlap log and job size. */
906 CHECK( ZSTD_CCtxParams_setParameter(params
, ZSTD_c_overlapLog
, 5) );
907 CHECK( ZSTD_CCtxParams_setParameter(params
, ZSTD_c_jobSize
, 2 MB
) );
908 CHECK( ZSTD_CCtxParams_getParameter(params
, ZSTD_c_overlapLog
, &value
) );
910 CHECK( ZSTD_CCtxParams_getParameter(params
, ZSTD_c_jobSize
, &value
) );
911 CHECK_EQ(value
, 2 MB
);
912 /* Set the number of workers and check the overlap log and job size. */
913 CHECK( ZSTD_CCtxParams_setParameter(params
, ZSTD_c_nbWorkers
, 2) );
914 CHECK( ZSTD_CCtxParams_getParameter(params
, ZSTD_c_overlapLog
, &value
) );
916 CHECK( ZSTD_CCtxParams_getParameter(params
, ZSTD_c_jobSize
, &value
) );
917 CHECK_EQ(value
, 2 MB
);
918 ZSTD_freeCCtxParams(params
);
921 DISPLAYLEVEL(3, "OK \n");
923 /* Simple API multiframe test */
924 DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb
++);
928 /* only use the first half so we don't push against size limit of compressedBuffer */
929 size_t const segSize
= (CNBuffSize
/ 2) / segs
;
930 for (i
= 0; i
< segs
; i
++) {
931 CHECK_V(r
, ZSTD_compress(
932 (BYTE
*)compressedBuffer
+ off
, CNBuffSize
- off
,
933 (BYTE
*)CNBuffer
+ segSize
* i
,
937 /* insert skippable frame */
938 const U32 skipLen
= 129 KB
;
939 MEM_writeLE32((BYTE
*)compressedBuffer
+ off
, ZSTD_MAGIC_SKIPPABLE_START
);
940 MEM_writeLE32((BYTE
*)compressedBuffer
+ off
+ 4, skipLen
);
941 off
+= skipLen
+ ZSTD_SKIPPABLEHEADERSIZE
;
946 DISPLAYLEVEL(3, "OK \n");
948 DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb
++);
949 { unsigned long long const r
= ZSTD_findDecompressedSize(compressedBuffer
, cSize
);
950 if (r
!= CNBuffSize
/ 2) goto _output_error
; }
951 DISPLAYLEVEL(3, "OK \n");
953 DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb
++);
954 { unsigned long long const bound
= ZSTD_decompressBound(compressedBuffer
, cSize
);
955 if (bound
!= CNBuffSize
/ 2) goto _output_error
; }
956 DISPLAYLEVEL(3, "OK \n");
958 DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb
++);
959 { CHECK_V(r
, ZSTD_decompress(decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
));
960 if (r
!= CNBuffSize
/ 2) goto _output_error
; }
961 DISPLAYLEVEL(3, "OK \n");
963 DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb
++);
964 if (memcmp(decodedBuffer
, CNBuffer
, CNBuffSize
/ 2) != 0) goto _output_error
;
965 DISPLAYLEVEL(3, "OK \n");
967 /* Dictionary and CCtx Duplication tests */
968 { ZSTD_CCtx
* const ctxOrig
= ZSTD_createCCtx();
969 ZSTD_CCtx
* const ctxDuplicated
= ZSTD_createCCtx();
970 ZSTD_DCtx
* const dctx
= ZSTD_createDCtx();
971 static const size_t dictSize
= 551;
972 assert(dctx
!= NULL
); assert(ctxOrig
!= NULL
); assert(ctxDuplicated
!= NULL
);
974 DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb
++);
975 { size_t const copyResult
= ZSTD_copyCCtx(ctxDuplicated
, ctxOrig
, 0);
976 if (!ZSTD_isError(copyResult
)) goto _output_error
; } /* error must be detected */
977 DISPLAYLEVEL(3, "OK \n");
979 DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb
++);
980 CHECK( ZSTD_compressBegin_usingDict(ctxOrig
, CNBuffer
, dictSize
, 2) );
981 CHECK( ZSTD_copyCCtx(ctxDuplicated
, ctxOrig
, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */
982 DISPLAYLEVEL(3, "OK \n");
984 DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb
++);
986 CHECKPLUS(r
, ZSTD_compressEnd(ctxOrig
, compressedBuffer
, compressedBufferSize
,
987 (const char*)CNBuffer
+ dictSize
, CNBuffSize
- dictSize
),
989 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize
, (double)cSize
/CNBuffSize
*100);
991 DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb
++);
992 CHECKPLUS(r
, ZSTD_decompress_usingDict(dctx
,
993 decodedBuffer
, CNBuffSize
,
994 compressedBuffer
, cSize
,
996 if (r
!= CNBuffSize
- dictSize
) goto _output_error
);
997 DISPLAYLEVEL(3, "OK \n");
999 DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb
++);
1000 { size_t const cSizeOrig
= cSize
;
1002 CHECKPLUS(r
, ZSTD_compressEnd(ctxDuplicated
, compressedBuffer
, compressedBufferSize
,
1003 (const char*)CNBuffer
+ dictSize
, CNBuffSize
- dictSize
),
1005 if (cSize
!= cSizeOrig
) goto _output_error
; /* should be identical ==> same size */
1007 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize
, (double)cSize
/CNBuffSize
*100);
1009 DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb
++);
1010 CHECKPLUS(r
, ZSTD_decompress_usingDict(dctx
,
1011 decodedBuffer
, CNBuffSize
,
1012 compressedBuffer
, cSize
,
1013 CNBuffer
, dictSize
),
1014 if (r
!= CNBuffSize
- dictSize
) goto _output_error
);
1015 DISPLAYLEVEL(3, "OK \n");
1017 DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb
++);
1018 { ZSTD_DDict
* const ddict
= ZSTD_createDDict(CNBuffer
, dictSize
);
1019 size_t const r
= ZSTD_decompress_usingDDict(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
, ddict
);
1020 if (r
!= CNBuffSize
- dictSize
) goto _output_error
;
1021 DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict
));
1022 ZSTD_freeDDict(ddict
);
1025 DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb
++);
1026 { size_t const ddictBufferSize
= ZSTD_estimateDDictSize(dictSize
, ZSTD_dlm_byCopy
);
1027 void* ddictBuffer
= malloc(ddictBufferSize
);
1028 if (ddictBuffer
== NULL
) goto _output_error
;
1029 { const ZSTD_DDict
* const ddict
= ZSTD_initStaticDDict(ddictBuffer
, ddictBufferSize
, CNBuffer
, dictSize
, ZSTD_dlm_byCopy
, ZSTD_dct_auto
);
1030 size_t const r
= ZSTD_decompress_usingDDict(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
, ddict
);
1031 if (r
!= CNBuffSize
- dictSize
) goto _output_error
;
1034 DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize
);
1037 DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb
++);
1038 { size_t const testSize
= CNBuffSize
/ 3;
1039 { ZSTD_parameters p
= ZSTD_getParams(2, testSize
, dictSize
);
1040 p
.fParams
.contentSizeFlag
= 1;
1041 CHECK( ZSTD_compressBegin_advanced(ctxOrig
, CNBuffer
, dictSize
, p
, testSize
-1) );
1043 CHECK( ZSTD_copyCCtx(ctxDuplicated
, ctxOrig
, testSize
) );
1045 CHECKPLUS(r
, ZSTD_compressEnd(ctxDuplicated
, compressedBuffer
, ZSTD_compressBound(testSize
),
1046 (const char*)CNBuffer
+ dictSize
, testSize
),
1048 { ZSTD_frameHeader zfh
;
1049 if (ZSTD_getFrameHeader(&zfh
, compressedBuffer
, cSize
)) goto _output_error
;
1050 if ((zfh
.frameContentSize
!= testSize
) && (zfh
.frameContentSize
!= 0)) goto _output_error
;
1052 DISPLAYLEVEL(3, "OK \n");
1054 ZSTD_freeCCtx(ctxOrig
);
1055 ZSTD_freeCCtx(ctxDuplicated
);
1056 ZSTD_freeDCtx(dctx
);
1059 /* Dictionary and dictBuilder tests */
1060 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
1061 size_t const dictBufferCapacity
= 16 KB
;
1062 void* dictBuffer
= malloc(dictBufferCapacity
);
1063 size_t const totalSampleSize
= 1 MB
;
1064 size_t const sampleUnitSize
= 8 KB
;
1065 U32
const nbSamples
= (U32
)(totalSampleSize
/ sampleUnitSize
);
1066 size_t* const samplesSizes
= (size_t*) malloc(nbSamples
* sizeof(size_t));
1070 if (dictBuffer
==NULL
|| samplesSizes
==NULL
) {
1076 DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb
++);
1077 assert(compressedBufferSize
>= totalSampleSize
);
1078 { U32 u
; for (u
=0; u
<totalSampleSize
; u
++) ((BYTE
*)decodedBuffer
)[u
] = (BYTE
)u
; }
1079 { U32 u
; for (u
=0; u
<nbSamples
; u
++) samplesSizes
[u
] = sampleUnitSize
; }
1080 { size_t const sDictSize
= ZDICT_trainFromBuffer(dictBuffer
, dictBufferCapacity
,
1081 decodedBuffer
, samplesSizes
, nbSamples
);
1082 if (ZDICT_isError(sDictSize
)) goto _output_error
;
1083 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize
);
1086 DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb
++);
1087 { U32 u
; for (u
=0; u
<nbSamples
; u
++) samplesSizes
[u
] = sampleUnitSize
; }
1088 dictSize
= ZDICT_trainFromBuffer(dictBuffer
, dictBufferCapacity
,
1089 CNBuffer
, samplesSizes
, nbSamples
);
1090 if (ZDICT_isError(dictSize
)) goto _output_error
;
1091 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize
);
1093 DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb
++);
1094 { U32 u
; for (u
=0; u
<nbSamples
; u
++) samplesSizes
[u
] = sampleUnitSize
; }
1095 { ZDICT_cover_params_t coverParams
;
1096 memset(&coverParams
, 0, sizeof(coverParams
));
1097 coverParams
.steps
= 8;
1098 coverParams
.nbThreads
= 4;
1099 dictSize
= ZDICT_optimizeTrainFromBuffer_cover(
1100 dictBuffer
, dictBufferCapacity
,
1101 CNBuffer
, samplesSizes
, nbSamples
/8, /* less samples for faster tests */
1103 if (ZDICT_isError(dictSize
)) goto _output_error
;
1105 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize
);
1107 DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb
++);
1108 { U32 u
; for (u
=0; u
<nbSamples
; u
++) samplesSizes
[u
] = sampleUnitSize
; }
1109 { ZDICT_fastCover_params_t fastCoverParams
;
1110 memset(&fastCoverParams
, 0, sizeof(fastCoverParams
));
1111 fastCoverParams
.steps
= 8;
1112 fastCoverParams
.nbThreads
= 4;
1113 dictSize
= ZDICT_optimizeTrainFromBuffer_fastCover(
1114 dictBuffer
, dictBufferCapacity
,
1115 CNBuffer
, samplesSizes
, nbSamples
,
1117 if (ZDICT_isError(dictSize
)) goto _output_error
;
1119 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize
);
1121 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb
++);
1122 dictID
= ZDICT_getDictID(dictBuffer
, dictSize
);
1123 if (dictID
==0) goto _output_error
;
1124 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID
);
1126 DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb
++);
1127 cSize
= ZSTD_compress_usingDict(cctx
, compressedBuffer
, compressedBufferSize
,
1128 CNBuffer
, CNBuffSize
,
1129 dictBuffer
, dictSize
, 4);
1130 if (ZSTD_isError(cSize
)) goto _output_error
;
1131 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize
, (double)cSize
/CNBuffSize
*100);
1133 DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb
++);
1134 { U32
const did
= ZSTD_getDictID_fromDict(dictBuffer
, dictSize
);
1135 if (did
!= dictID
) goto _output_error
; /* non-conformant (content-only) dictionary */
1137 DISPLAYLEVEL(3, "OK \n");
1139 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb
++);
1140 { U32
const did
= ZSTD_getDictID_fromFrame(compressedBuffer
, cSize
);
1141 if (did
!= dictID
) goto _output_error
; /* non-conformant (content-only) dictionary */
1143 DISPLAYLEVEL(3, "OK \n");
1145 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb
++);
1146 { ZSTD_DCtx
* const dctx
= ZSTD_createDCtx(); assert(dctx
!= NULL
);
1147 CHECKPLUS(r
, ZSTD_decompress_usingDict(dctx
,
1148 decodedBuffer
, CNBuffSize
,
1149 compressedBuffer
, cSize
,
1150 dictBuffer
, dictSize
),
1151 if (r
!= CNBuffSize
) goto _output_error
);
1152 ZSTD_freeDCtx(dctx
);
1154 DISPLAYLEVEL(3, "OK \n");
1156 DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb
++);
1157 { ZSTD_compressionParameters
const cParams
= ZSTD_getCParams(1, CNBuffSize
, dictSize
);
1158 size_t const estimatedSize
= ZSTD_estimateCDictSize_advanced(dictSize
, cParams
, ZSTD_dlm_byRef
);
1159 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize
);
1162 DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb
++);
1163 { ZSTD_compressionParameters
const cParams
= ZSTD_getCParams(1, CNBuffSize
, dictSize
);
1164 ZSTD_CDict
* const cdict
= ZSTD_createCDict_advanced(dictBuffer
, dictSize
,
1165 ZSTD_dlm_byRef
, ZSTD_dct_auto
,
1166 cParams
, ZSTD_defaultCMem
);
1167 DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict
));
1168 cSize
= ZSTD_compress_usingCDict(cctx
, compressedBuffer
, compressedBufferSize
,
1169 CNBuffer
, CNBuffSize
, cdict
);
1170 ZSTD_freeCDict(cdict
);
1171 if (ZSTD_isError(cSize
)) goto _output_error
;
1173 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize
, (double)cSize
/CNBuffSize
*100);
1175 DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb
++);
1176 { U32
const did
= ZSTD_getDictID_fromFrame(compressedBuffer
, cSize
);
1177 if (did
!= dictID
) goto _output_error
; /* non-conformant (content-only) dictionary */
1179 DISPLAYLEVEL(3, "OK \n");
1181 DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb
++);
1182 { ZSTD_DCtx
* const dctx
= ZSTD_createDCtx(); assert(dctx
!= NULL
);
1183 CHECKPLUS(r
, ZSTD_decompress_usingDict(dctx
,
1184 decodedBuffer
, CNBuffSize
,
1185 compressedBuffer
, cSize
,
1186 dictBuffer
, dictSize
),
1187 if (r
!= CNBuffSize
) goto _output_error
);
1188 ZSTD_freeDCtx(dctx
);
1190 DISPLAYLEVEL(3, "OK \n");
1192 DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb
++);
1193 { int const maxLevel
= ZSTD_maxCLevel();
1195 for (level
= 1; level
<= maxLevel
; ++level
) {
1196 ZSTD_compressionParameters
const cParams
= ZSTD_getCParams(level
, CNBuffSize
, dictSize
);
1197 size_t const cdictSize
= ZSTD_estimateCDictSize_advanced(dictSize
, cParams
, ZSTD_dlm_byCopy
);
1198 void* const cdictBuffer
= malloc(cdictSize
);
1199 if (cdictBuffer
==NULL
) goto _output_error
;
1200 { const ZSTD_CDict
* const cdict
= ZSTD_initStaticCDict(
1201 cdictBuffer
, cdictSize
,
1202 dictBuffer
, dictSize
,
1203 ZSTD_dlm_byCopy
, ZSTD_dct_auto
,
1205 if (cdict
== NULL
) {
1206 DISPLAY("ZSTD_initStaticCDict failed ");
1209 cSize
= ZSTD_compress_usingCDict(cctx
,
1210 compressedBuffer
, compressedBufferSize
,
1211 CNBuffer
, MIN(10 KB
, CNBuffSize
), cdict
);
1212 if (ZSTD_isError(cSize
)) {
1213 DISPLAY("ZSTD_compress_usingCDict failed ");
1218 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize
, (double)cSize
/CNBuffSize
*100);
1220 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb
++);
1221 { ZSTD_frameParameters
const fParams
= { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ };
1222 ZSTD_compressionParameters
const cParams
= ZSTD_getCParams(1, CNBuffSize
, dictSize
);
1223 ZSTD_CDict
* const cdict
= ZSTD_createCDict_advanced(dictBuffer
, dictSize
, ZSTD_dlm_byRef
, ZSTD_dct_auto
, cParams
, ZSTD_defaultCMem
);
1224 cSize
= ZSTD_compress_usingCDict_advanced(cctx
, compressedBuffer
, compressedBufferSize
,
1225 CNBuffer
, CNBuffSize
, cdict
, fParams
);
1226 ZSTD_freeCDict(cdict
);
1227 if (ZSTD_isError(cSize
)) goto _output_error
;
1229 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize
, (double)cSize
/CNBuffSize
*100);
1231 DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb
++);
1232 { U64
const contentSize
= ZSTD_getFrameContentSize(compressedBuffer
, cSize
);
1233 if (contentSize
!= ZSTD_CONTENTSIZE_UNKNOWN
) goto _output_error
;
1235 DISPLAYLEVEL(3, "OK (unknown)\n");
1237 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb
++);
1238 { ZSTD_DCtx
* const dctx
= ZSTD_createDCtx(); assert(dctx
!= NULL
);
1239 CHECKPLUS(r
, ZSTD_decompress_usingDict(dctx
,
1240 decodedBuffer
, CNBuffSize
,
1241 compressedBuffer
, cSize
,
1242 dictBuffer
, dictSize
),
1243 if (r
!= CNBuffSize
) goto _output_error
);
1244 ZSTD_freeDCtx(dctx
);
1246 DISPLAYLEVEL(3, "OK \n");
1248 DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb
++);
1249 { ZSTD_parameters p
= ZSTD_getParams(3, CNBuffSize
, dictSize
);
1250 p
.fParams
.noDictIDFlag
= 1;
1251 cSize
= ZSTD_compress_advanced(cctx
, compressedBuffer
, compressedBufferSize
,
1252 CNBuffer
, CNBuffSize
,
1253 dictBuffer
, dictSize
, p
);
1254 if (ZSTD_isError(cSize
)) goto _output_error
;
1256 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize
, (double)cSize
/CNBuffSize
*100);
1258 DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb
++);
1259 { ZSTD_DCtx
* const dctx
= ZSTD_createDCtx(); assert(dctx
!= NULL
);
1260 CHECKPLUS(r
, ZSTD_decompress_usingDict(dctx
,
1261 decodedBuffer
, CNBuffSize
,
1262 compressedBuffer
, cSize
,
1263 dictBuffer
, dictSize
),
1264 if (r
!= CNBuffSize
) goto _output_error
);
1265 ZSTD_freeDCtx(dctx
);
1267 DISPLAYLEVEL(3, "OK \n");
1269 DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb
++);
1270 { ZSTD_DCtx
* const dctx
= ZSTD_createDCtx();
1271 assert(dctx
!= NULL
);
1272 { const size_t ret
= ZSTD_decompress_usingDict(
1273 dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
,
1274 "\x37\xa4\x30\xec\x11\x22\x33\x44", 8);
1275 if (ZSTD_getErrorCode(ret
) != ZSTD_error_dictionary_corrupted
)
1278 ZSTD_freeDCtx(dctx
);
1280 DISPLAYLEVEL(3, "OK \n");
1282 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb
++);
1283 { ZSTD_compressionParameters
const cParams
= ZSTD_getCParams(1, CNBuffSize
, dictSize
);
1284 ZSTD_CDict
* const cdict
= ZSTD_createCDict_advanced(dictBuffer
, dictSize
, ZSTD_dlm_byRef
, ZSTD_dct_fullDict
, cParams
, ZSTD_defaultCMem
);
1285 if (cdict
==NULL
) goto _output_error
;
1286 ZSTD_freeCDict(cdict
);
1288 DISPLAYLEVEL(3, "OK \n");
1290 DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb
++);
1291 { ZSTD_compressionParameters
const cParams
= ZSTD_getCParams(1, CNBuffSize
, dictSize
);
1292 ZSTD_CDict
* const cdict
= ZSTD_createCDict_advanced((const char*)dictBuffer
+1, dictSize
-1, ZSTD_dlm_byRef
, ZSTD_dct_fullDict
, cParams
, ZSTD_defaultCMem
);
1293 if (cdict
!=NULL
) goto _output_error
;
1294 ZSTD_freeCDict(cdict
);
1296 DISPLAYLEVEL(3, "OK \n");
1298 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_auto should fail : ", testNb
++);
1301 MEM_writeLE32((char*)dictBuffer
+2, ZSTD_MAGIC_DICTIONARY
);
1302 /* Either operation is allowed to fail, but one must fail. */
1303 ret
= ZSTD_CCtx_loadDictionary_advanced(
1304 cctx
, (const char*)dictBuffer
+2, dictSize
-2, ZSTD_dlm_byRef
, ZSTD_dct_auto
);
1305 if (!ZSTD_isError(ret
)) {
1306 ret
= ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100));
1307 if (!ZSTD_isError(ret
)) goto _output_error
;
1310 DISPLAYLEVEL(3, "OK \n");
1312 DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_rawContent should pass : ", testNb
++);
1315 MEM_writeLE32((char*)dictBuffer
+2, ZSTD_MAGIC_DICTIONARY
);
1316 ret
= ZSTD_CCtx_loadDictionary_advanced(
1317 cctx
, (const char*)dictBuffer
+2, dictSize
-2, ZSTD_dlm_byRef
, ZSTD_dct_rawContent
);
1318 if (ZSTD_isError(ret
)) goto _output_error
;
1319 ret
= ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100));
1320 if (ZSTD_isError(ret
)) goto _output_error
;
1322 DISPLAYLEVEL(3, "OK \n");
1324 DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb
++);
1325 { ZSTD_CDict
* const cdict
= ZSTD_createCDict(CNBuffer
, dictSize
, 1);
1326 CHECK_Z( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_compressionLevel
, 1) );
1327 CHECK_Z( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_hashLog
, 12 ));
1328 CHECK_Z( ZSTD_CCtx_refCDict(cctx
, cdict
) );
1329 CHECK_Z( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_compressionLevel
, 1) );
1330 CHECK_Z( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_hashLog
, 12 ));
1331 ZSTD_freeCDict(cdict
);
1333 DISPLAYLEVEL(3, "OK \n");
1335 DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb
++);
1337 size_t size1
, size2
;
1338 ZSTD_CCtx_reset(cctx
, ZSTD_reset_session_and_parameters
);
1339 CHECK_Z( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_compressionLevel
, 7) );
1340 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx
, CNBuffer
, MIN(CNBuffSize
, 10 KB
)) );
1341 size1
= ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100 KB
));
1342 if (ZSTD_isError(size1
)) goto _output_error
;
1344 ZSTD_CCtx_reset(cctx
, ZSTD_reset_session_and_parameters
);
1345 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx
, CNBuffer
, MIN(CNBuffSize
, 10 KB
)) );
1346 CHECK_Z( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_compressionLevel
, 7) );
1347 size2
= ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100 KB
));
1348 if (ZSTD_isError(size2
)) goto _output_error
;
1350 if (size1
!= size2
) goto _output_error
;
1352 DISPLAYLEVEL(3, "OK \n");
1354 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb
++);
1356 CHECK_Z( ZSTD_CCtx_refPrefix(cctx
, (const char*)dictBuffer
, dictSize
) );
1357 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx
, (const char*)dictBuffer
, dictSize
) );
1358 CHECK_Z( ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100)) );
1360 DISPLAYLEVEL(3, "OK \n");
1362 DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb
++);
1364 ZSTD_CDict
* const cdict
= ZSTD_createCDict(dictBuffer
, dictSize
, 1);
1365 CHECK_Z( ZSTD_CCtx_refCDict(cctx
, cdict
) );
1366 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx
, (const char*)dictBuffer
, dictSize
) );
1367 CHECK_Z( ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100)) );
1368 ZSTD_freeCDict(cdict
);
1370 DISPLAYLEVEL(3, "OK \n");
1372 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb
++);
1374 ZSTD_CDict
* const cdict
= ZSTD_createCDict(dictBuffer
, dictSize
, 1);
1375 CHECK_Z( ZSTD_CCtx_refPrefix(cctx
, (const char*)dictBuffer
, dictSize
) );
1376 CHECK_Z( ZSTD_CCtx_refCDict(cctx
, cdict
) );
1377 CHECK_Z( ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100)) );
1378 ZSTD_freeCDict(cdict
);
1380 DISPLAYLEVEL(3, "OK \n");
1382 DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb
++);
1384 ZSTD_CDict
* const cdict
= ZSTD_createCDict(dictBuffer
, dictSize
, 1);
1385 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx
, (const char*)dictBuffer
, dictSize
) );
1386 CHECK_Z( ZSTD_CCtx_refCDict(cctx
, cdict
) );
1387 CHECK_Z( ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100)) );
1388 ZSTD_freeCDict(cdict
);
1390 DISPLAYLEVEL(3, "OK \n");
1392 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb
++);
1394 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx
, (const char*)dictBuffer
, dictSize
) );
1395 CHECK_Z( ZSTD_CCtx_refPrefix(cctx
, (const char*)dictBuffer
, dictSize
) );
1396 CHECK_Z( ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100)) );
1398 DISPLAYLEVEL(3, "OK \n");
1400 DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb
++);
1402 ZSTD_CDict
* const cdict
= ZSTD_createCDict(dictBuffer
, dictSize
, 1);
1403 CHECK_Z( ZSTD_CCtx_refCDict(cctx
, cdict
) );
1404 CHECK_Z( ZSTD_CCtx_refPrefix(cctx
, (const char*)dictBuffer
, dictSize
) );
1405 CHECK_Z( ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100)) );
1406 ZSTD_freeCDict(cdict
);
1408 DISPLAYLEVEL(3, "OK \n");
1410 DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb
++);
1412 size_t size1
, size2
;
1413 ZSTD_CCtx_reset(cctx
, ZSTD_reset_session_and_parameters
);
1414 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx
, CNBuffer
, MIN(CNBuffSize
, 10 KB
)) );
1415 size1
= ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100 KB
));
1416 if (ZSTD_isError(size1
)) goto _output_error
;
1418 ZSTD_CCtx_reset(cctx
, ZSTD_reset_session_only
);
1419 size2
= ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100 KB
));
1420 if (ZSTD_isError(size2
)) goto _output_error
;
1422 if (size1
!= size2
) goto _output_error
;
1424 DISPLAYLEVEL(3, "OK \n");
1426 DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb
++);
1428 size_t size1
, size2
;
1429 ZSTD_CCtx_reset(cctx
, ZSTD_reset_session_and_parameters
);
1430 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx
, CNBuffer
, MIN(CNBuffSize
, 10 KB
)) );
1431 size1
= ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100 KB
));
1432 if (ZSTD_isError(size1
)) goto _output_error
;
1434 ZSTD_CCtx_reset(cctx
, ZSTD_reset_session_and_parameters
);
1435 size2
= ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100 KB
));
1436 if (ZSTD_isError(size2
)) goto _output_error
;
1438 if (size1
== size2
) goto _output_error
;
1440 DISPLAYLEVEL(3, "OK \n");
1442 ZSTD_CCtx_reset(cctx
, ZSTD_reset_session_and_parameters
);
1443 CHECK_Z( ZSTD_CCtx_loadDictionary(cctx
, dictBuffer
, dictSize
) );
1444 cSize
= ZSTD_compress2(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, MIN(CNBuffSize
, 100 KB
));
1446 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb
++);
1448 ZSTD_DCtx
* dctx
= ZSTD_createDCtx();
1450 /* We should fail to decompress without a dictionary. */
1451 ZSTD_DCtx_reset(dctx
, ZSTD_reset_session_and_parameters
);
1452 ret
= ZSTD_decompressDCtx(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
);
1453 if (!ZSTD_isError(ret
)) goto _output_error
;
1454 /* We should succeed to decompress with the dictionary. */
1455 ZSTD_DCtx_reset(dctx
, ZSTD_reset_session_and_parameters
);
1456 CHECK_Z( ZSTD_DCtx_loadDictionary(dctx
, dictBuffer
, dictSize
) );
1457 CHECK_Z( ZSTD_decompressDCtx(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
) );
1458 /* The dictionary should presist across calls. */
1459 CHECK_Z( ZSTD_decompressDCtx(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
) );
1460 /* When we reset the context the dictionary is cleared. */
1461 ZSTD_DCtx_reset(dctx
, ZSTD_reset_session_and_parameters
);
1462 ret
= ZSTD_decompressDCtx(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
);
1463 if (!ZSTD_isError(ret
)) goto _output_error
;
1464 ZSTD_freeDCtx(dctx
);
1466 DISPLAYLEVEL(3, "OK \n");
1468 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb
++);
1470 ZSTD_DCtx
* dctx
= ZSTD_createDCtx();
1471 ZSTD_DDict
* ddict
= ZSTD_createDDict(dictBuffer
, dictSize
);
1473 /* We should succeed to decompress with the ddict. */
1474 ZSTD_DCtx_reset(dctx
, ZSTD_reset_session_and_parameters
);
1475 CHECK_Z( ZSTD_DCtx_refDDict(dctx
, ddict
) );
1476 CHECK_Z( ZSTD_decompressDCtx(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
) );
1477 /* The ddict should presist across calls. */
1478 CHECK_Z( ZSTD_decompressDCtx(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
) );
1479 /* When we reset the context the ddict is cleared. */
1480 ZSTD_DCtx_reset(dctx
, ZSTD_reset_session_and_parameters
);
1481 ret
= ZSTD_decompressDCtx(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
);
1482 if (!ZSTD_isError(ret
)) goto _output_error
;
1483 ZSTD_freeDCtx(dctx
);
1484 ZSTD_freeDDict(ddict
);
1486 DISPLAYLEVEL(3, "OK \n");
1488 DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb
++);
1490 ZSTD_DCtx
* dctx
= ZSTD_createDCtx();
1492 /* We should succeed to decompress with the prefix. */
1493 ZSTD_DCtx_reset(dctx
, ZSTD_reset_session_and_parameters
);
1494 CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx
, dictBuffer
, dictSize
, ZSTD_dct_auto
) );
1495 CHECK_Z( ZSTD_decompressDCtx(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
) );
1496 /* The prefix should be cleared after the first compression. */
1497 ret
= ZSTD_decompressDCtx(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
);
1498 if (!ZSTD_isError(ret
)) goto _output_error
;
1499 ZSTD_freeDCtx(dctx
);
1501 DISPLAYLEVEL(3, "OK \n");
1503 DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb
++);
1504 { U32 u
; for (u
=0; u
<nbSamples
; u
++) samplesSizes
[u
] = sampleUnitSize
; }
1505 dictSize
= ZDICT_trainFromBuffer(dictBuffer
, dictSize
,
1506 CNBuffer
, samplesSizes
, nbSamples
);
1507 if (ZDICT_isError(dictSize
)) goto _output_error
;
1508 /* Set all the repcodes to non-default */
1510 BYTE
* dictPtr
= (BYTE
*)dictBuffer
;
1511 BYTE
* dictLimit
= dictPtr
+ dictSize
- 12;
1512 /* Find the repcodes */
1513 while (dictPtr
< dictLimit
&&
1514 (MEM_readLE32(dictPtr
) != 1 || MEM_readLE32(dictPtr
+ 4) != 4 ||
1515 MEM_readLE32(dictPtr
+ 8) != 8)) {
1518 if (dictPtr
>= dictLimit
) goto _output_error
;
1519 MEM_writeLE32(dictPtr
+ 0, 10);
1520 MEM_writeLE32(dictPtr
+ 4, 10);
1521 MEM_writeLE32(dictPtr
+ 8, 10);
1522 /* Set the last 8 bytes to 'x' */
1523 memset((BYTE
*)dictBuffer
+ dictSize
- 8, 'x', 8);
1525 /* The optimal parser checks all the repcodes.
1526 * Make sure at least one is a match >= targetLength so that it is
1527 * immediately chosen. This will make sure that the compressor and
1528 * decompressor agree on at least one of the repcodes.
1532 ZSTD_DCtx
* const dctx
= ZSTD_createDCtx();
1533 ZSTD_compressionParameters
const cParams
= ZSTD_getCParams(19, CNBuffSize
, dictSize
);
1534 ZSTD_CDict
* const cdict
= ZSTD_createCDict_advanced(dictBuffer
, dictSize
,
1535 ZSTD_dlm_byRef
, ZSTD_dct_auto
,
1536 cParams
, ZSTD_defaultCMem
);
1537 assert(dctx
!= NULL
); assert(cdict
!= NULL
);
1538 memset(data
, 'x', sizeof(data
));
1539 cSize
= ZSTD_compress_usingCDict(cctx
, compressedBuffer
, compressedBufferSize
,
1540 data
, sizeof(data
), cdict
);
1541 ZSTD_freeCDict(cdict
);
1542 if (ZSTD_isError(cSize
)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize
)); goto _output_error
; }
1543 dSize
= ZSTD_decompress_usingDict(dctx
, decodedBuffer
, sizeof(data
), compressedBuffer
, cSize
, dictBuffer
, dictSize
);
1544 if (ZSTD_isError(dSize
)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize
)); goto _output_error
; }
1545 if (memcmp(data
, decodedBuffer
, sizeof(data
))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error
; }
1546 ZSTD_freeDCtx(dctx
);
1548 DISPLAYLEVEL(3, "OK \n");
1550 ZSTD_freeCCtx(cctx
);
1555 /* COVER dictionary builder tests */
1556 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
1557 size_t dictSize
= 16 KB
;
1558 size_t optDictSize
= dictSize
;
1559 void* dictBuffer
= malloc(dictSize
);
1560 size_t const totalSampleSize
= 1 MB
;
1561 size_t const sampleUnitSize
= 8 KB
;
1562 U32
const nbSamples
= (U32
)(totalSampleSize
/ sampleUnitSize
);
1563 size_t* const samplesSizes
= (size_t*) malloc(nbSamples
* sizeof(size_t));
1564 ZDICT_cover_params_t params
;
1567 if (dictBuffer
==NULL
|| samplesSizes
==NULL
) {
1573 DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb
++);
1574 { U32 u
; for (u
=0; u
<nbSamples
; u
++) samplesSizes
[u
] = sampleUnitSize
; }
1575 memset(¶ms
, 0, sizeof(params
));
1576 params
.d
= 1 + (FUZ_rand(&seed
) % 16);
1577 params
.k
= params
.d
+ (FUZ_rand(&seed
) % 256);
1578 dictSize
= ZDICT_trainFromBuffer_cover(dictBuffer
, dictSize
,
1579 CNBuffer
, samplesSizes
, nbSamples
,
1581 if (ZDICT_isError(dictSize
)) goto _output_error
;
1582 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize
);
1584 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb
++);
1585 dictID
= ZDICT_getDictID(dictBuffer
, dictSize
);
1586 if (dictID
==0) goto _output_error
;
1587 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID
);
1589 DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb
++);
1590 memset(¶ms
, 0, sizeof(params
));
1592 optDictSize
= ZDICT_optimizeTrainFromBuffer_cover(dictBuffer
, optDictSize
,
1593 CNBuffer
, samplesSizes
,
1594 nbSamples
/ 4, ¶ms
);
1595 if (ZDICT_isError(optDictSize
)) goto _output_error
;
1596 DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize
);
1598 DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb
++);
1599 dictID
= ZDICT_getDictID(dictBuffer
, optDictSize
);
1600 if (dictID
==0) goto _output_error
;
1601 DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID
);
1603 ZSTD_freeCCtx(cctx
);
1608 /* Decompression defense tests */
1609 DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb
++);
1610 { size_t const r
= ZSTD_decompress(decodedBuffer
, CNBuffSize
, CNBuffer
, 3); /* too small input */
1611 if (!ZSTD_isError(r
)) goto _output_error
;
1612 if (ZSTD_getErrorCode(r
) != ZSTD_error_srcSize_wrong
) goto _output_error
; }
1613 DISPLAYLEVEL(3, "OK \n");
1615 DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb
++);
1616 ((char*)(CNBuffer
))[0] = 1;
1617 { size_t const r
= ZSTD_decompress(decodedBuffer
, CNBuffSize
, CNBuffer
, 4);
1618 if (!ZSTD_isError(r
)) goto _output_error
; }
1619 DISPLAYLEVEL(3, "OK \n");
1621 /* content size verification test */
1622 DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb
++);
1623 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
1624 size_t const srcSize
= 5000;
1625 size_t const wrongSrcSize
= (srcSize
+ 1000);
1626 ZSTD_parameters params
= ZSTD_getParams(1, wrongSrcSize
, 0);
1627 params
.fParams
.contentSizeFlag
= 1;
1628 CHECK( ZSTD_compressBegin_advanced(cctx
, NULL
, 0, params
, wrongSrcSize
) );
1629 { size_t const result
= ZSTD_compressEnd(cctx
, decodedBuffer
, CNBuffSize
, CNBuffer
, srcSize
);
1630 if (!ZSTD_isError(result
)) goto _output_error
;
1631 if (ZSTD_getErrorCode(result
) != ZSTD_error_srcSize_wrong
) goto _output_error
;
1632 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result
));
1634 ZSTD_freeCCtx(cctx
);
1637 /* negative compression level test : ensure simple API and advanced API produce same result */
1638 DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb
++);
1639 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
1640 size_t const srcSize
= CNBuffSize
/ 5;
1641 int const compressionLevel
= -1;
1643 assert(cctx
!= NULL
);
1644 { ZSTD_parameters
const params
= ZSTD_getParams(compressionLevel
, srcSize
, 0);
1645 size_t const cSize_1pass
= ZSTD_compress_advanced(cctx
,
1646 compressedBuffer
, compressedBufferSize
,
1650 if (ZSTD_isError(cSize_1pass
)) goto _output_error
;
1652 CHECK( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_compressionLevel
, compressionLevel
) );
1653 { size_t const compressionResult
= ZSTD_compress2(cctx
,
1654 compressedBuffer
, compressedBufferSize
,
1656 DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass
, compressionResult
);
1657 if (ZSTD_isError(compressionResult
)) goto _output_error
;
1658 if (compressionResult
!= cSize_1pass
) goto _output_error
;
1660 ZSTD_freeCCtx(cctx
);
1662 DISPLAYLEVEL(3, "OK \n");
1664 /* parameters order test */
1665 { size_t const inputSize
= CNBuffSize
/ 2;
1668 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
1669 DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb
++);
1670 assert(cctx
!= NULL
);
1671 CHECK( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_compressionLevel
, 2) );
1672 CHECK( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_enableLongDistanceMatching
, 1) );
1673 CHECK( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_windowLog
, 18) );
1674 { size_t const compressedSize
= ZSTD_compress2(cctx
,
1675 compressedBuffer
, ZSTD_compressBound(inputSize
),
1676 CNBuffer
, inputSize
);
1677 CHECK(compressedSize
);
1678 cSize
= compressedSize
;
1679 xxh64
= XXH64(compressedBuffer
, compressedSize
, 0);
1681 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize
, (unsigned)cSize
);
1682 ZSTD_freeCCtx(cctx
);
1685 { ZSTD_CCtx
* cctx
= ZSTD_createCCtx();
1686 DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb
++);
1687 CHECK( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_windowLog
, 18) );
1688 CHECK( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_enableLongDistanceMatching
, 1) );
1689 CHECK( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_compressionLevel
, 2) );
1690 { size_t const result
= ZSTD_compress2(cctx
,
1691 compressedBuffer
, ZSTD_compressBound(inputSize
),
1692 CNBuffer
, inputSize
);
1694 if (result
!= cSize
) goto _output_error
; /* must result in same compressed result, hence same size */
1695 if (XXH64(compressedBuffer
, result
, 0) != xxh64
) goto _output_error
; /* must result in exactly same content, hence same hash */
1696 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize
, (unsigned)result
);
1698 ZSTD_freeCCtx(cctx
);
1702 /* advanced parameters for decompression */
1703 { ZSTD_DCtx
* const dctx
= ZSTD_createDCtx();
1704 assert(dctx
!= NULL
);
1706 DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb
++);
1707 { ZSTD_bounds
const bounds
= ZSTD_dParam_getBounds(ZSTD_d_windowLogMax
);
1708 CHECK(bounds
.error
);
1710 DISPLAYLEVEL(3, "OK \n");
1712 DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb
++);
1713 { size_t const sr
= ZSTD_DCtx_setParameter(dctx
, (ZSTD_dParameter
)999999, 0);
1714 if (!ZSTD_isError(sr
)) goto _output_error
;
1716 { ZSTD_bounds
const bounds
= ZSTD_dParam_getBounds((ZSTD_dParameter
)999998);
1717 if (!ZSTD_isError(bounds
.error
)) goto _output_error
;
1719 DISPLAYLEVEL(3, "OK \n");
1721 DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb
++);
1722 { size_t const sr
= ZSTD_DCtx_setParameter(dctx
, ZSTD_d_windowLogMax
, 9999);
1723 if (!ZSTD_isError(sr
)) goto _output_error
;
1725 { size_t const sr
= ZSTD_DCtx_setParameter(dctx
, ZSTD_d_format
, (ZSTD_format_e
)888);
1726 if (!ZSTD_isError(sr
)) goto _output_error
;
1728 DISPLAYLEVEL(3, "OK \n");
1730 ZSTD_freeDCtx(dctx
);
1734 /* custom formats tests */
1735 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
1736 ZSTD_DCtx
* const dctx
= ZSTD_createDCtx();
1737 size_t const inputSize
= CNBuffSize
/ 2; /* won't cause pb with small dict size */
1738 assert(dctx
!= NULL
); assert(cctx
!= NULL
);
1740 /* basic block compression */
1741 DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb
++);
1742 CHECK( ZSTD_CCtx_setParameter(cctx
, ZSTD_c_format
, ZSTD_f_zstd1_magicless
) );
1743 { ZSTD_inBuffer in
= { CNBuffer
, inputSize
, 0 };
1744 ZSTD_outBuffer out
= { compressedBuffer
, ZSTD_compressBound(inputSize
), 0 };
1745 size_t const result
= ZSTD_compressStream2(cctx
, &out
, &in
, ZSTD_e_end
);
1746 if (result
!= 0) goto _output_error
;
1747 if (in
.pos
!= in
.size
) goto _output_error
;
1750 DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize
, (unsigned)cSize
);
1752 DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb
++);
1753 { size_t const decodeResult
= ZSTD_decompressDCtx(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
);
1754 if (ZSTD_getErrorCode(decodeResult
) != ZSTD_error_prefix_unknown
) goto _output_error
;
1755 DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult
));
1758 DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb
++);
1759 ZSTD_DCtx_reset(dctx
, ZSTD_reset_session_and_parameters
);
1760 CHECK( ZSTD_DCtx_setParameter(dctx
, ZSTD_d_format
, ZSTD_f_zstd1_magicless
) );
1761 { ZSTD_frameHeader zfh
;
1762 size_t const zfhrt
= ZSTD_getFrameHeader_advanced(&zfh
, compressedBuffer
, cSize
, ZSTD_f_zstd1_magicless
);
1763 if (zfhrt
!= 0) goto _output_error
;
1766 { size_t const result
= ZSTD_decompressDCtx(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
);
1767 if (result
!= inputSize
) goto _output_error
;
1768 DISPLAYLEVEL(3, "one-shot OK, ");
1771 { ZSTD_inBuffer in
= { compressedBuffer
, cSize
, 0 };
1772 ZSTD_outBuffer out
= { decodedBuffer
, CNBuffSize
, 0 };
1773 size_t const result
= ZSTD_decompressStream(dctx
, &out
, &in
);
1774 if (result
!= 0) goto _output_error
;
1775 if (in
.pos
!= in
.size
) goto _output_error
;
1776 if (out
.pos
!= inputSize
) goto _output_error
;
1777 DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out
.pos
);
1780 ZSTD_freeCCtx(cctx
);
1781 ZSTD_freeDCtx(dctx
);
1784 /* block API tests */
1785 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
1786 ZSTD_DCtx
* const dctx
= ZSTD_createDCtx();
1787 static const size_t dictSize
= 65 KB
;
1788 static const size_t blockSize
= 100 KB
; /* won't cause pb with small dict size */
1790 assert(cctx
!= NULL
); assert(dctx
!= NULL
);
1792 /* basic block compression */
1793 DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb
++);
1794 CHECK( ZSTD_compressBegin(cctx
, 5) );
1795 CHECK( ZSTD_getBlockSize(cctx
) >= blockSize
);
1796 cSize
= ZSTD_compressBlock(cctx
, compressedBuffer
, ZSTD_compressBound(blockSize
), CNBuffer
, blockSize
);
1797 if (ZSTD_isError(cSize
)) goto _output_error
;
1798 DISPLAYLEVEL(3, "OK \n");
1800 DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb
++);
1801 CHECK( ZSTD_decompressBegin(dctx
) );
1802 { CHECK_V(r
, ZSTD_decompressBlock(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
) );
1803 if (r
!= blockSize
) goto _output_error
; }
1804 DISPLAYLEVEL(3, "OK \n");
1806 /* very long stream of block compression */
1807 DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb
++);
1808 CHECK( ZSTD_compressBegin(cctx
, -199) ); /* we just want to quickly overflow internal U32 index */
1809 CHECK( ZSTD_getBlockSize(cctx
) >= blockSize
);
1810 { U64
const toCompress
= 5000000000ULL; /* > 4 GB */
1812 while (compressed
< toCompress
) {
1813 size_t const blockCSize
= ZSTD_compressBlock(cctx
, compressedBuffer
, ZSTD_compressBound(blockSize
), CNBuffer
, blockSize
);
1814 assert(blockCSize
!= 0);
1815 if (ZSTD_isError(blockCSize
)) goto _output_error
;
1816 compressed
+= blockCSize
;
1819 DISPLAYLEVEL(3, "OK \n");
1821 /* dictionary block compression */
1822 DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb
++);
1823 CHECK( ZSTD_compressBegin_usingDict(cctx
, CNBuffer
, dictSize
, 5) );
1824 cSize
= ZSTD_compressBlock(cctx
, compressedBuffer
, ZSTD_compressBound(blockSize
), (char*)CNBuffer
+dictSize
, blockSize
);
1825 if (ZSTD_isError(cSize
)) goto _output_error
;
1826 cSize2
= ZSTD_compressBlock(cctx
, (char*)compressedBuffer
+cSize
, ZSTD_compressBound(blockSize
), (char*)CNBuffer
+dictSize
+blockSize
, blockSize
);
1827 if (ZSTD_isError(cSize2
)) goto _output_error
;
1828 memcpy((char*)compressedBuffer
+cSize
, (char*)CNBuffer
+dictSize
+blockSize
, blockSize
); /* fake non-compressed block */
1829 cSize2
= ZSTD_compressBlock(cctx
, (char*)compressedBuffer
+cSize
+blockSize
, ZSTD_compressBound(blockSize
),
1830 (char*)CNBuffer
+dictSize
+2*blockSize
, blockSize
);
1831 if (ZSTD_isError(cSize2
)) goto _output_error
;
1832 DISPLAYLEVEL(3, "OK \n");
1834 DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb
++);
1835 CHECK( ZSTD_decompressBegin_usingDict(dctx
, CNBuffer
, dictSize
) );
1836 { CHECK_V( r
, ZSTD_decompressBlock(dctx
, decodedBuffer
, CNBuffSize
, compressedBuffer
, cSize
) );
1837 if (r
!= blockSize
) goto _output_error
; }
1838 ZSTD_insertBlock(dctx
, (char*)decodedBuffer
+blockSize
, blockSize
); /* insert non-compressed block into dctx history */
1839 { CHECK_V( r
, ZSTD_decompressBlock(dctx
, (char*)decodedBuffer
+2*blockSize
, CNBuffSize
, (char*)compressedBuffer
+cSize
+blockSize
, cSize2
) );
1840 if (r
!= blockSize
) goto _output_error
; }
1841 DISPLAYLEVEL(3, "OK \n");
1843 DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb
++);
1844 { ZSTD_CDict
* const cdict
= ZSTD_createCDict(CNBuffer
, dictSize
, 3);
1845 if (cdict
==NULL
) goto _output_error
;
1846 CHECK( ZSTD_compressBegin_usingCDict(cctx
, cdict
) );
1847 CHECK( ZSTD_compressBlock(cctx
, compressedBuffer
, ZSTD_compressBound(blockSize
), (char*)CNBuffer
+dictSize
, blockSize
) );
1848 ZSTD_freeCDict(cdict
);
1850 DISPLAYLEVEL(3, "OK \n");
1852 ZSTD_freeCCtx(cctx
);
1853 ZSTD_freeDCtx(dctx
);
1857 { size_t sampleSize
= 0;
1858 DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb
++);
1859 RDG_genBuffer(CNBuffer
, sampleSize
, compressibility
, 0., seed
+1);
1860 memset((char*)CNBuffer
+sampleSize
, 'B', 256 KB
- 1);
1861 sampleSize
+= 256 KB
- 1;
1862 RDG_genBuffer((char*)CNBuffer
+sampleSize
, 96 KB
, compressibility
, 0., seed
+2);
1863 sampleSize
+= 96 KB
;
1864 cSize
= ZSTD_compress(compressedBuffer
, ZSTD_compressBound(sampleSize
), CNBuffer
, sampleSize
, 1);
1865 if (ZSTD_isError(cSize
)) goto _output_error
;
1866 { CHECK_V(regenSize
, ZSTD_decompress(decodedBuffer
, sampleSize
, compressedBuffer
, cSize
));
1867 if (regenSize
!=sampleSize
) goto _output_error
; }
1868 DISPLAYLEVEL(3, "OK \n");
1871 /* All zeroes test (test bug #137) */
1872 #define ZEROESLENGTH 100
1873 DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb
++, ZEROESLENGTH
);
1874 memset(CNBuffer
, 0, ZEROESLENGTH
);
1875 { CHECK_V(r
, ZSTD_compress(compressedBuffer
, ZSTD_compressBound(ZEROESLENGTH
), CNBuffer
, ZEROESLENGTH
, 1) );
1877 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize
, (double)cSize
/ZEROESLENGTH
*100);
1879 DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb
++, ZEROESLENGTH
);
1880 { CHECK_V(r
, ZSTD_decompress(decodedBuffer
, ZEROESLENGTH
, compressedBuffer
, cSize
) );
1881 if (r
!= ZEROESLENGTH
) goto _output_error
; }
1882 DISPLAYLEVEL(3, "OK \n");
1884 /* nbSeq limit test */
1885 #define _3BYTESTESTLENGTH 131000
1886 #define NB3BYTESSEQLOG 9
1887 #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG)
1888 #define NB3BYTESSEQMASK (NB3BYTESSEQ-1)
1889 /* creates a buffer full of 3-bytes sequences */
1890 { BYTE _3BytesSeqs
[NB3BYTESSEQ
][3];
1893 /* create batch of 3-bytes sequences */
1895 for (i
=0; i
< NB3BYTESSEQ
; i
++) {
1896 _3BytesSeqs
[i
][0] = (BYTE
)(FUZ_rand(&rSeed
) & 255);
1897 _3BytesSeqs
[i
][1] = (BYTE
)(FUZ_rand(&rSeed
) & 255);
1898 _3BytesSeqs
[i
][2] = (BYTE
)(FUZ_rand(&rSeed
) & 255);
1901 /* randomly fills CNBuffer with prepared 3-bytes sequences */
1903 for (i
=0; i
< _3BYTESTESTLENGTH
; i
+= 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */
1904 U32
const id
= FUZ_rand(&rSeed
) & NB3BYTESSEQMASK
;
1905 ((BYTE
*)CNBuffer
)[i
+0] = _3BytesSeqs
[id
][0];
1906 ((BYTE
*)CNBuffer
)[i
+1] = _3BytesSeqs
[id
][1];
1907 ((BYTE
*)CNBuffer
)[i
+2] = _3BytesSeqs
[id
][2];
1909 DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb
++);
1910 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
1911 size_t const maxNbSeq
= _3BYTESTESTLENGTH
/ 3;
1912 size_t const bound
= ZSTD_compressBound(_3BYTESTESTLENGTH
);
1914 while (nbSeq
<= maxNbSeq
) {
1915 CHECK(ZSTD_compressCCtx(cctx
, compressedBuffer
, bound
, CNBuffer
, nbSeq
* 3, 19));
1916 /* Check every sequence for the first 100, then skip more rapidly. */
1920 nbSeq
+= (nbSeq
>> 2);
1923 ZSTD_freeCCtx(cctx
);
1925 DISPLAYLEVEL(3, "OK \n");
1927 DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb
++);
1928 { CHECK_V(r
, ZSTD_compress(compressedBuffer
, ZSTD_compressBound(_3BYTESTESTLENGTH
),
1929 CNBuffer
, _3BYTESTESTLENGTH
, 19) );
1931 DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize
, (double)cSize
/_3BYTESTESTLENGTH
*100);
1933 DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb
++);
1934 { CHECK_V(r
, ZSTD_decompress(decodedBuffer
, _3BYTESTESTLENGTH
, compressedBuffer
, cSize
) );
1935 if (r
!= _3BYTESTESTLENGTH
) goto _output_error
; }
1936 DISPLAYLEVEL(3, "OK \n");
1939 DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb
++);
1940 RDG_genBuffer(CNBuffer
, CNBuffSize
, 0.0, 0.1, seed
);
1941 { ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
1942 size_t const bound
= ZSTD_compressBound(CNBuffSize
);
1944 while (size
<= CNBuffSize
) {
1945 CHECK(ZSTD_compressCCtx(cctx
, compressedBuffer
, bound
, CNBuffer
, size
, 3));
1946 /* Check every size for the first 100, then skip more rapidly. */
1950 size
+= (size
>> 2);
1953 ZSTD_freeCCtx(cctx
);
1955 DISPLAYLEVEL(3, "OK \n");
1957 DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb
++);
1958 { /* Train a dictionary on low characters */
1959 size_t dictSize
= 16 KB
;
1960 void* const dictBuffer
= malloc(dictSize
);
1961 size_t const totalSampleSize
= 1 MB
;
1962 size_t const sampleUnitSize
= 8 KB
;
1963 U32
const nbSamples
= (U32
)(totalSampleSize
/ sampleUnitSize
);
1964 size_t* const samplesSizes
= (size_t*) malloc(nbSamples
* sizeof(size_t));
1965 if (!dictBuffer
|| !samplesSizes
) goto _output_error
;
1966 { U32 u
; for (u
=0; u
<nbSamples
; u
++) samplesSizes
[u
] = sampleUnitSize
; }
1967 dictSize
= ZDICT_trainFromBuffer(dictBuffer
, dictSize
, CNBuffer
, samplesSizes
, nbSamples
);
1968 if (ZDICT_isError(dictSize
)) goto _output_error
;
1969 /* Reverse the characters to make the dictionary ill suited */
1971 for (u
= 0; u
< CNBuffSize
; ++u
) {
1972 ((BYTE
*)CNBuffer
)[u
] = 255 - ((BYTE
*)CNBuffer
)[u
];
1975 { /* Compress the data */
1976 size_t const inputSize
= 500;
1977 size_t const outputSize
= ZSTD_compressBound(inputSize
);
1978 void* const outputBuffer
= malloc(outputSize
);
1979 ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
1980 if (!outputBuffer
|| !cctx
) goto _output_error
;
1981 CHECK(ZSTD_compress_usingDict(cctx
, outputBuffer
, outputSize
, CNBuffer
, inputSize
, dictBuffer
, dictSize
, 1));
1983 ZSTD_freeCCtx(cctx
);
1989 DISPLAYLEVEL(3, "OK \n");
1992 /* findFrameCompressedSize on skippable frames */
1993 DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb
++);
1994 { const char* frame
= "\x50\x2a\x4d\x18\x05\x0\x0\0abcde";
1995 size_t const frameSrcSize
= 13;
1996 if (ZSTD_findFrameCompressedSize(frame
, frameSrcSize
) != frameSrcSize
) goto _output_error
; }
1997 DISPLAYLEVEL(3, "OK \n");
1999 /* error string tests */
2000 DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb
++);
2001 if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode
)(0-ZSTD_error_no_error
))) != 0) goto _output_error
;
2002 if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error
)) != 0) goto _output_error
;
2003 if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode
)(0-ZSTD_error_GENERIC
))) != 0) goto _output_error
;
2004 if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC
)) != 0) goto _output_error
;
2005 if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC
)) != 0) goto _output_error
;
2006 if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC
)) != 0) goto _output_error
;
2007 DISPLAYLEVEL(3, "OK \n");
2009 DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb
++);
2010 RDG_genBuffer(CNBuffer
, CNBuffSize
, compressibility
, 0., seed
);
2012 size_t const size
= MIN(128 KB
, CNBuffSize
);
2013 ZSTD_CCtx
* const cctx
= ZSTD_createCCtx();
2014 ZSTD_CDict
* const lgCDict
= ZSTD_createCDict(CNBuffer
, size
, 1);
2015 ZSTD_CDict
* const smCDict
= ZSTD_createCDict(CNBuffer
, 1 KB
, 1);
2016 ZSTD_frameHeader lgHeader
;
2017 ZSTD_frameHeader smHeader
;
2019 CHECK_Z(ZSTD_compress_usingCDict(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, size
, lgCDict
));
2020 CHECK_Z(ZSTD_getFrameHeader(&lgHeader
, compressedBuffer
, compressedBufferSize
));
2021 CHECK_Z(ZSTD_compress_usingCDict(cctx
, compressedBuffer
, compressedBufferSize
, CNBuffer
, size
, smCDict
));
2022 CHECK_Z(ZSTD_getFrameHeader(&smHeader
, compressedBuffer
, compressedBufferSize
));
2024 if (lgHeader
.windowSize
!= smHeader
.windowSize
) goto _output_error
;
2026 ZSTD_freeCDict(smCDict
);
2027 ZSTD_freeCDict(lgCDict
);
2028 ZSTD_freeCCtx(cctx
);
2030 DISPLAYLEVEL(3, "OK \n");
2032 DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb
++);
2036 unsigned const tableLog
= 5;
2037 size_t const nbSeq
= 32;
2038 unsigned const maxSymbolValue
= 31;
2041 for (i
= 0; i
< 32; ++i
)
2043 /* Calling FSE_normalizeCount() on a uniform distribution should not
2044 * cause a division by zero.
2046 FSE_normalizeCount(norm
, tableLog
, count
, nbSeq
, maxSymbolValue
);
2048 DISPLAYLEVEL(3, "OK \n");
2052 free(compressedBuffer
);
2053 free(decodedBuffer
);
2058 DISPLAY("Error detected in Unit tests ! \n");
2063 static size_t findDiff(const void* buf1
, const void* buf2
, size_t max
)
2065 const BYTE
* b1
= (const BYTE
*)buf1
;
2066 const BYTE
* b2
= (const BYTE
*)buf2
;
2068 for (u
=0; u
<max
; u
++) {
2069 if (b1
[u
] != b2
[u
]) break;
2075 static ZSTD_parameters
FUZ_makeParams(ZSTD_compressionParameters cParams
, ZSTD_frameParameters fParams
)
2077 ZSTD_parameters params
;
2078 params
.cParams
= cParams
;
2079 params
.fParams
= fParams
;
2083 static size_t FUZ_rLogLength(U32
* seed
, U32 logLength
)
2085 size_t const lengthMask
= ((size_t)1 << logLength
) - 1;
2086 return (lengthMask
+1) + (FUZ_rand(seed
) & lengthMask
);
2089 static size_t FUZ_randomLength(U32
* seed
, U32 maxLog
)
2091 U32
const logLength
= FUZ_rand(seed
) % maxLog
;
2092 return FUZ_rLogLength(seed
, logLength
);
2096 #define CHECK(cond, ...) { \
2098 DISPLAY("Error => "); \
2099 DISPLAY(__VA_ARGS__); \
2100 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
2101 goto _output_error; \
2105 #define CHECK_Z(f) { \
2106 size_t const err = f; \
2107 if (ZSTD_isError(err)) { \
2108 DISPLAY("Error => %s : %s ", \
2109 #f, ZSTD_getErrorName(err)); \
2110 DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \
2111 goto _output_error; \
2115 static int fuzzerTests(U32 seed
, unsigned nbTests
, unsigned startTest
, U32
const maxDurationS
, double compressibility
, int bigTests
)
2117 static const U32 maxSrcLog
= 23;
2118 static const U32 maxSampleLog
= 22;
2119 size_t const srcBufferSize
= (size_t)1<<maxSrcLog
;
2120 size_t const dstBufferSize
= (size_t)1<<maxSampleLog
;
2121 size_t const cBufferSize
= ZSTD_compressBound(dstBufferSize
);
2122 BYTE
* cNoiseBuffer
[5];
2123 BYTE
* const cBuffer
= (BYTE
*) malloc (cBufferSize
);
2124 BYTE
* const dstBuffer
= (BYTE
*) malloc (dstBufferSize
);
2125 BYTE
* const mirrorBuffer
= (BYTE
*) malloc (dstBufferSize
);
2126 ZSTD_CCtx
* const refCtx
= ZSTD_createCCtx();
2127 ZSTD_CCtx
* const ctx
= ZSTD_createCCtx();
2128 ZSTD_DCtx
* const dctx
= ZSTD_createDCtx();
2130 unsigned testNb
= 0;
2131 U32 coreSeed
= seed
;
2132 UTIL_time_t
const startClock
= UTIL_getTime();
2133 U64
const maxClockSpan
= maxDurationS
* SEC_TO_MICRO
;
2134 int const cLevelLimiter
= bigTests
? 3 : 2;
2137 cNoiseBuffer
[0] = (BYTE
*)malloc (srcBufferSize
);
2138 cNoiseBuffer
[1] = (BYTE
*)malloc (srcBufferSize
);
2139 cNoiseBuffer
[2] = (BYTE
*)malloc (srcBufferSize
);
2140 cNoiseBuffer
[3] = (BYTE
*)malloc (srcBufferSize
);
2141 cNoiseBuffer
[4] = (BYTE
*)malloc (srcBufferSize
);
2142 CHECK (!cNoiseBuffer
[0] || !cNoiseBuffer
[1] || !cNoiseBuffer
[2] || !cNoiseBuffer
[3] || !cNoiseBuffer
[4]
2143 || !dstBuffer
|| !mirrorBuffer
|| !cBuffer
|| !refCtx
|| !ctx
|| !dctx
,
2144 "Not enough memory, fuzzer tests cancelled");
2146 /* Create initial samples */
2147 RDG_genBuffer(cNoiseBuffer
[0], srcBufferSize
, 0.00, 0., coreSeed
); /* pure noise */
2148 RDG_genBuffer(cNoiseBuffer
[1], srcBufferSize
, 0.05, 0., coreSeed
); /* barely compressible */
2149 RDG_genBuffer(cNoiseBuffer
[2], srcBufferSize
, compressibility
, 0., coreSeed
);
2150 RDG_genBuffer(cNoiseBuffer
[3], srcBufferSize
, 0.95, 0., coreSeed
); /* highly compressible */
2151 RDG_genBuffer(cNoiseBuffer
[4], srcBufferSize
, 1.00, 0., coreSeed
); /* sparse content */
2153 /* catch up testNb */
2154 for (testNb
=1; testNb
< startTest
; testNb
++) FUZ_rand(&coreSeed
);
2156 /* main test loop */
2157 for ( ; (testNb
<= nbTests
) || (UTIL_clockSpanMicro(startClock
) < maxClockSpan
); testNb
++ ) {
2158 BYTE
* srcBuffer
; /* jumping pointer */
2160 size_t sampleSize
, maxTestSize
, totalTestSize
;
2161 size_t cSize
, totalCSize
, totalGenSize
;
2168 if (nbTests
>= testNb
) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb
, nbTests
); }
2169 else { DISPLAYUPDATE(2, "\r%6u ", testNb
); }
2171 FUZ_rand(&coreSeed
);
2172 { U32
const prime1
= 2654435761U; lseed
= coreSeed
^ prime1
; }
2174 /* srcBuffer selection [0-4] */
2175 { U32 buffNb
= FUZ_rand(&lseed
) & 0x7F;
2176 if (buffNb
& 7) buffNb
=2; /* most common : compressible (P) */
2180 const U32 tnb
[2] = { 1, 3 }; /* barely/highly compressible */
2181 buffNb
= tnb
[buffNb
>> 3];
2183 const U32 tnb
[2] = { 0, 4 }; /* not compressible / sparse */
2184 buffNb
= tnb
[buffNb
>> 3];
2186 srcBuffer
= cNoiseBuffer
[buffNb
];
2189 /* select src segment */
2190 sampleSize
= FUZ_randomLength(&lseed
, maxSampleLog
);
2192 /* create sample buffer (to catch read error with valgrind & sanitizers) */
2193 sampleBuffer
= (BYTE
*)malloc(sampleSize
);
2194 CHECK(sampleBuffer
==NULL
, "not enough memory for sample buffer");
2195 { size_t const sampleStart
= FUZ_rand(&lseed
) % (srcBufferSize
- sampleSize
);
2196 memcpy(sampleBuffer
, srcBuffer
+ sampleStart
, sampleSize
); }
2197 crcOrig
= XXH64(sampleBuffer
, sampleSize
, 0);
2199 /* compression tests */
2200 { int const cLevelPositive
=
2201 ( FUZ_rand(&lseed
) %
2202 (ZSTD_maxCLevel() - (FUZ_highbit32((U32
)sampleSize
) / cLevelLimiter
)) )
2204 int const cLevel
= ((FUZ_rand(&lseed
) & 15) == 3) ?
2205 - (int)((FUZ_rand(&lseed
) & 7) + 1) : /* test negative cLevel */
2207 DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb
, cLevel
);
2208 cSize
= ZSTD_compressCCtx(ctx
, cBuffer
, cBufferSize
, sampleBuffer
, sampleSize
, cLevel
);
2209 CHECK(ZSTD_isError(cSize
), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize
));
2211 /* compression failure test : too small dest buffer */
2213 { const size_t missing
= (FUZ_rand(&lseed
) % (cSize
-2)) + 1;
2214 const size_t tooSmallSize
= cSize
- missing
;
2215 const unsigned endMark
= 0x4DC2B1A9;
2216 memcpy(dstBuffer
+tooSmallSize
, &endMark
, sizeof(endMark
));
2217 DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n",
2218 testNb
, (unsigned)tooSmallSize
, (unsigned)missing
);
2219 { size_t const errorCode
= ZSTD_compressCCtx(ctx
, dstBuffer
, tooSmallSize
, sampleBuffer
, sampleSize
, cLevel
);
2220 CHECK(!ZSTD_isError(errorCode
), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize
, (unsigned)cSize
); }
2221 { unsigned endCheck
; memcpy(&endCheck
, dstBuffer
+tooSmallSize
, sizeof(endCheck
));
2222 CHECK(endCheck
!= endMark
, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck
, endMark
); }
2225 /* frame header decompression test */
2226 { ZSTD_frameHeader zfh
;
2227 CHECK_Z( ZSTD_getFrameHeader(&zfh
, cBuffer
, cSize
) );
2228 CHECK(zfh
.frameContentSize
!= sampleSize
, "Frame content size incorrect");
2231 /* Decompressed size test */
2232 { unsigned long long const rSize
= ZSTD_findDecompressedSize(cBuffer
, cSize
);
2233 CHECK(rSize
!= sampleSize
, "decompressed size incorrect");
2236 /* successful decompression test */
2237 DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb
);
2238 { size_t const margin
= (FUZ_rand(&lseed
) & 1) ? 0 : (FUZ_rand(&lseed
) & 31) + 1;
2239 size_t const dSize
= ZSTD_decompress(dstBuffer
, sampleSize
+ margin
, cBuffer
, cSize
);
2240 CHECK(dSize
!= sampleSize
, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize
), (unsigned)sampleSize
, (unsigned)cSize
);
2241 { U64
const crcDest
= XXH64(dstBuffer
, sampleSize
, 0);
2242 CHECK(crcOrig
!= crcDest
, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer
, dstBuffer
, sampleSize
), (unsigned)sampleSize
);
2245 free(sampleBuffer
); /* no longer useful after this point */
2247 /* truncated src decompression test */
2248 DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb
);
2249 { size_t const missing
= (FUZ_rand(&lseed
) % (cSize
-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
2250 size_t const tooSmallSize
= cSize
- missing
;
2251 void* cBufferTooSmall
= malloc(tooSmallSize
); /* valgrind will catch read overflows */
2252 CHECK(cBufferTooSmall
== NULL
, "not enough memory !");
2253 memcpy(cBufferTooSmall
, cBuffer
, tooSmallSize
);
2254 { size_t const errorCode
= ZSTD_decompress(dstBuffer
, dstBufferSize
, cBufferTooSmall
, tooSmallSize
);
2255 CHECK(!ZSTD_isError(errorCode
), "ZSTD_decompress should have failed ! (truncated src buffer)"); }
2256 free(cBufferTooSmall
);
2259 /* too small dst decompression test */
2260 DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb
);
2261 if (sampleSize
> 3) {
2262 size_t const missing
= (FUZ_rand(&lseed
) % (sampleSize
-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */
2263 size_t const tooSmallSize
= sampleSize
- missing
;
2264 static const BYTE token
= 0xA9;
2265 dstBuffer
[tooSmallSize
] = token
;
2266 { size_t const errorCode
= ZSTD_decompress(dstBuffer
, tooSmallSize
, cBuffer
, cSize
);
2267 CHECK(!ZSTD_isError(errorCode
), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode
, (unsigned)tooSmallSize
); }
2268 CHECK(dstBuffer
[tooSmallSize
] != token
, "ZSTD_decompress : dst buffer overflow");
2271 /* noisy src decompression test */
2273 /* insert noise into src */
2274 { U32
const maxNbBits
= FUZ_highbit32((U32
)(cSize
-4));
2275 size_t pos
= 4; /* preserve magic number (too easy to detect) */
2277 /* keep some original src */
2278 { U32
const nbBits
= FUZ_rand(&lseed
) % maxNbBits
;
2279 size_t const mask
= (1<<nbBits
) - 1;
2280 size_t const skipLength
= FUZ_rand(&lseed
) & mask
;
2283 if (pos
>= cSize
) break;
2285 { U32
const nbBitsCodes
= FUZ_rand(&lseed
) % maxNbBits
;
2286 U32
const nbBits
= nbBitsCodes
? nbBitsCodes
-1 : 0;
2287 size_t const mask
= (1<<nbBits
) - 1;
2288 size_t const rNoiseLength
= (FUZ_rand(&lseed
) & mask
) + 1;
2289 size_t const noiseLength
= MIN(rNoiseLength
, cSize
-pos
);
2290 size_t const noiseStart
= FUZ_rand(&lseed
) % (srcBufferSize
- noiseLength
);
2291 memcpy(cBuffer
+ pos
, srcBuffer
+ noiseStart
, noiseLength
);
2295 /* decompress noisy source */
2296 DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb
);
2297 { U32
const endMark
= 0xA9B1C3D6;
2298 memcpy(dstBuffer
+sampleSize
, &endMark
, 4);
2299 { size_t const decompressResult
= ZSTD_decompress(dstBuffer
, sampleSize
, cBuffer
, cSize
);
2300 /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */
2301 CHECK((!ZSTD_isError(decompressResult
)) && (decompressResult
>sampleSize
),
2302 "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult
, (unsigned)sampleSize
);
2304 { U32 endCheck
; memcpy(&endCheck
, dstBuffer
+sampleSize
, 4);
2305 CHECK(endMark
!=endCheck
, "ZSTD_decompress on noisy src : dst buffer overflow");
2306 } } } /* noisy src decompression test */
2308 /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/
2309 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb
);
2310 { U32
const testLog
= FUZ_rand(&lseed
) % maxSrcLog
;
2311 U32
const dictLog
= FUZ_rand(&lseed
) % maxSrcLog
;
2312 int const cLevel
= (FUZ_rand(&lseed
) %
2314 (MAX(testLog
, dictLog
) / cLevelLimiter
))) +
2316 maxTestSize
= FUZ_rLogLength(&lseed
, testLog
);
2317 if (maxTestSize
>= dstBufferSize
) maxTestSize
= dstBufferSize
-1;
2319 dictSize
= FUZ_rLogLength(&lseed
, dictLog
); /* needed also for decompression */
2320 dict
= srcBuffer
+ (FUZ_rand(&lseed
) % (srcBufferSize
- dictSize
));
2322 DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n",
2323 testNb
, (unsigned)maxTestSize
, cLevel
, (unsigned)dictSize
);
2325 if (FUZ_rand(&lseed
) & 0xF) {
2326 CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx
, dict
, dictSize
, cLevel
) );
2328 ZSTD_compressionParameters
const cPar
= ZSTD_getCParams(cLevel
, ZSTD_CONTENTSIZE_UNKNOWN
, dictSize
);
2329 ZSTD_frameParameters
const fPar
= { FUZ_rand(&lseed
)&1 /* contentSizeFlag */,
2330 !(FUZ_rand(&lseed
)&3) /* contentChecksumFlag*/,
2331 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */
2332 ZSTD_parameters
const p
= FUZ_makeParams(cPar
, fPar
);
2333 CHECK_Z ( ZSTD_compressBegin_advanced(refCtx
, dict
, dictSize
, p
, 0) );
2335 CHECK_Z( ZSTD_copyCCtx(ctx
, refCtx
, 0) );
2338 { U32
const nbChunks
= (FUZ_rand(&lseed
) & 127) + 2;
2340 XXH64_state_t xxhState
;
2341 XXH64_reset(&xxhState
, 0);
2342 for (totalTestSize
=0, cSize
=0, n
=0 ; n
<nbChunks
; n
++) {
2343 size_t const segmentSize
= FUZ_randomLength(&lseed
, maxSampleLog
);
2344 size_t const segmentStart
= FUZ_rand(&lseed
) % (srcBufferSize
- segmentSize
);
2346 if (cBufferSize
-cSize
< ZSTD_compressBound(segmentSize
)) break; /* avoid invalid dstBufferTooSmall */
2347 if (totalTestSize
+segmentSize
> maxTestSize
) break;
2349 { size_t const compressResult
= ZSTD_compressContinue(ctx
, cBuffer
+cSize
, cBufferSize
-cSize
, srcBuffer
+segmentStart
, segmentSize
);
2350 CHECK (ZSTD_isError(compressResult
), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult
));
2351 cSize
+= compressResult
;
2353 XXH64_update(&xxhState
, srcBuffer
+segmentStart
, segmentSize
);
2354 memcpy(mirrorBuffer
+ totalTestSize
, srcBuffer
+segmentStart
, segmentSize
);
2355 totalTestSize
+= segmentSize
;
2358 { size_t const flushResult
= ZSTD_compressEnd(ctx
, cBuffer
+cSize
, cBufferSize
-cSize
, NULL
, 0);
2359 CHECK (ZSTD_isError(flushResult
), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult
));
2360 cSize
+= flushResult
;
2362 crcOrig
= XXH64_digest(&xxhState
);
2365 /* streaming decompression test */
2366 DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb
);
2367 /* ensure memory requirement is good enough (should always be true) */
2368 { ZSTD_frameHeader zfh
;
2369 CHECK( ZSTD_getFrameHeader(&zfh
, cBuffer
, ZSTD_FRAMEHEADERSIZE_MAX
),
2370 "ZSTD_getFrameHeader(): error retrieving frame information");
2371 { size_t const roundBuffSize
= ZSTD_decodingBufferSize_min(zfh
.windowSize
, zfh
.frameContentSize
);
2372 CHECK_Z(roundBuffSize
);
2373 CHECK((roundBuffSize
> totalTestSize
) && (zfh
.frameContentSize
!=ZSTD_CONTENTSIZE_UNKNOWN
),
2374 "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)",
2375 (unsigned)roundBuffSize
, (unsigned)totalTestSize
);
2377 if (dictSize
<8) dictSize
=0, dict
=NULL
; /* disable dictionary */
2378 CHECK_Z( ZSTD_decompressBegin_usingDict(dctx
, dict
, dictSize
) );
2381 while (totalCSize
< cSize
) {
2382 size_t const inSize
= ZSTD_nextSrcSizeToDecompress(dctx
);
2383 size_t const genSize
= ZSTD_decompressContinue(dctx
, dstBuffer
+totalGenSize
, dstBufferSize
-totalGenSize
, cBuffer
+totalCSize
, inSize
);
2384 CHECK (ZSTD_isError(genSize
), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize
));
2385 totalGenSize
+= genSize
;
2386 totalCSize
+= inSize
;
2388 CHECK (ZSTD_nextSrcSizeToDecompress(dctx
) != 0, "frame not fully decoded");
2389 CHECK (totalGenSize
!= totalTestSize
, "streaming decompressed data : wrong size")
2390 CHECK (totalCSize
!= cSize
, "compressed data should be fully read")
2391 { U64
const crcDest
= XXH64(dstBuffer
, totalTestSize
, 0);
2392 CHECK(crcOrig
!= crcDest
, "streaming decompressed data corrupted (pos %u / %u)",
2393 (unsigned)findDiff(mirrorBuffer
, dstBuffer
, totalTestSize
), (unsigned)totalTestSize
);
2395 } /* for ( ; (testNb <= nbTests) */
2396 DISPLAY("\r%u fuzzer tests completed \n", testNb
-1);
2399 ZSTD_freeCCtx(refCtx
);
2401 ZSTD_freeDCtx(dctx
);
2402 free(cNoiseBuffer
[0]);
2403 free(cNoiseBuffer
[1]);
2404 free(cNoiseBuffer
[2]);
2405 free(cNoiseBuffer
[3]);
2406 free(cNoiseBuffer
[4]);
2418 /*_*******************************************************
2420 *********************************************************/
2421 static int FUZ_usage(const char* programName
)
2423 DISPLAY( "Usage :\n");
2424 DISPLAY( " %s [args]\n", programName
);
2426 DISPLAY( "Arguments :\n");
2427 DISPLAY( " -i# : Nb of tests (default:%i) \n", nbTestsDefault
);
2428 DISPLAY( " -s# : Select seed (default:prompt user)\n");
2429 DISPLAY( " -t# : Select starting test number (default:0)\n");
2430 DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default
);
2431 DISPLAY( " -v : verbose\n");
2432 DISPLAY( " -p : pause at the end\n");
2433 DISPLAY( " -h : display help and exit\n");
2437 /*! readU32FromChar() :
2438 @return : unsigned integer value read from input in `char` format
2439 allows and interprets K, KB, KiB, M, MB and MiB suffix.
2440 Will also modify `*stringPtr`, advancing it to position where it stopped reading.
2441 Note : function result can overflow if digit string > MAX_UINT */
2442 static unsigned readU32FromChar(const char** stringPtr
)
2444 unsigned result
= 0;
2445 while ((**stringPtr
>='0') && (**stringPtr
<='9'))
2446 result
*= 10, result
+= **stringPtr
- '0', (*stringPtr
)++ ;
2447 if ((**stringPtr
=='K') || (**stringPtr
=='M')) {
2449 if (**stringPtr
=='M') result
<<= 10;
2451 if (**stringPtr
=='i') (*stringPtr
)++;
2452 if (**stringPtr
=='B') (*stringPtr
)++;
2457 /** longCommandWArg() :
2458 * check if *stringPtr is the same as longCommand.
2459 * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
2460 * @return 0 and doesn't modify *stringPtr otherwise.
2462 static unsigned longCommandWArg(const char** stringPtr
, const char* longCommand
)
2464 size_t const comSize
= strlen(longCommand
);
2465 int const result
= !strncmp(*stringPtr
, longCommand
, comSize
);
2466 if (result
) *stringPtr
+= comSize
;
2470 int main(int argc
, const char** argv
)
2475 int nbTests
= nbTestsDefault
;
2477 int proba
= FUZ_compressibility_default
;
2480 U32 maxDuration
= 0;
2482 U32 memTestsOnly
= 0;
2483 const char* const programName
= argv
[0];
2485 /* Check command line */
2486 for (argNb
=1; argNb
<argc
; argNb
++) {
2487 const char* argument
= argv
[argNb
];
2488 if(!argument
) continue; /* Protection if argument empty */
2490 /* Handle commands. Aggregated commands are allowed */
2491 if (argument
[0]=='-') {
2493 if (longCommandWArg(&argument
, "--memtest=")) { memTestsOnly
= readU32FromChar(&argument
); continue; }
2495 if (!strcmp(argument
, "--memtest")) { memTestsOnly
=1; continue; }
2496 if (!strcmp(argument
, "--no-big-tests")) { bigTests
=0; continue; }
2499 while (*argument
!=0) {
2503 return FUZ_usage(programName
);
2515 case 'p': /* pause at the end */
2521 argument
++; maxDuration
= 0;
2522 nbTests
= readU32FromChar(&argument
);
2528 maxDuration
= readU32FromChar(&argument
);
2529 if (*argument
=='s') argument
++; /* seconds */
2530 if (*argument
=='m') maxDuration
*= 60, argument
++; /* minutes */
2531 if (*argument
=='n') argument
++;
2537 seed
= readU32FromChar(&argument
);
2542 testNb
= readU32FromChar(&argument
);
2545 case 'P': /* compressibility % */
2547 proba
= readU32FromChar(&argument
);
2548 if (proba
>100) proba
= 100;
2552 return (FUZ_usage(programName
), 1);
2553 } } } } /* for (argNb=1; argNb<argc; argNb++) */
2556 DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING
);
2559 time_t const t
= time(NULL
);
2560 U32
const h
= XXH32(&t
, sizeof(t
), 1);
2564 DISPLAY("Seed = %u\n", (unsigned)seed
);
2565 if (proba
!=FUZ_compressibility_default
) DISPLAY("Compressibility : %i%%\n", proba
);
2568 g_displayLevel
= MAX(3, g_displayLevel
);
2569 return FUZ_mallocTests(seed
, ((double)proba
) / 100, memTestsOnly
);
2572 if (nbTests
< testNb
) nbTests
= testNb
;
2575 result
= basicUnitTests(0, ((double)proba
) / 100); /* constant seed for predictability */
2577 result
= fuzzerTests(seed
, nbTests
, testNb
, maxDuration
, ((double)proba
) / 100, bigTests
);
2580 DISPLAY("Press Enter \n");