]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | /* |
2 | * Copyright (c) 2015-present, Yann Collet, Facebook, Inc. | |
7c673cae FG |
3 | * All rights reserved. |
4 | * | |
11fdf7f2 TL |
5 | * This source code is licensed under both the BSD-style license (found in the |
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found | |
7 | * in the COPYING file in the root directory of this source tree). | |
8 | * You may select, at your option, one of the above-listed licenses. | |
7c673cae FG |
9 | */ |
10 | ||
11 | ||
12 | /*-************************************ | |
13 | * Compiler specific | |
14 | **************************************/ | |
15 | #ifdef _MSC_VER /* Visual Studio */ | |
11fdf7f2 TL |
16 | # define _CRT_SECURE_NO_WARNINGS /* fgets */ |
17 | # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ | |
18 | # pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ | |
7c673cae FG |
19 | #endif |
20 | ||
21 | ||
22 | /*-************************************ | |
23 | * Includes | |
24 | **************************************/ | |
25 | #include <stdlib.h> /* free */ | |
26 | #include <stdio.h> /* fgets, sscanf */ | |
27 | #include <string.h> /* strcmp */ | |
9f95a23c | 28 | #include <assert.h> |
11fdf7f2 | 29 | #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ |
9f95a23c | 30 | #include "fse.h" |
7c673cae FG |
31 | #include "zstd.h" /* ZSTD_VERSION_STRING */ |
32 | #include "zstd_errors.h" /* ZSTD_getErrorCode */ | |
11fdf7f2 TL |
33 | #include "zstdmt_compress.h" |
34 | #define ZDICT_STATIC_LINKING_ONLY | |
7c673cae FG |
35 | #include "zdict.h" /* ZDICT_trainFromBuffer */ |
36 | #include "datagen.h" /* RDG_genBuffer */ | |
37 | #include "mem.h" | |
9f95a23c | 38 | #define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ |
7c673cae | 39 | #include "xxhash.h" /* XXH64 */ |
9f95a23c TL |
40 | #include "util.h" |
41 | #include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */ | |
7c673cae FG |
42 | |
43 | ||
44 | /*-************************************ | |
45 | * Constants | |
46 | **************************************/ | |
47 | #define KB *(1U<<10) | |
48 | #define MB *(1U<<20) | |
49 | #define GB *(1U<<30) | |
50 | ||
9f95a23c TL |
51 | static const int FUZ_compressibility_default = 50; |
52 | static const int nbTestsDefault = 30000; | |
7c673cae FG |
53 | |
54 | ||
55 | /*-************************************ | |
56 | * Display Macros | |
57 | **************************************/ | |
9f95a23c | 58 | #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) |
7c673cae FG |
59 | #define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } |
60 | static U32 g_displayLevel = 2; | |
61 | ||
9f95a23c TL |
62 | static const U64 g_refreshRate = SEC_TO_MICRO / 6; |
63 | static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; | |
64 | ||
7c673cae | 65 | #define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ |
9f95a23c TL |
66 | if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ |
67 | { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ | |
68 | if (g_displayLevel>=4) fflush(stderr); } } | |
69 | ||
70 | ||
71 | /*-******************************************************* | |
72 | * Compile time test | |
73 | *********************************************************/ | |
74 | #undef MIN | |
75 | #undef MAX | |
76 | /* Declaring the function is it isn't unused */ | |
77 | void FUZ_bug976(void); | |
78 | void FUZ_bug976(void) | |
79 | { /* these constants shall not depend on MIN() macro */ | |
80 | assert(ZSTD_HASHLOG_MAX < 31); | |
81 | assert(ZSTD_CHAINLOG_MAX < 31); | |
82 | } | |
7c673cae FG |
83 | |
84 | ||
85 | /*-******************************************************* | |
9f95a23c | 86 | * Internal functions |
7c673cae FG |
87 | *********************************************************/ |
88 | #define MIN(a,b) ((a)<(b)?(a):(b)) | |
11fdf7f2 | 89 | #define MAX(a,b) ((a)>(b)?(a):(b)) |
7c673cae | 90 | |
7c673cae | 91 | #define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) |
9f95a23c | 92 | static U32 FUZ_rand(U32* src) |
7c673cae FG |
93 | { |
94 | static const U32 prime1 = 2654435761U; | |
95 | static const U32 prime2 = 2246822519U; | |
96 | U32 rand32 = *src; | |
97 | rand32 *= prime1; | |
98 | rand32 += prime2; | |
99 | rand32 = FUZ_rotl32(rand32, 13); | |
100 | *src = rand32; | |
101 | return rand32 >> 5; | |
102 | } | |
103 | ||
9f95a23c | 104 | static U32 FUZ_highbit32(U32 v32) |
7c673cae FG |
105 | { |
106 | unsigned nbBits = 0; | |
107 | if (v32==0) return 0; | |
108 | while (v32) v32 >>= 1, nbBits++; | |
109 | return nbBits; | |
110 | } | |
111 | ||
112 | ||
9f95a23c TL |
113 | /*============================================= |
114 | * Test macros | |
115 | =============================================*/ | |
116 | #define CHECK_Z(f) { \ | |
117 | size_t const err = f; \ | |
118 | if (ZSTD_isError(err)) { \ | |
119 | DISPLAY("Error => %s : %s ", \ | |
120 | #f, ZSTD_getErrorName(err)); \ | |
121 | exit(1); \ | |
122 | } } | |
123 | ||
124 | #define CHECK_V(var, fn) size_t const var = fn; if (ZSTD_isError(var)) goto _output_error | |
125 | #define CHECK(fn) { CHECK_V(err, fn); } | |
126 | #define CHECKPLUS(var, fn, more) { CHECK_V(var, fn); more; } | |
127 | ||
128 | #define CHECK_OP(op, lhs, rhs) { \ | |
129 | if (!((lhs) op (rhs))) { \ | |
130 | DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \ | |
131 | goto _output_error; \ | |
132 | } \ | |
133 | } | |
134 | #define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs) | |
135 | #define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs) | |
136 | ||
137 | ||
7c673cae | 138 | /*============================================= |
11fdf7f2 TL |
139 | * Memory Tests |
140 | =============================================*/ | |
141 | #if defined(__APPLE__) && defined(__MACH__) | |
142 | ||
143 | #include <malloc/malloc.h> /* malloc_size */ | |
144 | ||
145 | typedef struct { | |
146 | unsigned long long totalMalloc; | |
147 | size_t currentMalloc; | |
148 | size_t peakMalloc; | |
149 | unsigned nbMalloc; | |
150 | unsigned nbFree; | |
151 | } mallocCounter_t; | |
152 | ||
153 | static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 }; | |
154 | ||
155 | static void* FUZ_mallocDebug(void* counter, size_t size) | |
156 | { | |
157 | mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; | |
158 | void* const ptr = malloc(size); | |
159 | if (ptr==NULL) return NULL; | |
160 | DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n", | |
9f95a23c | 161 | (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */ |
11fdf7f2 TL |
162 | mcPtr->totalMalloc += size; |
163 | mcPtr->currentMalloc += size; | |
164 | if (mcPtr->currentMalloc > mcPtr->peakMalloc) | |
165 | mcPtr->peakMalloc = mcPtr->currentMalloc; | |
166 | mcPtr->nbMalloc += 1; | |
167 | return ptr; | |
168 | } | |
169 | ||
170 | static void FUZ_freeDebug(void* counter, void* address) | |
171 | { | |
172 | mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; | |
9f95a23c | 173 | DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10)); |
11fdf7f2 TL |
174 | mcPtr->nbFree += 1; |
175 | mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */ | |
176 | free(address); | |
177 | } | |
178 | ||
179 | static void FUZ_displayMallocStats(mallocCounter_t count) | |
180 | { | |
181 | DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n", | |
9f95a23c | 182 | (unsigned)(count.peakMalloc >> 10), |
11fdf7f2 | 183 | count.nbMalloc, |
9f95a23c | 184 | (unsigned)(count.totalMalloc >> 10)); |
11fdf7f2 TL |
185 | } |
186 | ||
9f95a23c TL |
187 | static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part, |
188 | void* inBuffer, size_t inSize, void* outBuffer, size_t outSize) | |
11fdf7f2 | 189 | { |
11fdf7f2 TL |
190 | /* test only played in verbose mode, as they are long */ |
191 | if (g_displayLevel<3) return 0; | |
192 | ||
193 | /* Create compressible noise */ | |
194 | if (!inBuffer || !outBuffer) { | |
195 | DISPLAY("Not enough memory, aborting\n"); | |
196 | exit(1); | |
197 | } | |
198 | RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed); | |
199 | ||
200 | /* simple compression tests */ | |
201 | if (part <= 1) | |
202 | { int compressionLevel; | |
203 | for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { | |
204 | mallocCounter_t malcount = INIT_MALLOC_COUNTER; | |
205 | ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; | |
206 | ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); | |
207 | CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) ); | |
208 | ZSTD_freeCCtx(cctx); | |
209 | DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel); | |
210 | FUZ_displayMallocStats(malcount); | |
211 | } } | |
212 | ||
213 | /* streaming compression tests */ | |
214 | if (part <= 2) | |
215 | { int compressionLevel; | |
216 | for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { | |
217 | mallocCounter_t malcount = INIT_MALLOC_COUNTER; | |
218 | ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; | |
219 | ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem); | |
220 | ZSTD_outBuffer out = { outBuffer, outSize, 0 }; | |
221 | ZSTD_inBuffer in = { inBuffer, inSize, 0 }; | |
222 | CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) ); | |
223 | CHECK_Z( ZSTD_compressStream(cstream, &out, &in) ); | |
224 | CHECK_Z( ZSTD_endStream(cstream, &out) ); | |
225 | ZSTD_freeCStream(cstream); | |
226 | DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel); | |
227 | FUZ_displayMallocStats(malcount); | |
228 | } } | |
229 | ||
230 | /* advanced MT API test */ | |
231 | if (part <= 3) | |
9f95a23c | 232 | { unsigned nbThreads; |
11fdf7f2 TL |
233 | for (nbThreads=1; nbThreads<=4; nbThreads++) { |
234 | int compressionLevel; | |
235 | for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { | |
236 | mallocCounter_t malcount = INIT_MALLOC_COUNTER; | |
237 | ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; | |
238 | ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); | |
9f95a23c TL |
239 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); |
240 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) ); | |
241 | CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) ); | |
11fdf7f2 TL |
242 | ZSTD_freeCCtx(cctx); |
243 | DISPLAYLEVEL(3, "compress_generic,-T%u,end level %i : ", | |
244 | nbThreads, compressionLevel); | |
245 | FUZ_displayMallocStats(malcount); | |
246 | } } } | |
247 | ||
248 | /* advanced MT streaming API test */ | |
249 | if (part <= 4) | |
9f95a23c | 250 | { unsigned nbThreads; |
11fdf7f2 TL |
251 | for (nbThreads=1; nbThreads<=4; nbThreads++) { |
252 | int compressionLevel; | |
253 | for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { | |
254 | mallocCounter_t malcount = INIT_MALLOC_COUNTER; | |
255 | ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; | |
256 | ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); | |
257 | ZSTD_outBuffer out = { outBuffer, outSize, 0 }; | |
258 | ZSTD_inBuffer in = { inBuffer, inSize, 0 }; | |
9f95a23c TL |
259 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); |
260 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) ); | |
261 | CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) ); | |
262 | while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {} | |
11fdf7f2 TL |
263 | ZSTD_freeCCtx(cctx); |
264 | DISPLAYLEVEL(3, "compress_generic,-T%u,continue level %i : ", | |
265 | nbThreads, compressionLevel); | |
266 | FUZ_displayMallocStats(malcount); | |
267 | } } } | |
268 | ||
269 | return 0; | |
270 | } | |
271 | ||
9f95a23c TL |
272 | static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) |
273 | { | |
274 | size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */ | |
275 | size_t const outSize = ZSTD_compressBound(inSize); | |
276 | void* const inBuffer = malloc(inSize); | |
277 | void* const outBuffer = malloc(outSize); | |
278 | int result; | |
279 | ||
280 | /* Create compressible noise */ | |
281 | if (!inBuffer || !outBuffer) { | |
282 | DISPLAY("Not enough memory, aborting \n"); | |
283 | exit(1); | |
284 | } | |
285 | ||
286 | result = FUZ_mallocTests_internal(seed, compressibility, part, | |
287 | inBuffer, inSize, outBuffer, outSize); | |
288 | ||
289 | free(inBuffer); | |
290 | free(outBuffer); | |
291 | return result; | |
292 | } | |
293 | ||
11fdf7f2 TL |
294 | #else |
295 | ||
296 | static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) | |
297 | { | |
298 | (void)seed; (void)compressibility; (void)part; | |
299 | return 0; | |
300 | } | |
301 | ||
302 | #endif | |
303 | ||
304 | /*============================================= | |
305 | * Unit tests | |
7c673cae FG |
306 | =============================================*/ |
307 | ||
7c673cae FG |
308 | static int basicUnitTests(U32 seed, double compressibility) |
309 | { | |
310 | size_t const CNBuffSize = 5 MB; | |
311 | void* const CNBuffer = malloc(CNBuffSize); | |
11fdf7f2 TL |
312 | size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); |
313 | void* const compressedBuffer = malloc(compressedBufferSize); | |
7c673cae FG |
314 | void* const decodedBuffer = malloc(CNBuffSize); |
315 | int testResult = 0; | |
9f95a23c | 316 | unsigned testNb=0; |
7c673cae FG |
317 | size_t cSize; |
318 | ||
319 | /* Create compressible noise */ | |
320 | if (!CNBuffer || !compressedBuffer || !decodedBuffer) { | |
321 | DISPLAY("Not enough memory, aborting\n"); | |
322 | testResult = 1; | |
323 | goto _end; | |
324 | } | |
325 | RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); | |
326 | ||
327 | /* Basic tests */ | |
9f95a23c | 328 | DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++); |
7c673cae | 329 | { const char* errorString = ZSTD_getErrorName(0); |
9f95a23c | 330 | DISPLAYLEVEL(3, "OK : %s \n", errorString); |
7c673cae FG |
331 | } |
332 | ||
9f95a23c | 333 | DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++); |
7c673cae | 334 | { const char* errorString = ZSTD_getErrorName(499); |
9f95a23c | 335 | DISPLAYLEVEL(3, "OK : %s \n", errorString); |
7c673cae FG |
336 | } |
337 | ||
9f95a23c TL |
338 | DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++); |
339 | { int const mcl = ZSTD_minCLevel(); | |
340 | DISPLAYLEVEL(3, "%i (OK) \n", mcl); | |
341 | } | |
11fdf7f2 | 342 | |
9f95a23c TL |
343 | DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize); |
344 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
11fdf7f2 TL |
345 | if (cctx==NULL) goto _output_error; |
346 | CHECKPLUS(r, ZSTD_compressCCtx(cctx, | |
347 | compressedBuffer, compressedBufferSize, | |
348 | CNBuffer, CNBuffSize, 1), | |
349 | cSize=r ); | |
9f95a23c | 350 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
11fdf7f2 | 351 | |
9f95a23c | 352 | DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++); |
11fdf7f2 | 353 | { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx); |
9f95a23c | 354 | DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize); |
11fdf7f2 TL |
355 | } |
356 | ZSTD_freeCCtx(cctx); | |
357 | } | |
358 | ||
9f95a23c TL |
359 | DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++); |
360 | { | |
361 | char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff"; | |
362 | size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8); | |
363 | if (!ZSTD_isError(size)) goto _output_error; | |
364 | } | |
365 | DISPLAYLEVEL(3, "OK \n"); | |
366 | ||
11fdf7f2 | 367 | |
9f95a23c | 368 | DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++); |
11fdf7f2 TL |
369 | { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); |
370 | if (rSize != CNBuffSize) goto _output_error; | |
371 | } | |
9f95a23c | 372 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 373 | |
9f95a23c | 374 | DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++); |
11fdf7f2 | 375 | { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize); |
7c673cae FG |
376 | if (rSize != CNBuffSize) goto _output_error; |
377 | } | |
9f95a23c TL |
378 | DISPLAYLEVEL(3, "OK \n"); |
379 | ||
380 | DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++); | |
381 | { | |
382 | unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize); | |
383 | if (bound != CNBuffSize) goto _output_error; | |
384 | } | |
385 | DISPLAYLEVEL(3, "OK \n"); | |
386 | ||
387 | DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++); | |
388 | { | |
389 | unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1); | |
390 | if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error; | |
391 | } | |
392 | DISPLAYLEVEL(3, "OK \n"); | |
7c673cae | 393 | |
9f95a23c | 394 | DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); |
7c673cae FG |
395 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
396 | if (r != CNBuffSize) goto _output_error; } | |
9f95a23c | 397 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 398 | |
9f95a23c | 399 | DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); |
7c673cae FG |
400 | { size_t u; |
401 | for (u=0; u<CNBuffSize; u++) { | |
402 | if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;; | |
403 | } } | |
9f95a23c | 404 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 405 | |
11fdf7f2 | 406 | |
9f95a23c TL |
407 | DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++); |
408 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); | |
409 | { size_t const r = ZSTD_decompress_usingDict(dctx, | |
410 | decodedBuffer, CNBuffSize, | |
411 | compressedBuffer, cSize, | |
412 | NULL, 0); | |
413 | if (r != CNBuffSize) goto _output_error; | |
414 | } | |
415 | ZSTD_freeDCtx(dctx); | |
416 | } | |
417 | DISPLAYLEVEL(3, "OK \n"); | |
418 | ||
419 | DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++); | |
420 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); | |
421 | { size_t const r = ZSTD_decompress_usingDDict(dctx, | |
422 | decodedBuffer, CNBuffSize, | |
423 | compressedBuffer, cSize, | |
424 | NULL); | |
425 | if (r != CNBuffSize) goto _output_error; | |
426 | } | |
427 | ZSTD_freeDCtx(dctx); | |
428 | } | |
429 | DISPLAYLEVEL(3, "OK \n"); | |
11fdf7f2 | 430 | |
9f95a23c | 431 | DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++); |
7c673cae FG |
432 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1); |
433 | if (!ZSTD_isError(r)) goto _output_error; | |
434 | if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; } | |
9f95a23c | 435 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 436 | |
9f95a23c | 437 | DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++); |
7c673cae FG |
438 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1); |
439 | if (!ZSTD_isError(r)) goto _output_error; | |
440 | if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } | |
9f95a23c | 441 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 442 | |
9f95a23c TL |
443 | DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++); |
444 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize); | |
445 | if (!ZSTD_isError(r)) goto _output_error; | |
446 | if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } | |
447 | DISPLAYLEVEL(3, "OK \n"); | |
448 | ||
449 | DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++); | |
450 | { /* create compressed buffer with content size missing */ | |
451 | ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
452 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) ); | |
453 | CHECKPLUS(r, ZSTD_compress2(cctx, | |
454 | compressedBuffer, compressedBufferSize, | |
455 | CNBuffer, CNBuffSize), | |
456 | cSize=r ); | |
457 | ZSTD_freeCCtx(cctx); | |
458 | } | |
459 | { /* ensure frame content size is missing */ | |
460 | ZSTD_frameHeader zfh; | |
461 | size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize); | |
462 | if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; | |
463 | } | |
464 | { /* ensure CNBuffSize <= decompressBound */ | |
465 | unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize); | |
466 | if (CNBuffSize > bound) goto _output_error; | |
467 | } | |
468 | DISPLAYLEVEL(3, "OK \n"); | |
469 | ||
470 | DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++); | |
471 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
472 | size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19); | |
473 | if (ZSTD_isError(r)) goto _output_error; | |
474 | if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error; | |
475 | ZSTD_freeCCtx(cctx); | |
476 | cSize = r; | |
477 | } | |
478 | DISPLAYLEVEL(3, "OK \n"); | |
479 | ||
480 | DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++); | |
481 | { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize); | |
482 | if (ZSTD_isError(r)) goto _output_error; | |
483 | if (r != 0) goto _output_error; | |
484 | } | |
485 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
486 | ZSTD_outBuffer output; | |
487 | if (cctx==NULL) goto _output_error; | |
488 | output.dst = compressedBuffer; | |
489 | output.size = compressedBufferSize; | |
490 | output.pos = 0; | |
491 | CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */ | |
492 | CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */ | |
493 | CHECK_Z( ZSTD_endStream(cctx, &output) ); | |
494 | ZSTD_freeCCtx(cctx); | |
495 | /* single scan decompression */ | |
496 | { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos); | |
497 | if (ZSTD_isError(r)) goto _output_error; | |
498 | if (r != 0) goto _output_error; | |
499 | } | |
500 | /* streaming decompression */ | |
501 | { ZSTD_DCtx* const dstream = ZSTD_createDStream(); | |
502 | ZSTD_inBuffer dinput; | |
503 | ZSTD_outBuffer doutput; | |
504 | size_t ipos; | |
505 | if (dstream==NULL) goto _output_error; | |
506 | dinput.src = compressedBuffer; | |
507 | dinput.size = 0; | |
508 | dinput.pos = 0; | |
509 | doutput.dst = NULL; | |
510 | doutput.size = 0; | |
511 | doutput.pos = 0; | |
512 | CHECK_Z ( ZSTD_initDStream(dstream) ); | |
513 | for (ipos=1; ipos<=output.pos; ipos++) { | |
514 | dinput.size = ipos; | |
515 | CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) ); | |
516 | } | |
517 | if (doutput.pos != 0) goto _output_error; | |
518 | ZSTD_freeDStream(dstream); | |
519 | } | |
520 | } | |
521 | DISPLAYLEVEL(3, "OK \n"); | |
522 | ||
523 | DISPLAYLEVEL(3, "test%3d : re-use CCtx with expanding block size : ", testNb++); | |
524 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
525 | ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0); | |
526 | assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */ | |
527 | CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) ); | |
528 | CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */ | |
529 | ||
530 | CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */ | |
531 | { size_t const inSize = 2* 128 KB; | |
532 | size_t const outSize = ZSTD_compressBound(inSize); | |
533 | CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) ); | |
534 | /* will fail if blockSize is not resized */ | |
535 | } | |
536 | ZSTD_freeCCtx(cctx); | |
537 | } | |
538 | DISPLAYLEVEL(3, "OK \n"); | |
539 | ||
540 | DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++); | |
541 | { size_t const sampleSize = 30; | |
542 | int i; | |
543 | for (i=0; i<20; i++) | |
544 | ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */ | |
545 | memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */ | |
546 | for (i=1; i<=19; i++) { | |
547 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
548 | size_t size1, size2; | |
549 | DISPLAYLEVEL(5, "l%i ", i); | |
550 | size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i); | |
551 | CHECK_Z(size1); | |
552 | ||
553 | size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i); | |
554 | CHECK_Z(size2); | |
555 | CHECK_EQ(size1, size2); | |
556 | ||
557 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) ); | |
558 | size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize); | |
559 | CHECK_Z(size2); | |
560 | CHECK_EQ(size1, size2); | |
561 | ||
562 | size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */ | |
563 | CHECK_Z(size2); | |
564 | CHECK_EQ(size1, size2); | |
565 | ||
566 | { ZSTD_inBuffer inb; | |
567 | ZSTD_outBuffer outb; | |
568 | inb.src = CNBuffer; | |
569 | inb.pos = 0; | |
570 | inb.size = sampleSize; | |
571 | outb.dst = compressedBuffer; | |
572 | outb.pos = 0; | |
573 | outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */ | |
574 | CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); | |
575 | assert(inb.pos == inb.size); | |
576 | CHECK_EQ(size1, outb.pos); | |
577 | } | |
578 | ||
579 | ZSTD_freeCCtx(cctx); | |
580 | } | |
581 | } | |
582 | DISPLAYLEVEL(3, "OK \n"); | |
583 | ||
584 | DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++); | |
585 | { size_t const sampleSize = 1024; | |
586 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
587 | ZSTD_inBuffer inb; | |
588 | ZSTD_outBuffer outb; | |
589 | inb.src = CNBuffer; | |
590 | inb.pos = 0; | |
591 | inb.size = 0; | |
592 | outb.dst = compressedBuffer; | |
593 | outb.pos = 0; | |
594 | outb.size = compressedBufferSize; | |
595 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) ); | |
596 | ||
597 | inb.size = sampleSize; /* start with something, so that context is already used */ | |
598 | CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */ | |
599 | assert(inb.pos == inb.size); | |
600 | outb.pos = 0; /* cancel output */ | |
601 | ||
602 | CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) ); | |
603 | inb.size = 4; /* too small size : compression will be skipped */ | |
604 | inb.pos = 0; | |
605 | CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); | |
606 | assert(inb.pos == inb.size); | |
607 | ||
608 | inb.size += 5; /* too small size : compression will be skipped */ | |
609 | CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); | |
610 | assert(inb.pos == inb.size); | |
611 | ||
612 | inb.size += 11; /* small enough to attempt compression */ | |
613 | CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); | |
614 | assert(inb.pos == inb.size); | |
615 | ||
616 | assert(inb.pos < sampleSize); | |
617 | inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */ | |
618 | CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */ | |
619 | assert(inb.pos == inb.size); | |
620 | ZSTD_freeCCtx(cctx); | |
621 | } | |
622 | DISPLAYLEVEL(3, "OK \n"); | |
623 | ||
624 | DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++); | |
625 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
626 | ZSTD_outBuffer out = {NULL, 0, 0}; | |
627 | ZSTD_inBuffer in = {NULL, 0, 0}; | |
628 | int value; | |
629 | ||
630 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); | |
631 | CHECK_EQ(value, 3); | |
632 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
633 | CHECK_EQ(value, 0); | |
634 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN)); | |
635 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); | |
636 | CHECK_EQ(value, 3); | |
637 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
638 | CHECK_EQ(value, ZSTD_HASHLOG_MIN); | |
639 | CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7)); | |
640 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); | |
641 | CHECK_EQ(value, 7); | |
642 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
643 | CHECK_EQ(value, ZSTD_HASHLOG_MIN); | |
644 | /* Start a compression job */ | |
645 | ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue); | |
646 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); | |
647 | CHECK_EQ(value, 7); | |
648 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
649 | CHECK_EQ(value, ZSTD_HASHLOG_MIN); | |
650 | /* Reset the CCtx */ | |
651 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); | |
652 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); | |
653 | CHECK_EQ(value, 7); | |
654 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
655 | CHECK_EQ(value, ZSTD_HASHLOG_MIN); | |
656 | /* Reset the parameters */ | |
657 | ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); | |
658 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); | |
659 | CHECK_EQ(value, 3); | |
660 | CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); | |
661 | CHECK_EQ(value, 0); | |
662 | ||
663 | ZSTD_freeCCtx(cctx); | |
664 | } | |
665 | DISPLAYLEVEL(3, "OK \n"); | |
666 | ||
667 | /* this test is really too long, and should be made faster */ | |
668 | DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++); | |
669 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
670 | ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0); | |
671 | size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */ | |
672 | size_t cnb; | |
673 | assert(cctx != NULL); | |
674 | params.fParams.contentSizeFlag = 0; | |
675 | params.cParams.windowLog = ZSTD_WINDOWLOG_MAX; | |
676 | for (cnb = 0; cnb < nbCompressions; ++cnb) { | |
677 | DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions); | |
678 | CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */ | |
679 | CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) ); | |
680 | } | |
681 | ZSTD_freeCCtx(cctx); | |
682 | } | |
683 | DISPLAYLEVEL(3, "OK \n"); | |
684 | ||
685 | DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++); | |
686 | { ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx(); | |
687 | assert(largeCCtx != NULL); | |
688 | CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */ | |
689 | CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); | |
690 | { size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */ | |
691 | { ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx(); | |
692 | assert(smallCCtx != NULL); | |
693 | CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1)); | |
694 | { size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx); | |
695 | DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ", | |
696 | largeCCtxSize>>10, smallCCtxSize>>10); | |
697 | assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c . | |
698 | * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */ | |
699 | } | |
700 | ZSTD_freeCCtx(smallCCtx); | |
701 | } | |
702 | { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c. | |
703 | * currently defined as 128x, but could be adjusted in the future. | |
704 | * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */ | |
705 | unsigned u; | |
706 | for (u=0; u<maxNbAttempts; u++) { | |
707 | CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1)); | |
708 | if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */ | |
709 | } | |
710 | DISPLAYLEVEL(5, "size down after %u attempts : ", u); | |
711 | if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */ | |
712 | } | |
713 | } | |
714 | ZSTD_freeCCtx(largeCCtx); | |
715 | } | |
716 | DISPLAYLEVEL(3, "OK \n"); | |
11fdf7f2 TL |
717 | |
718 | /* Static CCtx tests */ | |
719 | #define STATIC_CCTX_LEVEL 3 | |
9f95a23c | 720 | DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u :", testNb++, STATIC_CCTX_LEVEL); |
11fdf7f2 TL |
721 | { size_t const staticCCtxSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL); |
722 | void* const staticCCtxBuffer = malloc(staticCCtxSize); | |
723 | size_t const staticDCtxSize = ZSTD_estimateDCtxSize(); | |
724 | void* const staticDCtxBuffer = malloc(staticDCtxSize); | |
725 | if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) { | |
726 | free(staticCCtxBuffer); | |
727 | free(staticDCtxBuffer); | |
728 | DISPLAY("Not enough memory, aborting\n"); | |
729 | testResult = 1; | |
730 | goto _end; | |
731 | } | |
732 | { ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize); | |
733 | ZSTD_DCtx* staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize); | |
734 | if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error; | |
9f95a23c | 735 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 736 | |
9f95a23c | 737 | DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); |
11fdf7f2 TL |
738 | { size_t const r = ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL); |
739 | if (ZSTD_isError(r)) goto _output_error; } | |
9f95a23c | 740 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 741 | |
9f95a23c | 742 | DISPLAYLEVEL(3, "test%3i : simple compression test with static CCtx : ", testNb++); |
11fdf7f2 TL |
743 | CHECKPLUS(r, ZSTD_compressCCtx(staticCCtx, |
744 | compressedBuffer, compressedBufferSize, | |
745 | CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL), | |
746 | cSize=r ); | |
9f95a23c TL |
747 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", |
748 | (unsigned)cSize, (double)cSize/CNBuffSize*100); | |
11fdf7f2 | 749 | |
9f95a23c | 750 | DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++); |
11fdf7f2 TL |
751 | { size_t const r = ZSTD_decompressDCtx(staticDCtx, |
752 | decodedBuffer, CNBuffSize, | |
753 | compressedBuffer, cSize); | |
754 | if (r != CNBuffSize) goto _output_error; } | |
9f95a23c | 755 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 756 | |
9f95a23c | 757 | DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); |
11fdf7f2 TL |
758 | { size_t u; |
759 | for (u=0; u<CNBuffSize; u++) { | |
760 | if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) | |
761 | goto _output_error;; | |
762 | } } | |
9f95a23c | 763 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 764 | |
9f95a23c | 765 | DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++); |
11fdf7f2 TL |
766 | { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel()); |
767 | if (!ZSTD_isError(r)) goto _output_error; } | |
9f95a23c | 768 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 769 | |
9f95a23c | 770 | DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1); |
11fdf7f2 TL |
771 | { size_t const r = ZSTD_compressBegin(staticCCtx, 1); |
772 | if (ZSTD_isError(r)) goto _output_error; } | |
9f95a23c | 773 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 774 | |
9f95a23c | 775 | DISPLAYLEVEL(3, "test%3i : init CStream for small level %u : ", testNb++, 1); |
11fdf7f2 TL |
776 | { size_t const r = ZSTD_initCStream(staticCCtx, 1); |
777 | if (ZSTD_isError(r)) goto _output_error; } | |
9f95a23c | 778 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 779 | |
9f95a23c | 780 | DISPLAYLEVEL(3, "test%3i : init CStream with dictionary (should fail) : ", testNb++); |
11fdf7f2 TL |
781 | { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1); |
782 | if (!ZSTD_isError(r)) goto _output_error; } | |
9f95a23c | 783 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 784 | |
9f95a23c | 785 | DISPLAYLEVEL(3, "test%3i : init DStream (should fail) : ", testNb++); |
11fdf7f2 TL |
786 | { size_t const r = ZSTD_initDStream(staticDCtx); |
787 | if (ZSTD_isError(r)) goto _output_error; } | |
788 | { ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 }; | |
789 | ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 }; | |
790 | size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input); | |
791 | if (!ZSTD_isError(r)) goto _output_error; | |
792 | } | |
9f95a23c | 793 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 TL |
794 | } |
795 | free(staticCCtxBuffer); | |
796 | free(staticDCtxBuffer); | |
797 | } | |
798 | ||
9f95a23c TL |
799 | DISPLAYLEVEL(3, "test%3i : Static negative levels : ", testNb++); |
800 | { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1); | |
801 | size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1); | |
802 | size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1); | |
803 | size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1); | |
804 | ||
805 | if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error; | |
806 | if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error; | |
807 | } | |
808 | DISPLAYLEVEL(3, "OK \n"); | |
11fdf7f2 TL |
809 | |
810 | ||
811 | /* ZSTDMT simple MT compression test */ | |
9f95a23c | 812 | DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++); |
11fdf7f2 TL |
813 | { ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2); |
814 | if (mtctx==NULL) { | |
815 | DISPLAY("mtctx : mot enough memory, aborting \n"); | |
816 | testResult = 1; | |
817 | goto _end; | |
818 | } | |
9f95a23c | 819 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 820 | |
9f95a23c | 821 | DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize); |
11fdf7f2 TL |
822 | CHECKPLUS(r, ZSTDMT_compressCCtx(mtctx, |
823 | compressedBuffer, compressedBufferSize, | |
824 | CNBuffer, CNBuffSize, | |
825 | 1), | |
826 | cSize=r ); | |
9f95a23c | 827 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
11fdf7f2 | 828 | |
9f95a23c | 829 | DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++); |
11fdf7f2 TL |
830 | { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); |
831 | if (rSize != CNBuffSize) { | |
9f95a23c | 832 | DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize); |
11fdf7f2 TL |
833 | goto _output_error; |
834 | } } | |
9f95a23c | 835 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 836 | |
9f95a23c | 837 | DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); |
11fdf7f2 TL |
838 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
839 | if (r != CNBuffSize) goto _output_error; } | |
9f95a23c | 840 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 841 | |
9f95a23c | 842 | DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); |
11fdf7f2 TL |
843 | { size_t u; |
844 | for (u=0; u<CNBuffSize; u++) { | |
845 | if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error;; | |
846 | } } | |
9f95a23c | 847 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 848 | |
9f95a23c | 849 | DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++); |
11fdf7f2 TL |
850 | { ZSTD_parameters params = ZSTD_getParams(1, CNBuffSize, 0); |
851 | params.fParams.checksumFlag = 1; | |
852 | params.fParams.contentSizeFlag = 1; | |
853 | CHECKPLUS(r, ZSTDMT_compress_advanced(mtctx, | |
854 | compressedBuffer, compressedBufferSize, | |
855 | CNBuffer, CNBuffSize, | |
856 | NULL, params, 3 /*overlapRLog*/), | |
857 | cSize=r ); | |
858 | } | |
9f95a23c | 859 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
11fdf7f2 | 860 | |
9f95a23c | 861 | DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); |
11fdf7f2 TL |
862 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
863 | if (r != CNBuffSize) goto _output_error; } | |
9f95a23c | 864 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 TL |
865 | |
866 | ZSTDMT_freeCCtx(mtctx); | |
867 | } | |
868 | ||
9f95a23c TL |
869 | DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++) |
870 | { ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
871 | size_t cSize1, cSize2; | |
872 | CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); | |
873 | CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) ); | |
874 | cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
875 | CHECK(cSize1); | |
876 | CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_lcm_uncompressed) ); | |
877 | cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); | |
878 | CHECK(cSize2); | |
879 | CHECK_LT(cSize1, cSize2); | |
880 | ZSTD_freeCCtx(cctx); | |
881 | } | |
882 | DISPLAYLEVEL(3, "OK \n"); | |
883 | ||
884 | DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++) | |
885 | { ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
886 | /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so | |
887 | * ZSTDMT is forced to not take the shortcut. | |
888 | */ | |
889 | CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); | |
890 | CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) ); | |
891 | CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) ); | |
892 | CHECK( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) ); | |
893 | ZSTD_freeCCtx(cctx); | |
894 | } | |
895 | DISPLAYLEVEL(3, "OK \n"); | |
896 | ||
897 | DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++) | |
898 | { ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); | |
899 | int value; | |
900 | /* Check that the overlap log and job size are unset. */ | |
901 | CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); | |
902 | CHECK_EQ(value, 0); | |
903 | CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); | |
904 | CHECK_EQ(value, 0); | |
905 | /* Set and check the overlap log and job size. */ | |
906 | CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) ); | |
907 | CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, 2 MB) ); | |
908 | CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); | |
909 | CHECK_EQ(value, 5); | |
910 | CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); | |
911 | CHECK_EQ(value, 2 MB); | |
912 | /* Set the number of workers and check the overlap log and job size. */ | |
913 | CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) ); | |
914 | CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); | |
915 | CHECK_EQ(value, 5); | |
916 | CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); | |
917 | CHECK_EQ(value, 2 MB); | |
918 | ZSTD_freeCCtxParams(params); | |
919 | ||
920 | } | |
921 | DISPLAYLEVEL(3, "OK \n"); | |
11fdf7f2 TL |
922 | |
923 | /* Simple API multiframe test */ | |
9f95a23c | 924 | DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++); |
11fdf7f2 TL |
925 | { size_t off = 0; |
926 | int i; | |
927 | int const segs = 4; | |
928 | /* only use the first half so we don't push against size limit of compressedBuffer */ | |
929 | size_t const segSize = (CNBuffSize / 2) / segs; | |
930 | for (i = 0; i < segs; i++) { | |
931 | CHECK_V(r, ZSTD_compress( | |
932 | (BYTE *)compressedBuffer + off, CNBuffSize - off, | |
933 | (BYTE *)CNBuffer + segSize * i, | |
934 | segSize, 5)); | |
935 | off += r; | |
936 | if (i == segs/2) { | |
937 | /* insert skippable frame */ | |
9f95a23c | 938 | const U32 skipLen = 129 KB; |
11fdf7f2 TL |
939 | MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START); |
940 | MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen); | |
9f95a23c | 941 | off += skipLen + ZSTD_SKIPPABLEHEADERSIZE; |
11fdf7f2 TL |
942 | } |
943 | } | |
944 | cSize = off; | |
945 | } | |
9f95a23c | 946 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 947 | |
9f95a23c | 948 | DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++); |
11fdf7f2 TL |
949 | { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize); |
950 | if (r != CNBuffSize / 2) goto _output_error; } | |
9f95a23c TL |
951 | DISPLAYLEVEL(3, "OK \n"); |
952 | ||
953 | DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++); | |
954 | { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize); | |
955 | if (bound != CNBuffSize / 2) goto _output_error; } | |
956 | DISPLAYLEVEL(3, "OK \n"); | |
11fdf7f2 | 957 | |
9f95a23c | 958 | DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++); |
11fdf7f2 TL |
959 | { CHECK_V(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize)); |
960 | if (r != CNBuffSize / 2) goto _output_error; } | |
9f95a23c | 961 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 962 | |
9f95a23c | 963 | DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); |
11fdf7f2 | 964 | if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error; |
9f95a23c | 965 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 966 | |
7c673cae FG |
967 | /* Dictionary and CCtx Duplication tests */ |
968 | { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx(); | |
969 | ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx(); | |
9f95a23c | 970 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
7c673cae | 971 | static const size_t dictSize = 551; |
9f95a23c | 972 | assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL); |
7c673cae | 973 | |
9f95a23c | 974 | DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++); |
7c673cae FG |
975 | { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0); |
976 | if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */ | |
9f95a23c | 977 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 978 | |
9f95a23c | 979 | DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++); |
7c673cae | 980 | CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) ); |
11fdf7f2 | 981 | CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */ |
9f95a23c | 982 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 983 | |
9f95a23c | 984 | DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++); |
7c673cae | 985 | cSize = 0; |
11fdf7f2 | 986 | CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, compressedBuffer, compressedBufferSize, |
7c673cae FG |
987 | (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), |
988 | cSize += r); | |
9f95a23c | 989 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
7c673cae | 990 | |
9f95a23c | 991 | DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++); |
7c673cae FG |
992 | CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, |
993 | decodedBuffer, CNBuffSize, | |
994 | compressedBuffer, cSize, | |
995 | CNBuffer, dictSize), | |
996 | if (r != CNBuffSize - dictSize) goto _output_error); | |
9f95a23c | 997 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 998 | |
9f95a23c | 999 | DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++); |
7c673cae FG |
1000 | { size_t const cSizeOrig = cSize; |
1001 | cSize = 0; | |
11fdf7f2 | 1002 | CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, compressedBufferSize, |
7c673cae FG |
1003 | (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), |
1004 | cSize += r); | |
1005 | if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */ | |
1006 | } | |
9f95a23c | 1007 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
7c673cae | 1008 | |
9f95a23c | 1009 | DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); |
7c673cae FG |
1010 | CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, |
1011 | decodedBuffer, CNBuffSize, | |
1012 | compressedBuffer, cSize, | |
1013 | CNBuffer, dictSize), | |
1014 | if (r != CNBuffSize - dictSize) goto _output_error); | |
9f95a23c | 1015 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 1016 | |
9f95a23c | 1017 | DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++); |
11fdf7f2 TL |
1018 | { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize); |
1019 | size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); | |
1020 | if (r != CNBuffSize - dictSize) goto _output_error; | |
9f95a23c | 1021 | DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict)); |
11fdf7f2 TL |
1022 | ZSTD_freeDDict(ddict); |
1023 | } | |
1024 | ||
9f95a23c | 1025 | DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++); |
11fdf7f2 TL |
1026 | { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); |
1027 | void* ddictBuffer = malloc(ddictBufferSize); | |
1028 | if (ddictBuffer == NULL) goto _output_error; | |
9f95a23c | 1029 | { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); |
11fdf7f2 TL |
1030 | size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); |
1031 | if (r != CNBuffSize - dictSize) goto _output_error; | |
1032 | } | |
1033 | free(ddictBuffer); | |
9f95a23c | 1034 | DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize); |
11fdf7f2 TL |
1035 | } |
1036 | ||
9f95a23c | 1037 | DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++); |
7c673cae FG |
1038 | { size_t const testSize = CNBuffSize / 3; |
1039 | { ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize); | |
1040 | p.fParams.contentSizeFlag = 1; | |
1041 | CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) ); | |
1042 | } | |
1043 | CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) ); | |
1044 | ||
1045 | CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), | |
1046 | (const char*)CNBuffer + dictSize, testSize), | |
1047 | cSize = r); | |
11fdf7f2 TL |
1048 | { ZSTD_frameHeader zfh; |
1049 | if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error; | |
1050 | if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error; | |
7c673cae | 1051 | } } |
9f95a23c | 1052 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae FG |
1053 | |
1054 | ZSTD_freeCCtx(ctxOrig); | |
1055 | ZSTD_freeCCtx(ctxDuplicated); | |
9f95a23c | 1056 | ZSTD_freeDCtx(dctx); |
7c673cae FG |
1057 | } |
1058 | ||
1059 | /* Dictionary and dictBuilder tests */ | |
1060 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
9f95a23c TL |
1061 | size_t const dictBufferCapacity = 16 KB; |
1062 | void* dictBuffer = malloc(dictBufferCapacity); | |
7c673cae FG |
1063 | size_t const totalSampleSize = 1 MB; |
1064 | size_t const sampleUnitSize = 8 KB; | |
1065 | U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); | |
1066 | size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); | |
9f95a23c | 1067 | size_t dictSize; |
7c673cae FG |
1068 | U32 dictID; |
1069 | ||
1070 | if (dictBuffer==NULL || samplesSizes==NULL) { | |
1071 | free(dictBuffer); | |
1072 | free(samplesSizes); | |
1073 | goto _output_error; | |
1074 | } | |
1075 | ||
9f95a23c TL |
1076 | DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++); |
1077 | assert(compressedBufferSize >= totalSampleSize); | |
1078 | { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; } | |
7c673cae | 1079 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } |
9f95a23c TL |
1080 | { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, |
1081 | decodedBuffer, samplesSizes, nbSamples); | |
1082 | if (ZDICT_isError(sDictSize)) goto _output_error; | |
1083 | DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize); | |
1084 | } | |
1085 | ||
1086 | DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++); | |
1087 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
1088 | dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, | |
7c673cae FG |
1089 | CNBuffer, samplesSizes, nbSamples); |
1090 | if (ZDICT_isError(dictSize)) goto _output_error; | |
9f95a23c TL |
1091 | DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); |
1092 | ||
1093 | DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++); | |
1094 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
1095 | { ZDICT_cover_params_t coverParams; | |
1096 | memset(&coverParams, 0, sizeof(coverParams)); | |
1097 | coverParams.steps = 8; | |
1098 | coverParams.nbThreads = 4; | |
1099 | dictSize = ZDICT_optimizeTrainFromBuffer_cover( | |
1100 | dictBuffer, dictBufferCapacity, | |
1101 | CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */ | |
1102 | &coverParams); | |
1103 | if (ZDICT_isError(dictSize)) goto _output_error; | |
1104 | } | |
1105 | DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); | |
1106 | ||
1107 | DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++); | |
1108 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
1109 | { ZDICT_fastCover_params_t fastCoverParams; | |
1110 | memset(&fastCoverParams, 0, sizeof(fastCoverParams)); | |
1111 | fastCoverParams.steps = 8; | |
1112 | fastCoverParams.nbThreads = 4; | |
1113 | dictSize = ZDICT_optimizeTrainFromBuffer_fastCover( | |
1114 | dictBuffer, dictBufferCapacity, | |
1115 | CNBuffer, samplesSizes, nbSamples, | |
1116 | &fastCoverParams); | |
1117 | if (ZDICT_isError(dictSize)) goto _output_error; | |
1118 | } | |
1119 | DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); | |
7c673cae | 1120 | |
9f95a23c | 1121 | DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); |
7c673cae FG |
1122 | dictID = ZDICT_getDictID(dictBuffer, dictSize); |
1123 | if (dictID==0) goto _output_error; | |
9f95a23c | 1124 | DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); |
7c673cae | 1125 | |
9f95a23c | 1126 | DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++); |
11fdf7f2 | 1127 | cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, |
7c673cae FG |
1128 | CNBuffer, CNBuffSize, |
1129 | dictBuffer, dictSize, 4); | |
1130 | if (ZSTD_isError(cSize)) goto _output_error; | |
9f95a23c | 1131 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
7c673cae | 1132 | |
9f95a23c | 1133 | DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++); |
7c673cae FG |
1134 | { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize); |
1135 | if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ | |
1136 | } | |
9f95a23c | 1137 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 1138 | |
9f95a23c | 1139 | DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++); |
7c673cae FG |
1140 | { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); |
1141 | if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ | |
1142 | } | |
9f95a23c | 1143 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 1144 | |
9f95a23c TL |
1145 | DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); |
1146 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); | |
1147 | CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, | |
1148 | decodedBuffer, CNBuffSize, | |
1149 | compressedBuffer, cSize, | |
1150 | dictBuffer, dictSize), | |
1151 | if (r != CNBuffSize) goto _output_error); | |
1152 | ZSTD_freeDCtx(dctx); | |
1153 | } | |
1154 | DISPLAYLEVEL(3, "OK \n"); | |
7c673cae | 1155 | |
9f95a23c | 1156 | DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++); |
11fdf7f2 TL |
1157 | { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); |
1158 | size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef); | |
9f95a23c | 1159 | DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize); |
11fdf7f2 TL |
1160 | } |
1161 | ||
9f95a23c | 1162 | DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++); |
11fdf7f2 TL |
1163 | { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); |
1164 | ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, | |
9f95a23c | 1165 | ZSTD_dlm_byRef, ZSTD_dct_auto, |
11fdf7f2 | 1166 | cParams, ZSTD_defaultCMem); |
9f95a23c | 1167 | DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict)); |
11fdf7f2 TL |
1168 | cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, |
1169 | CNBuffer, CNBuffSize, cdict); | |
1170 | ZSTD_freeCDict(cdict); | |
1171 | if (ZSTD_isError(cSize)) goto _output_error; | |
1172 | } | |
9f95a23c | 1173 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
11fdf7f2 | 1174 | |
9f95a23c | 1175 | DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++); |
11fdf7f2 TL |
1176 | { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); |
1177 | if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ | |
1178 | } | |
9f95a23c | 1179 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 1180 | |
9f95a23c TL |
1181 | DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); |
1182 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); | |
1183 | CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, | |
1184 | decodedBuffer, CNBuffSize, | |
1185 | compressedBuffer, cSize, | |
1186 | dictBuffer, dictSize), | |
1187 | if (r != CNBuffSize) goto _output_error); | |
1188 | ZSTD_freeDCtx(dctx); | |
11fdf7f2 | 1189 | } |
9f95a23c TL |
1190 | DISPLAYLEVEL(3, "OK \n"); |
1191 | ||
1192 | DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++); | |
1193 | { int const maxLevel = ZSTD_maxCLevel(); | |
1194 | int level; | |
1195 | for (level = 1; level <= maxLevel; ++level) { | |
1196 | ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, CNBuffSize, dictSize); | |
1197 | size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); | |
1198 | void* const cdictBuffer = malloc(cdictSize); | |
1199 | if (cdictBuffer==NULL) goto _output_error; | |
1200 | { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict( | |
1201 | cdictBuffer, cdictSize, | |
1202 | dictBuffer, dictSize, | |
1203 | ZSTD_dlm_byCopy, ZSTD_dct_auto, | |
1204 | cParams); | |
1205 | if (cdict == NULL) { | |
1206 | DISPLAY("ZSTD_initStaticCDict failed "); | |
1207 | goto _output_error; | |
1208 | } | |
1209 | cSize = ZSTD_compress_usingCDict(cctx, | |
1210 | compressedBuffer, compressedBufferSize, | |
1211 | CNBuffer, MIN(10 KB, CNBuffSize), cdict); | |
1212 | if (ZSTD_isError(cSize)) { | |
1213 | DISPLAY("ZSTD_compress_usingCDict failed "); | |
1214 | goto _output_error; | |
1215 | } } | |
1216 | free(cdictBuffer); | |
1217 | } } | |
1218 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); | |
11fdf7f2 | 1219 | |
9f95a23c | 1220 | DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++); |
11fdf7f2 TL |
1221 | { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ }; |
1222 | ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); | |
9f95a23c | 1223 | ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); |
11fdf7f2 TL |
1224 | cSize = ZSTD_compress_usingCDict_advanced(cctx, compressedBuffer, compressedBufferSize, |
1225 | CNBuffer, CNBuffSize, cdict, fParams); | |
1226 | ZSTD_freeCDict(cdict); | |
1227 | if (ZSTD_isError(cSize)) goto _output_error; | |
1228 | } | |
9f95a23c | 1229 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
11fdf7f2 | 1230 | |
9f95a23c | 1231 | DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++); |
11fdf7f2 TL |
1232 | { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); |
1233 | if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; | |
1234 | } | |
9f95a23c | 1235 | DISPLAYLEVEL(3, "OK (unknown)\n"); |
11fdf7f2 | 1236 | |
9f95a23c TL |
1237 | DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); |
1238 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); | |
1239 | CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, | |
1240 | decodedBuffer, CNBuffSize, | |
1241 | compressedBuffer, cSize, | |
1242 | dictBuffer, dictSize), | |
1243 | if (r != CNBuffSize) goto _output_error); | |
1244 | ZSTD_freeDCtx(dctx); | |
1245 | } | |
1246 | DISPLAYLEVEL(3, "OK \n"); | |
11fdf7f2 | 1247 | |
9f95a23c | 1248 | DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++); |
7c673cae FG |
1249 | { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize); |
1250 | p.fParams.noDictIDFlag = 1; | |
11fdf7f2 | 1251 | cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, |
7c673cae FG |
1252 | CNBuffer, CNBuffSize, |
1253 | dictBuffer, dictSize, p); | |
1254 | if (ZSTD_isError(cSize)) goto _output_error; | |
1255 | } | |
9f95a23c | 1256 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); |
7c673cae | 1257 | |
9f95a23c TL |
1258 | DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); |
1259 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); | |
1260 | CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, | |
1261 | decodedBuffer, CNBuffSize, | |
1262 | compressedBuffer, cSize, | |
1263 | dictBuffer, dictSize), | |
1264 | if (r != CNBuffSize) goto _output_error); | |
1265 | ZSTD_freeDCtx(dctx); | |
1266 | } | |
1267 | DISPLAYLEVEL(3, "OK \n"); | |
1268 | ||
1269 | DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++); | |
1270 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
1271 | assert(dctx != NULL); | |
1272 | { const size_t ret = ZSTD_decompress_usingDict( | |
1273 | dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, | |
1274 | "\x37\xa4\x30\xec\x11\x22\x33\x44", 8); | |
1275 | if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) | |
1276 | goto _output_error; | |
1277 | } | |
1278 | ZSTD_freeDCtx(dctx); | |
11fdf7f2 | 1279 | } |
9f95a23c | 1280 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 1281 | |
9f95a23c | 1282 | DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a good dictionary : ", testNb++); |
11fdf7f2 | 1283 | { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); |
9f95a23c | 1284 | ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); |
11fdf7f2 TL |
1285 | if (cdict==NULL) goto _output_error; |
1286 | ZSTD_freeCDict(cdict); | |
1287 | } | |
9f95a23c | 1288 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 1289 | |
9f95a23c | 1290 | DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dm_fullDict on a rawContent (must fail) : ", testNb++); |
11fdf7f2 | 1291 | { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); |
9f95a23c | 1292 | ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); |
11fdf7f2 TL |
1293 | if (cdict!=NULL) goto _output_error; |
1294 | ZSTD_freeCDict(cdict); | |
1295 | } | |
9f95a23c | 1296 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 1297 | |
9f95a23c | 1298 | DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_auto should fail : ", testNb++); |
11fdf7f2 TL |
1299 | { |
1300 | size_t ret; | |
1301 | MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); | |
9f95a23c | 1302 | /* Either operation is allowed to fail, but one must fail. */ |
11fdf7f2 | 1303 | ret = ZSTD_CCtx_loadDictionary_advanced( |
9f95a23c TL |
1304 | cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dct_auto); |
1305 | if (!ZSTD_isError(ret)) { | |
1306 | ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); | |
1307 | if (!ZSTD_isError(ret)) goto _output_error; | |
1308 | } | |
11fdf7f2 | 1309 | } |
9f95a23c | 1310 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 | 1311 | |
9f95a23c | 1312 | DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dm_rawContent should pass : ", testNb++); |
11fdf7f2 TL |
1313 | { |
1314 | size_t ret; | |
1315 | MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); | |
1316 | ret = ZSTD_CCtx_loadDictionary_advanced( | |
9f95a23c TL |
1317 | cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dct_rawContent); |
1318 | if (ZSTD_isError(ret)) goto _output_error; | |
1319 | ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); | |
11fdf7f2 TL |
1320 | if (ZSTD_isError(ret)) goto _output_error; |
1321 | } | |
9f95a23c TL |
1322 | DISPLAYLEVEL(3, "OK \n"); |
1323 | ||
1324 | DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++); | |
1325 | { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1); | |
1326 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); | |
1327 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); | |
1328 | CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); | |
1329 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); | |
1330 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); | |
1331 | ZSTD_freeCDict(cdict); | |
1332 | } | |
1333 | DISPLAYLEVEL(3, "OK \n"); | |
1334 | ||
1335 | DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++); | |
1336 | { | |
1337 | size_t size1, size2; | |
1338 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
1339 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) ); | |
1340 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); | |
1341 | size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); | |
1342 | if (ZSTD_isError(size1)) goto _output_error; | |
1343 | ||
1344 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
1345 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); | |
1346 | CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) ); | |
1347 | size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); | |
1348 | if (ZSTD_isError(size2)) goto _output_error; | |
1349 | ||
1350 | if (size1 != size2) goto _output_error; | |
1351 | } | |
1352 | DISPLAYLEVEL(3, "OK \n"); | |
1353 | ||
1354 | DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++); | |
1355 | { | |
1356 | CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); | |
1357 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); | |
1358 | CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); | |
1359 | } | |
1360 | DISPLAYLEVEL(3, "OK \n"); | |
1361 | ||
1362 | DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++); | |
1363 | { | |
1364 | ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); | |
1365 | CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); | |
1366 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); | |
1367 | CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); | |
1368 | ZSTD_freeCDict(cdict); | |
1369 | } | |
1370 | DISPLAYLEVEL(3, "OK \n"); | |
1371 | ||
1372 | DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++); | |
1373 | { | |
1374 | ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); | |
1375 | CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); | |
1376 | CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); | |
1377 | CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); | |
1378 | ZSTD_freeCDict(cdict); | |
1379 | } | |
1380 | DISPLAYLEVEL(3, "OK \n"); | |
1381 | ||
1382 | DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++); | |
1383 | { | |
1384 | ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); | |
1385 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); | |
1386 | CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); | |
1387 | CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); | |
1388 | ZSTD_freeCDict(cdict); | |
1389 | } | |
1390 | DISPLAYLEVEL(3, "OK \n"); | |
1391 | ||
1392 | DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++); | |
1393 | { | |
1394 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); | |
1395 | CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); | |
1396 | CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); | |
1397 | } | |
1398 | DISPLAYLEVEL(3, "OK \n"); | |
1399 | ||
1400 | DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++); | |
1401 | { | |
1402 | ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); | |
1403 | CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); | |
1404 | CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); | |
1405 | CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); | |
1406 | ZSTD_freeCDict(cdict); | |
1407 | } | |
1408 | DISPLAYLEVEL(3, "OK \n"); | |
1409 | ||
1410 | DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++); | |
1411 | { | |
1412 | size_t size1, size2; | |
1413 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
1414 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); | |
1415 | size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); | |
1416 | if (ZSTD_isError(size1)) goto _output_error; | |
1417 | ||
1418 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); | |
1419 | size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); | |
1420 | if (ZSTD_isError(size2)) goto _output_error; | |
1421 | ||
1422 | if (size1 != size2) goto _output_error; | |
1423 | } | |
1424 | DISPLAYLEVEL(3, "OK \n"); | |
1425 | ||
1426 | DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++); | |
1427 | { | |
1428 | size_t size1, size2; | |
1429 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
1430 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); | |
1431 | size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); | |
1432 | if (ZSTD_isError(size1)) goto _output_error; | |
1433 | ||
1434 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
1435 | size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); | |
1436 | if (ZSTD_isError(size2)) goto _output_error; | |
1437 | ||
1438 | if (size1 == size2) goto _output_error; | |
1439 | } | |
1440 | DISPLAYLEVEL(3, "OK \n"); | |
1441 | ||
1442 | ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); | |
1443 | CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) ); | |
1444 | cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); | |
1445 | CHECK_Z(cSize); | |
1446 | DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++); | |
1447 | { | |
1448 | ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
1449 | size_t ret; | |
1450 | /* We should fail to decompress without a dictionary. */ | |
1451 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
1452 | ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
1453 | if (!ZSTD_isError(ret)) goto _output_error; | |
1454 | /* We should succeed to decompress with the dictionary. */ | |
1455 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
1456 | CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) ); | |
1457 | CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); | |
1458 | /* The dictionary should presist across calls. */ | |
1459 | CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); | |
1460 | /* When we reset the context the dictionary is cleared. */ | |
1461 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
1462 | ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
1463 | if (!ZSTD_isError(ret)) goto _output_error; | |
1464 | ZSTD_freeDCtx(dctx); | |
1465 | } | |
1466 | DISPLAYLEVEL(3, "OK \n"); | |
1467 | ||
1468 | DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++); | |
1469 | { | |
1470 | ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
1471 | ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize); | |
1472 | size_t ret; | |
1473 | /* We should succeed to decompress with the ddict. */ | |
1474 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
1475 | CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) ); | |
1476 | CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); | |
1477 | /* The ddict should presist across calls. */ | |
1478 | CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); | |
1479 | /* When we reset the context the ddict is cleared. */ | |
1480 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
1481 | ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
1482 | if (!ZSTD_isError(ret)) goto _output_error; | |
1483 | ZSTD_freeDCtx(dctx); | |
1484 | ZSTD_freeDDict(ddict); | |
1485 | } | |
1486 | DISPLAYLEVEL(3, "OK \n"); | |
1487 | ||
1488 | DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++); | |
1489 | { | |
1490 | ZSTD_DCtx* dctx = ZSTD_createDCtx(); | |
1491 | size_t ret; | |
1492 | /* We should succeed to decompress with the prefix. */ | |
1493 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
1494 | CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) ); | |
1495 | CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); | |
1496 | /* The prefix should be cleared after the first compression. */ | |
1497 | ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
1498 | if (!ZSTD_isError(ret)) goto _output_error; | |
1499 | ZSTD_freeDCtx(dctx); | |
1500 | } | |
1501 | DISPLAYLEVEL(3, "OK \n"); | |
1502 | ||
1503 | DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++); | |
1504 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
1505 | dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, | |
1506 | CNBuffer, samplesSizes, nbSamples); | |
1507 | if (ZDICT_isError(dictSize)) goto _output_error; | |
1508 | /* Set all the repcodes to non-default */ | |
1509 | { | |
1510 | BYTE* dictPtr = (BYTE*)dictBuffer; | |
1511 | BYTE* dictLimit = dictPtr + dictSize - 12; | |
1512 | /* Find the repcodes */ | |
1513 | while (dictPtr < dictLimit && | |
1514 | (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 || | |
1515 | MEM_readLE32(dictPtr + 8) != 8)) { | |
1516 | ++dictPtr; | |
1517 | } | |
1518 | if (dictPtr >= dictLimit) goto _output_error; | |
1519 | MEM_writeLE32(dictPtr + 0, 10); | |
1520 | MEM_writeLE32(dictPtr + 4, 10); | |
1521 | MEM_writeLE32(dictPtr + 8, 10); | |
1522 | /* Set the last 8 bytes to 'x' */ | |
1523 | memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8); | |
1524 | } | |
1525 | /* The optimal parser checks all the repcodes. | |
1526 | * Make sure at least one is a match >= targetLength so that it is | |
1527 | * immediately chosen. This will make sure that the compressor and | |
1528 | * decompressor agree on at least one of the repcodes. | |
1529 | */ | |
1530 | { size_t dSize; | |
1531 | BYTE data[1024]; | |
1532 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
1533 | ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize); | |
1534 | ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, | |
1535 | ZSTD_dlm_byRef, ZSTD_dct_auto, | |
1536 | cParams, ZSTD_defaultCMem); | |
1537 | assert(dctx != NULL); assert(cdict != NULL); | |
1538 | memset(data, 'x', sizeof(data)); | |
1539 | cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, | |
1540 | data, sizeof(data), cdict); | |
1541 | ZSTD_freeCDict(cdict); | |
1542 | if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; } | |
1543 | dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize); | |
1544 | if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; } | |
1545 | if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; } | |
1546 | ZSTD_freeDCtx(dctx); | |
1547 | } | |
1548 | DISPLAYLEVEL(3, "OK \n"); | |
11fdf7f2 TL |
1549 | |
1550 | ZSTD_freeCCtx(cctx); | |
1551 | free(dictBuffer); | |
1552 | free(samplesSizes); | |
1553 | } | |
1554 | ||
1555 | /* COVER dictionary builder tests */ | |
1556 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
1557 | size_t dictSize = 16 KB; | |
1558 | size_t optDictSize = dictSize; | |
1559 | void* dictBuffer = malloc(dictSize); | |
1560 | size_t const totalSampleSize = 1 MB; | |
1561 | size_t const sampleUnitSize = 8 KB; | |
1562 | U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); | |
1563 | size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); | |
1564 | ZDICT_cover_params_t params; | |
1565 | U32 dictID; | |
1566 | ||
1567 | if (dictBuffer==NULL || samplesSizes==NULL) { | |
1568 | free(dictBuffer); | |
1569 | free(samplesSizes); | |
1570 | goto _output_error; | |
1571 | } | |
1572 | ||
9f95a23c | 1573 | DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++); |
11fdf7f2 TL |
1574 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } |
1575 | memset(¶ms, 0, sizeof(params)); | |
1576 | params.d = 1 + (FUZ_rand(&seed) % 16); | |
1577 | params.k = params.d + (FUZ_rand(&seed) % 256); | |
1578 | dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize, | |
1579 | CNBuffer, samplesSizes, nbSamples, | |
1580 | params); | |
1581 | if (ZDICT_isError(dictSize)) goto _output_error; | |
9f95a23c | 1582 | DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); |
11fdf7f2 | 1583 | |
9f95a23c | 1584 | DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); |
11fdf7f2 TL |
1585 | dictID = ZDICT_getDictID(dictBuffer, dictSize); |
1586 | if (dictID==0) goto _output_error; | |
9f95a23c | 1587 | DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); |
11fdf7f2 | 1588 | |
9f95a23c | 1589 | DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++); |
11fdf7f2 TL |
1590 | memset(¶ms, 0, sizeof(params)); |
1591 | params.steps = 4; | |
1592 | optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize, | |
1593 | CNBuffer, samplesSizes, | |
1594 | nbSamples / 4, ¶ms); | |
1595 | if (ZDICT_isError(optDictSize)) goto _output_error; | |
9f95a23c | 1596 | DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize); |
11fdf7f2 | 1597 | |
9f95a23c | 1598 | DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); |
11fdf7f2 TL |
1599 | dictID = ZDICT_getDictID(dictBuffer, optDictSize); |
1600 | if (dictID==0) goto _output_error; | |
9f95a23c | 1601 | DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); |
11fdf7f2 | 1602 | |
7c673cae | 1603 | ZSTD_freeCCtx(cctx); |
7c673cae FG |
1604 | free(dictBuffer); |
1605 | free(samplesSizes); | |
1606 | } | |
1607 | ||
1608 | /* Decompression defense tests */ | |
9f95a23c | 1609 | DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++); |
11fdf7f2 | 1610 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */ |
7c673cae | 1611 | if (!ZSTD_isError(r)) goto _output_error; |
11fdf7f2 | 1612 | if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } |
9f95a23c | 1613 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 1614 | |
9f95a23c | 1615 | DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++); |
7c673cae FG |
1616 | ((char*)(CNBuffer))[0] = 1; |
1617 | { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4); | |
1618 | if (!ZSTD_isError(r)) goto _output_error; } | |
9f95a23c | 1619 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 1620 | |
11fdf7f2 | 1621 | /* content size verification test */ |
9f95a23c | 1622 | DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++); |
11fdf7f2 TL |
1623 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1624 | size_t const srcSize = 5000; | |
1625 | size_t const wrongSrcSize = (srcSize + 1000); | |
1626 | ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0); | |
1627 | params.fParams.contentSizeFlag = 1; | |
1628 | CHECK( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) ); | |
1629 | { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize); | |
1630 | if (!ZSTD_isError(result)) goto _output_error; | |
1631 | if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error; | |
9f95a23c | 1632 | DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result)); |
11fdf7f2 TL |
1633 | } |
1634 | ZSTD_freeCCtx(cctx); | |
1635 | } | |
1636 | ||
9f95a23c TL |
1637 | /* negative compression level test : ensure simple API and advanced API produce same result */ |
1638 | DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++); | |
1639 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
1640 | size_t const srcSize = CNBuffSize / 5; | |
1641 | int const compressionLevel = -1; | |
1642 | ||
1643 | assert(cctx != NULL); | |
1644 | { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, srcSize, 0); | |
1645 | size_t const cSize_1pass = ZSTD_compress_advanced(cctx, | |
1646 | compressedBuffer, compressedBufferSize, | |
1647 | CNBuffer, srcSize, | |
1648 | NULL, 0, | |
1649 | params); | |
1650 | if (ZSTD_isError(cSize_1pass)) goto _output_error; | |
1651 | ||
1652 | CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); | |
1653 | { size_t const compressionResult = ZSTD_compress2(cctx, | |
1654 | compressedBuffer, compressedBufferSize, | |
1655 | CNBuffer, srcSize); | |
1656 | DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult); | |
1657 | if (ZSTD_isError(compressionResult)) goto _output_error; | |
1658 | if (compressionResult != cSize_1pass) goto _output_error; | |
1659 | } } | |
1660 | ZSTD_freeCCtx(cctx); | |
1661 | } | |
1662 | DISPLAYLEVEL(3, "OK \n"); | |
1663 | ||
1664 | /* parameters order test */ | |
1665 | { size_t const inputSize = CNBuffSize / 2; | |
1666 | U64 xxh64; | |
1667 | ||
1668 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
1669 | DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++); | |
1670 | assert(cctx != NULL); | |
1671 | CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); | |
1672 | CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) ); | |
1673 | CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); | |
1674 | { size_t const compressedSize = ZSTD_compress2(cctx, | |
1675 | compressedBuffer, ZSTD_compressBound(inputSize), | |
1676 | CNBuffer, inputSize); | |
1677 | CHECK(compressedSize); | |
1678 | cSize = compressedSize; | |
1679 | xxh64 = XXH64(compressedBuffer, compressedSize, 0); | |
1680 | } | |
1681 | DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize); | |
1682 | ZSTD_freeCCtx(cctx); | |
1683 | } | |
1684 | ||
1685 | { ZSTD_CCtx* cctx = ZSTD_createCCtx(); | |
1686 | DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++); | |
1687 | CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); | |
1688 | CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) ); | |
1689 | CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); | |
1690 | { size_t const result = ZSTD_compress2(cctx, | |
1691 | compressedBuffer, ZSTD_compressBound(inputSize), | |
1692 | CNBuffer, inputSize); | |
1693 | CHECK(result); | |
1694 | if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */ | |
1695 | if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */ | |
1696 | DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result); | |
1697 | } | |
1698 | ZSTD_freeCCtx(cctx); | |
1699 | } | |
1700 | } | |
1701 | ||
1702 | /* advanced parameters for decompression */ | |
1703 | { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
1704 | assert(dctx != NULL); | |
1705 | ||
1706 | DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++); | |
1707 | { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); | |
1708 | CHECK(bounds.error); | |
1709 | } | |
1710 | DISPLAYLEVEL(3, "OK \n"); | |
1711 | ||
1712 | DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++); | |
1713 | { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0); | |
1714 | if (!ZSTD_isError(sr)) goto _output_error; | |
1715 | } | |
1716 | { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998); | |
1717 | if (!ZSTD_isError(bounds.error)) goto _output_error; | |
1718 | } | |
1719 | DISPLAYLEVEL(3, "OK \n"); | |
1720 | ||
1721 | DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++); | |
1722 | { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999); | |
1723 | if (!ZSTD_isError(sr)) goto _output_error; | |
1724 | } | |
1725 | { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888); | |
1726 | if (!ZSTD_isError(sr)) goto _output_error; | |
1727 | } | |
1728 | DISPLAYLEVEL(3, "OK \n"); | |
1729 | ||
1730 | ZSTD_freeDCtx(dctx); | |
1731 | } | |
1732 | ||
1733 | ||
11fdf7f2 TL |
1734 | /* custom formats tests */ |
1735 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
9f95a23c | 1736 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
11fdf7f2 | 1737 | size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */ |
9f95a23c | 1738 | assert(dctx != NULL); assert(cctx != NULL); |
11fdf7f2 TL |
1739 | |
1740 | /* basic block compression */ | |
9f95a23c TL |
1741 | DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++); |
1742 | CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); | |
11fdf7f2 TL |
1743 | { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; |
1744 | ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; | |
9f95a23c | 1745 | size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); |
11fdf7f2 TL |
1746 | if (result != 0) goto _output_error; |
1747 | if (in.pos != in.size) goto _output_error; | |
1748 | cSize = out.pos; | |
1749 | } | |
9f95a23c | 1750 | DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize); |
11fdf7f2 | 1751 | |
9f95a23c | 1752 | DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++); |
11fdf7f2 TL |
1753 | { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); |
1754 | if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error; | |
9f95a23c | 1755 | DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult)); |
11fdf7f2 TL |
1756 | } |
1757 | ||
9f95a23c TL |
1758 | DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++); |
1759 | ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); | |
1760 | CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) ); | |
1761 | { ZSTD_frameHeader zfh; | |
1762 | size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless); | |
1763 | if (zfhrt != 0) goto _output_error; | |
1764 | } | |
1765 | /* one shot */ | |
1766 | { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); | |
1767 | if (result != inputSize) goto _output_error; | |
1768 | DISPLAYLEVEL(3, "one-shot OK, "); | |
1769 | } | |
1770 | /* streaming */ | |
11fdf7f2 TL |
1771 | { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; |
1772 | ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; | |
9f95a23c | 1773 | size_t const result = ZSTD_decompressStream(dctx, &out, &in); |
11fdf7f2 TL |
1774 | if (result != 0) goto _output_error; |
1775 | if (in.pos != in.size) goto _output_error; | |
1776 | if (out.pos != inputSize) goto _output_error; | |
9f95a23c | 1777 | DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos); |
11fdf7f2 TL |
1778 | } |
1779 | ||
1780 | ZSTD_freeCCtx(cctx); | |
9f95a23c | 1781 | ZSTD_freeDCtx(dctx); |
11fdf7f2 TL |
1782 | } |
1783 | ||
7c673cae FG |
1784 | /* block API tests */ |
1785 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
9f95a23c | 1786 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); |
7c673cae FG |
1787 | static const size_t dictSize = 65 KB; |
1788 | static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */ | |
1789 | size_t cSize2; | |
9f95a23c | 1790 | assert(cctx != NULL); assert(dctx != NULL); |
7c673cae FG |
1791 | |
1792 | /* basic block compression */ | |
9f95a23c | 1793 | DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++); |
7c673cae | 1794 | CHECK( ZSTD_compressBegin(cctx, 5) ); |
11fdf7f2 | 1795 | CHECK( ZSTD_getBlockSize(cctx) >= blockSize); |
7c673cae FG |
1796 | cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); |
1797 | if (ZSTD_isError(cSize)) goto _output_error; | |
9f95a23c | 1798 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 1799 | |
9f95a23c | 1800 | DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++); |
7c673cae FG |
1801 | CHECK( ZSTD_decompressBegin(dctx) ); |
1802 | { CHECK_V(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); | |
1803 | if (r != blockSize) goto _output_error; } | |
9f95a23c TL |
1804 | DISPLAYLEVEL(3, "OK \n"); |
1805 | ||
1806 | /* very long stream of block compression */ | |
1807 | DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++); | |
1808 | CHECK( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */ | |
1809 | CHECK( ZSTD_getBlockSize(cctx) >= blockSize); | |
1810 | { U64 const toCompress = 5000000000ULL; /* > 4 GB */ | |
1811 | U64 compressed = 0; | |
1812 | while (compressed < toCompress) { | |
1813 | size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); | |
1814 | assert(blockCSize != 0); | |
1815 | if (ZSTD_isError(blockCSize)) goto _output_error; | |
1816 | compressed += blockCSize; | |
1817 | } | |
1818 | } | |
1819 | DISPLAYLEVEL(3, "OK \n"); | |
7c673cae FG |
1820 | |
1821 | /* dictionary block compression */ | |
9f95a23c | 1822 | DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++); |
7c673cae FG |
1823 | CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) ); |
1824 | cSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize); | |
1825 | if (ZSTD_isError(cSize)) goto _output_error; | |
1826 | cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize); | |
1827 | if (ZSTD_isError(cSize2)) goto _output_error; | |
1828 | memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* fake non-compressed block */ | |
1829 | cSize2 = ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize), | |
1830 | (char*)CNBuffer+dictSize+2*blockSize, blockSize); | |
1831 | if (ZSTD_isError(cSize2)) goto _output_error; | |
9f95a23c | 1832 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 1833 | |
9f95a23c | 1834 | DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++); |
7c673cae FG |
1835 | CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) ); |
1836 | { CHECK_V( r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); | |
1837 | if (r != blockSize) goto _output_error; } | |
1838 | ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */ | |
1839 | { CHECK_V( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, CNBuffSize, (char*)compressedBuffer+cSize+blockSize, cSize2) ); | |
1840 | if (r != blockSize) goto _output_error; } | |
9f95a23c TL |
1841 | DISPLAYLEVEL(3, "OK \n"); |
1842 | ||
1843 | DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++); | |
1844 | { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3); | |
1845 | if (cdict==NULL) goto _output_error; | |
1846 | CHECK( ZSTD_compressBegin_usingCDict(cctx, cdict) ); | |
1847 | CHECK( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) ); | |
1848 | ZSTD_freeCDict(cdict); | |
1849 | } | |
1850 | DISPLAYLEVEL(3, "OK \n"); | |
7c673cae FG |
1851 | |
1852 | ZSTD_freeCCtx(cctx); | |
9f95a23c | 1853 | ZSTD_freeDCtx(dctx); |
7c673cae FG |
1854 | } |
1855 | ||
1856 | /* long rle test */ | |
1857 | { size_t sampleSize = 0; | |
9f95a23c | 1858 | DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++); |
7c673cae FG |
1859 | RDG_genBuffer(CNBuffer, sampleSize, compressibility, 0., seed+1); |
1860 | memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); | |
1861 | sampleSize += 256 KB - 1; | |
1862 | RDG_genBuffer((char*)CNBuffer+sampleSize, 96 KB, compressibility, 0., seed+2); | |
1863 | sampleSize += 96 KB; | |
1864 | cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); | |
1865 | if (ZSTD_isError(cSize)) goto _output_error; | |
1866 | { CHECK_V(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize)); | |
1867 | if (regenSize!=sampleSize) goto _output_error; } | |
9f95a23c | 1868 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae FG |
1869 | } |
1870 | ||
1871 | /* All zeroes test (test bug #137) */ | |
1872 | #define ZEROESLENGTH 100 | |
9f95a23c | 1873 | DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH); |
7c673cae FG |
1874 | memset(CNBuffer, 0, ZEROESLENGTH); |
1875 | { CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) ); | |
1876 | cSize = r; } | |
9f95a23c | 1877 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100); |
7c673cae | 1878 | |
9f95a23c | 1879 | DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH); |
7c673cae FG |
1880 | { CHECK_V(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) ); |
1881 | if (r != ZEROESLENGTH) goto _output_error; } | |
9f95a23c | 1882 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae FG |
1883 | |
1884 | /* nbSeq limit test */ | |
1885 | #define _3BYTESTESTLENGTH 131000 | |
1886 | #define NB3BYTESSEQLOG 9 | |
1887 | #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG) | |
1888 | #define NB3BYTESSEQMASK (NB3BYTESSEQ-1) | |
1889 | /* creates a buffer full of 3-bytes sequences */ | |
1890 | { BYTE _3BytesSeqs[NB3BYTESSEQ][3]; | |
1891 | U32 rSeed = 1; | |
1892 | ||
1893 | /* create batch of 3-bytes sequences */ | |
1894 | { int i; | |
1895 | for (i=0; i < NB3BYTESSEQ; i++) { | |
1896 | _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255); | |
1897 | _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255); | |
1898 | _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255); | |
1899 | } } | |
1900 | ||
1901 | /* randomly fills CNBuffer with prepared 3-bytes sequences */ | |
1902 | { int i; | |
1903 | for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */ | |
1904 | U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK; | |
1905 | ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0]; | |
1906 | ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; | |
1907 | ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; | |
1908 | } } } | |
9f95a23c TL |
1909 | DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb++); |
1910 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
1911 | size_t const maxNbSeq = _3BYTESTESTLENGTH / 3; | |
1912 | size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH); | |
1913 | size_t nbSeq = 1; | |
1914 | while (nbSeq <= maxNbSeq) { | |
1915 | CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19)); | |
1916 | /* Check every sequence for the first 100, then skip more rapidly. */ | |
1917 | if (nbSeq < 100) { | |
1918 | ++nbSeq; | |
1919 | } else { | |
1920 | nbSeq += (nbSeq >> 2); | |
1921 | } | |
1922 | } | |
1923 | ZSTD_freeCCtx(cctx); | |
1924 | } | |
1925 | DISPLAYLEVEL(3, "OK \n"); | |
1926 | ||
1927 | DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++); | |
7c673cae FG |
1928 | { CHECK_V(r, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), |
1929 | CNBuffer, _3BYTESTESTLENGTH, 19) ); | |
1930 | cSize = r; } | |
9f95a23c | 1931 | DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100); |
7c673cae | 1932 | |
9f95a23c | 1933 | DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++); |
7c673cae FG |
1934 | { CHECK_V(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) ); |
1935 | if (r != _3BYTESTESTLENGTH) goto _output_error; } | |
9f95a23c | 1936 | DISPLAYLEVEL(3, "OK \n"); |
7c673cae | 1937 | |
9f95a23c TL |
1938 | |
1939 | DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb++); | |
11fdf7f2 | 1940 | RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed); |
9f95a23c TL |
1941 | { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); |
1942 | size_t const bound = ZSTD_compressBound(CNBuffSize); | |
1943 | size_t size = 1; | |
1944 | while (size <= CNBuffSize) { | |
1945 | CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3)); | |
1946 | /* Check every size for the first 100, then skip more rapidly. */ | |
1947 | if (size < 100) { | |
1948 | ++size; | |
1949 | } else { | |
1950 | size += (size >> 2); | |
1951 | } | |
1952 | } | |
1953 | ZSTD_freeCCtx(cctx); | |
1954 | } | |
1955 | DISPLAYLEVEL(3, "OK \n"); | |
1956 | ||
1957 | DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++); | |
11fdf7f2 TL |
1958 | { /* Train a dictionary on low characters */ |
1959 | size_t dictSize = 16 KB; | |
1960 | void* const dictBuffer = malloc(dictSize); | |
1961 | size_t const totalSampleSize = 1 MB; | |
1962 | size_t const sampleUnitSize = 8 KB; | |
1963 | U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); | |
1964 | size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); | |
1965 | if (!dictBuffer || !samplesSizes) goto _output_error; | |
1966 | { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } | |
1967 | dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples); | |
1968 | if (ZDICT_isError(dictSize)) goto _output_error; | |
1969 | /* Reverse the characters to make the dictionary ill suited */ | |
1970 | { U32 u; | |
1971 | for (u = 0; u < CNBuffSize; ++u) { | |
1972 | ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u]; | |
1973 | } | |
1974 | } | |
1975 | { /* Compress the data */ | |
1976 | size_t const inputSize = 500; | |
1977 | size_t const outputSize = ZSTD_compressBound(inputSize); | |
1978 | void* const outputBuffer = malloc(outputSize); | |
1979 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
1980 | if (!outputBuffer || !cctx) goto _output_error; | |
1981 | CHECK(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1)); | |
1982 | free(outputBuffer); | |
1983 | ZSTD_freeCCtx(cctx); | |
1984 | } | |
1985 | ||
1986 | free(dictBuffer); | |
1987 | free(samplesSizes); | |
1988 | } | |
9f95a23c | 1989 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 TL |
1990 | |
1991 | ||
1992 | /* findFrameCompressedSize on skippable frames */ | |
9f95a23c | 1993 | DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb++); |
11fdf7f2 TL |
1994 | { const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde"; |
1995 | size_t const frameSrcSize = 13; | |
1996 | if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; } | |
9f95a23c | 1997 | DISPLAYLEVEL(3, "OK \n"); |
11fdf7f2 TL |
1998 | |
1999 | /* error string tests */ | |
9f95a23c | 2000 | DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++); |
11fdf7f2 TL |
2001 | if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error; |
2002 | if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error; | |
2003 | if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error; | |
2004 | if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error; | |
2005 | if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error; | |
2006 | if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error; | |
9f95a23c TL |
2007 | DISPLAYLEVEL(3, "OK \n"); |
2008 | ||
2009 | DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++); | |
2010 | RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); | |
2011 | { | |
2012 | size_t const size = MIN(128 KB, CNBuffSize); | |
2013 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
2014 | ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1); | |
2015 | ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1); | |
2016 | ZSTD_frameHeader lgHeader; | |
2017 | ZSTD_frameHeader smHeader; | |
2018 | ||
2019 | CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict)); | |
2020 | CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize)); | |
2021 | CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict)); | |
2022 | CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize)); | |
2023 | ||
2024 | if (lgHeader.windowSize != smHeader.windowSize) goto _output_error; | |
2025 | ||
2026 | ZSTD_freeCDict(smCDict); | |
2027 | ZSTD_freeCDict(lgCDict); | |
2028 | ZSTD_freeCCtx(cctx); | |
2029 | } | |
2030 | DISPLAYLEVEL(3, "OK \n"); | |
2031 | ||
2032 | DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++); | |
2033 | { | |
2034 | short norm[32]; | |
2035 | unsigned count[32]; | |
2036 | unsigned const tableLog = 5; | |
2037 | size_t const nbSeq = 32; | |
2038 | unsigned const maxSymbolValue = 31; | |
2039 | size_t i; | |
2040 | ||
2041 | for (i = 0; i < 32; ++i) | |
2042 | count[i] = 1; | |
2043 | /* Calling FSE_normalizeCount() on a uniform distribution should not | |
2044 | * cause a division by zero. | |
2045 | */ | |
2046 | FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue); | |
2047 | } | |
2048 | DISPLAYLEVEL(3, "OK \n"); | |
11fdf7f2 | 2049 | |
7c673cae FG |
2050 | _end: |
2051 | free(CNBuffer); | |
2052 | free(compressedBuffer); | |
2053 | free(decodedBuffer); | |
2054 | return testResult; | |
2055 | ||
2056 | _output_error: | |
2057 | testResult = 1; | |
2058 | DISPLAY("Error detected in Unit tests ! \n"); | |
2059 | goto _end; | |
2060 | } | |
2061 | ||
2062 | ||
2063 | static size_t findDiff(const void* buf1, const void* buf2, size_t max) | |
2064 | { | |
2065 | const BYTE* b1 = (const BYTE*)buf1; | |
2066 | const BYTE* b2 = (const BYTE*)buf2; | |
2067 | size_t u; | |
2068 | for (u=0; u<max; u++) { | |
2069 | if (b1[u] != b2[u]) break; | |
2070 | } | |
2071 | return u; | |
2072 | } | |
2073 | ||
2074 | ||
11fdf7f2 TL |
2075 | static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams) |
2076 | { | |
2077 | ZSTD_parameters params; | |
2078 | params.cParams = cParams; | |
2079 | params.fParams = fParams; | |
2080 | return params; | |
2081 | } | |
2082 | ||
7c673cae FG |
2083 | static size_t FUZ_rLogLength(U32* seed, U32 logLength) |
2084 | { | |
2085 | size_t const lengthMask = ((size_t)1 << logLength) - 1; | |
2086 | return (lengthMask+1) + (FUZ_rand(seed) & lengthMask); | |
2087 | } | |
2088 | ||
2089 | static size_t FUZ_randomLength(U32* seed, U32 maxLog) | |
2090 | { | |
2091 | U32 const logLength = FUZ_rand(seed) % maxLog; | |
2092 | return FUZ_rLogLength(seed, logLength); | |
2093 | } | |
2094 | ||
2095 | #undef CHECK | |
11fdf7f2 TL |
2096 | #define CHECK(cond, ...) { \ |
2097 | if (cond) { \ | |
2098 | DISPLAY("Error => "); \ | |
2099 | DISPLAY(__VA_ARGS__); \ | |
9f95a23c | 2100 | DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \ |
11fdf7f2 TL |
2101 | goto _output_error; \ |
2102 | } } | |
2103 | ||
2104 | #undef CHECK_Z | |
2105 | #define CHECK_Z(f) { \ | |
2106 | size_t const err = f; \ | |
2107 | if (ZSTD_isError(err)) { \ | |
2108 | DISPLAY("Error => %s : %s ", \ | |
2109 | #f, ZSTD_getErrorName(err)); \ | |
9f95a23c | 2110 | DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \ |
11fdf7f2 TL |
2111 | goto _output_error; \ |
2112 | } } | |
2113 | ||
2114 | ||
9f95a23c | 2115 | static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests) |
7c673cae FG |
2116 | { |
2117 | static const U32 maxSrcLog = 23; | |
2118 | static const U32 maxSampleLog = 22; | |
2119 | size_t const srcBufferSize = (size_t)1<<maxSrcLog; | |
2120 | size_t const dstBufferSize = (size_t)1<<maxSampleLog; | |
2121 | size_t const cBufferSize = ZSTD_compressBound(dstBufferSize); | |
2122 | BYTE* cNoiseBuffer[5]; | |
7c673cae FG |
2123 | BYTE* const cBuffer = (BYTE*) malloc (cBufferSize); |
2124 | BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize); | |
2125 | BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize); | |
2126 | ZSTD_CCtx* const refCtx = ZSTD_createCCtx(); | |
2127 | ZSTD_CCtx* const ctx = ZSTD_createCCtx(); | |
2128 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
2129 | U32 result = 0; | |
9f95a23c TL |
2130 | unsigned testNb = 0; |
2131 | U32 coreSeed = seed; | |
2132 | UTIL_time_t const startClock = UTIL_getTime(); | |
2133 | U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO; | |
11fdf7f2 | 2134 | int const cLevelLimiter = bigTests ? 3 : 2; |
7c673cae FG |
2135 | |
2136 | /* allocation */ | |
2137 | cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); | |
2138 | cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); | |
2139 | cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); | |
2140 | cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); | |
2141 | cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); | |
2142 | CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] | |
2143 | || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx, | |
2144 | "Not enough memory, fuzzer tests cancelled"); | |
2145 | ||
2146 | /* Create initial samples */ | |
2147 | RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ | |
2148 | RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ | |
2149 | RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); | |
2150 | RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ | |
2151 | RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ | |
7c673cae FG |
2152 | |
2153 | /* catch up testNb */ | |
2154 | for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed); | |
2155 | ||
2156 | /* main test loop */ | |
9f95a23c TL |
2157 | for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) { |
2158 | BYTE* srcBuffer; /* jumping pointer */ | |
2159 | U32 lseed; | |
7c673cae FG |
2160 | size_t sampleSize, maxTestSize, totalTestSize; |
2161 | size_t cSize, totalCSize, totalGenSize; | |
7c673cae FG |
2162 | U64 crcOrig; |
2163 | BYTE* sampleBuffer; | |
2164 | const BYTE* dict; | |
2165 | size_t dictSize; | |
2166 | ||
2167 | /* notification */ | |
2168 | if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } | |
2169 | else { DISPLAYUPDATE(2, "\r%6u ", testNb); } | |
2170 | ||
2171 | FUZ_rand(&coreSeed); | |
2172 | { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; } | |
2173 | ||
2174 | /* srcBuffer selection [0-4] */ | |
2175 | { U32 buffNb = FUZ_rand(&lseed) & 0x7F; | |
2176 | if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ | |
2177 | else { | |
2178 | buffNb >>= 3; | |
2179 | if (buffNb & 7) { | |
2180 | const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ | |
2181 | buffNb = tnb[buffNb >> 3]; | |
2182 | } else { | |
2183 | const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ | |
2184 | buffNb = tnb[buffNb >> 3]; | |
2185 | } } | |
2186 | srcBuffer = cNoiseBuffer[buffNb]; | |
2187 | } | |
2188 | ||
2189 | /* select src segment */ | |
2190 | sampleSize = FUZ_randomLength(&lseed, maxSampleLog); | |
2191 | ||
2192 | /* create sample buffer (to catch read error with valgrind & sanitizers) */ | |
2193 | sampleBuffer = (BYTE*)malloc(sampleSize); | |
2194 | CHECK(sampleBuffer==NULL, "not enough memory for sample buffer"); | |
2195 | { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); | |
2196 | memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); } | |
2197 | crcOrig = XXH64(sampleBuffer, sampleSize, 0); | |
2198 | ||
2199 | /* compression tests */ | |
9f95a23c | 2200 | { int const cLevelPositive = |
11fdf7f2 TL |
2201 | ( FUZ_rand(&lseed) % |
2202 | (ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / cLevelLimiter)) ) | |
9f95a23c TL |
2203 | + 1; |
2204 | int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ? | |
2205 | - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */ | |
2206 | cLevelPositive; | |
2207 | DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel); | |
7c673cae FG |
2208 | cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); |
2209 | CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize)); | |
2210 | ||
2211 | /* compression failure test : too small dest buffer */ | |
9f95a23c TL |
2212 | assert(cSize > 3); |
2213 | { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; | |
7c673cae | 2214 | const size_t tooSmallSize = cSize - missing; |
9f95a23c TL |
2215 | const unsigned endMark = 0x4DC2B1A9; |
2216 | memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark)); | |
2217 | DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n", | |
2218 | testNb, (unsigned)tooSmallSize, (unsigned)missing); | |
7c673cae | 2219 | { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); |
9f95a23c TL |
2220 | CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); } |
2221 | { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck)); | |
2222 | CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); } | |
7c673cae FG |
2223 | } } |
2224 | ||
11fdf7f2 TL |
2225 | /* frame header decompression test */ |
2226 | { ZSTD_frameHeader zfh; | |
2227 | CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) ); | |
2228 | CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect"); | |
7c673cae FG |
2229 | } |
2230 | ||
11fdf7f2 TL |
2231 | /* Decompressed size test */ |
2232 | { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize); | |
2233 | CHECK(rSize != sampleSize, "decompressed size incorrect"); | |
7c673cae FG |
2234 | } |
2235 | ||
2236 | /* successful decompression test */ | |
9f95a23c | 2237 | DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb); |
7c673cae FG |
2238 | { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; |
2239 | size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize); | |
9f95a23c | 2240 | CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize); |
7c673cae | 2241 | { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0); |
9f95a23c | 2242 | CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize); |
7c673cae FG |
2243 | } } |
2244 | ||
2245 | free(sampleBuffer); /* no longer useful after this point */ | |
2246 | ||
2247 | /* truncated src decompression test */ | |
9f95a23c | 2248 | DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb); |
7c673cae FG |
2249 | { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ |
2250 | size_t const tooSmallSize = cSize - missing; | |
2251 | void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */ | |
2252 | CHECK(cBufferTooSmall == NULL, "not enough memory !"); | |
2253 | memcpy(cBufferTooSmall, cBuffer, tooSmallSize); | |
2254 | { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize); | |
2255 | CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); } | |
2256 | free(cBufferTooSmall); | |
2257 | } | |
2258 | ||
2259 | /* too small dst decompression test */ | |
9f95a23c | 2260 | DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb); |
7c673cae FG |
2261 | if (sampleSize > 3) { |
2262 | size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ | |
2263 | size_t const tooSmallSize = sampleSize - missing; | |
2264 | static const BYTE token = 0xA9; | |
2265 | dstBuffer[tooSmallSize] = token; | |
2266 | { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize); | |
9f95a23c | 2267 | CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); } |
7c673cae FG |
2268 | CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow"); |
2269 | } | |
2270 | ||
2271 | /* noisy src decompression test */ | |
2272 | if (cSize > 6) { | |
2273 | /* insert noise into src */ | |
2274 | { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4)); | |
2275 | size_t pos = 4; /* preserve magic number (too easy to detect) */ | |
2276 | for (;;) { | |
2277 | /* keep some original src */ | |
2278 | { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits; | |
2279 | size_t const mask = (1<<nbBits) - 1; | |
2280 | size_t const skipLength = FUZ_rand(&lseed) & mask; | |
2281 | pos += skipLength; | |
2282 | } | |
9f95a23c | 2283 | if (pos >= cSize) break; |
7c673cae FG |
2284 | /* add noise */ |
2285 | { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits; | |
2286 | U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0; | |
2287 | size_t const mask = (1<<nbBits) - 1; | |
2288 | size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1; | |
2289 | size_t const noiseLength = MIN(rNoiseLength, cSize-pos); | |
2290 | size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength); | |
2291 | memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); | |
2292 | pos += noiseLength; | |
2293 | } } } | |
2294 | ||
2295 | /* decompress noisy source */ | |
9f95a23c | 2296 | DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb); |
7c673cae FG |
2297 | { U32 const endMark = 0xA9B1C3D6; |
2298 | memcpy(dstBuffer+sampleSize, &endMark, 4); | |
2299 | { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize); | |
2300 | /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */ | |
2301 | CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize), | |
9f95a23c | 2302 | "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize); |
7c673cae FG |
2303 | } |
2304 | { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4); | |
2305 | CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); | |
2306 | } } } /* noisy src decompression test */ | |
2307 | ||
9f95a23c TL |
2308 | /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/ |
2309 | DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb); | |
7c673cae | 2310 | { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; |
11fdf7f2 TL |
2311 | U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; |
2312 | int const cLevel = (FUZ_rand(&lseed) % | |
2313 | (ZSTD_maxCLevel() - | |
2314 | (MAX(testLog, dictLog) / cLevelLimiter))) + | |
2315 | 1; | |
7c673cae FG |
2316 | maxTestSize = FUZ_rLogLength(&lseed, testLog); |
2317 | if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1; | |
2318 | ||
11fdf7f2 | 2319 | dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */ |
7c673cae FG |
2320 | dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize)); |
2321 | ||
9f95a23c TL |
2322 | DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n", |
2323 | testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize); | |
2324 | ||
7c673cae | 2325 | if (FUZ_rand(&lseed) & 0xF) { |
11fdf7f2 | 2326 | CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) ); |
7c673cae | 2327 | } else { |
9f95a23c | 2328 | ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); |
11fdf7f2 | 2329 | ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */, |
7c673cae FG |
2330 | !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/, |
2331 | 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */ | |
11fdf7f2 TL |
2332 | ZSTD_parameters const p = FUZ_makeParams(cPar, fPar); |
2333 | CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) ); | |
7c673cae | 2334 | } |
11fdf7f2 TL |
2335 | CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) ); |
2336 | } | |
2337 | ||
7c673cae FG |
2338 | { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2; |
2339 | U32 n; | |
11fdf7f2 TL |
2340 | XXH64_state_t xxhState; |
2341 | XXH64_reset(&xxhState, 0); | |
7c673cae FG |
2342 | for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) { |
2343 | size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog); | |
2344 | size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize); | |
2345 | ||
2346 | if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */ | |
2347 | if (totalTestSize+segmentSize > maxTestSize) break; | |
2348 | ||
2349 | { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize); | |
2350 | CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult)); | |
2351 | cSize += compressResult; | |
2352 | } | |
2353 | XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize); | |
2354 | memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize); | |
2355 | totalTestSize += segmentSize; | |
11fdf7f2 | 2356 | } |
7c673cae | 2357 | |
11fdf7f2 TL |
2358 | { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0); |
2359 | CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); | |
2360 | cSize += flushResult; | |
2361 | } | |
2362 | crcOrig = XXH64_digest(&xxhState); | |
7c673cae | 2363 | } |
7c673cae FG |
2364 | |
2365 | /* streaming decompression test */ | |
9f95a23c | 2366 | DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb); |
11fdf7f2 TL |
2367 | /* ensure memory requirement is good enough (should always be true) */ |
2368 | { ZSTD_frameHeader zfh; | |
9f95a23c | 2369 | CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX), |
11fdf7f2 TL |
2370 | "ZSTD_getFrameHeader(): error retrieving frame information"); |
2371 | { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize); | |
2372 | CHECK_Z(roundBuffSize); | |
2373 | CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN), | |
2374 | "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)", | |
9f95a23c | 2375 | (unsigned)roundBuffSize, (unsigned)totalTestSize ); |
11fdf7f2 | 2376 | } } |
7c673cae | 2377 | if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */ |
11fdf7f2 | 2378 | CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) ); |
7c673cae FG |
2379 | totalCSize = 0; |
2380 | totalGenSize = 0; | |
2381 | while (totalCSize < cSize) { | |
2382 | size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx); | |
2383 | size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize); | |
2384 | CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize)); | |
2385 | totalGenSize += genSize; | |
2386 | totalCSize += inSize; | |
2387 | } | |
2388 | CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded"); | |
2389 | CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size") | |
2390 | CHECK (totalCSize != cSize, "compressed data should be fully read") | |
2391 | { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); | |
9f95a23c TL |
2392 | CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)", |
2393 | (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize); | |
2394 | } | |
7c673cae FG |
2395 | } /* for ( ; (testNb <= nbTests) */ |
2396 | DISPLAY("\r%u fuzzer tests completed \n", testNb-1); | |
2397 | ||
2398 | _cleanup: | |
2399 | ZSTD_freeCCtx(refCtx); | |
2400 | ZSTD_freeCCtx(ctx); | |
2401 | ZSTD_freeDCtx(dctx); | |
2402 | free(cNoiseBuffer[0]); | |
2403 | free(cNoiseBuffer[1]); | |
2404 | free(cNoiseBuffer[2]); | |
2405 | free(cNoiseBuffer[3]); | |
2406 | free(cNoiseBuffer[4]); | |
2407 | free(cBuffer); | |
2408 | free(dstBuffer); | |
2409 | free(mirrorBuffer); | |
2410 | return result; | |
2411 | ||
2412 | _output_error: | |
2413 | result = 1; | |
2414 | goto _cleanup; | |
2415 | } | |
2416 | ||
2417 | ||
2418 | /*_******************************************************* | |
2419 | * Command line | |
2420 | *********************************************************/ | |
11fdf7f2 | 2421 | static int FUZ_usage(const char* programName) |
7c673cae FG |
2422 | { |
2423 | DISPLAY( "Usage :\n"); | |
2424 | DISPLAY( " %s [args]\n", programName); | |
2425 | DISPLAY( "\n"); | |
2426 | DISPLAY( "Arguments :\n"); | |
9f95a23c | 2427 | DISPLAY( " -i# : Nb of tests (default:%i) \n", nbTestsDefault); |
7c673cae FG |
2428 | DISPLAY( " -s# : Select seed (default:prompt user)\n"); |
2429 | DISPLAY( " -t# : Select starting test number (default:0)\n"); | |
9f95a23c | 2430 | DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default); |
7c673cae FG |
2431 | DISPLAY( " -v : verbose\n"); |
2432 | DISPLAY( " -p : pause at the end\n"); | |
2433 | DISPLAY( " -h : display help and exit\n"); | |
2434 | return 0; | |
2435 | } | |
2436 | ||
11fdf7f2 TL |
2437 | /*! readU32FromChar() : |
2438 | @return : unsigned integer value read from input in `char` format | |
2439 | allows and interprets K, KB, KiB, M, MB and MiB suffix. | |
2440 | Will also modify `*stringPtr`, advancing it to position where it stopped reading. | |
2441 | Note : function result can overflow if digit string > MAX_UINT */ | |
2442 | static unsigned readU32FromChar(const char** stringPtr) | |
2443 | { | |
2444 | unsigned result = 0; | |
2445 | while ((**stringPtr >='0') && (**stringPtr <='9')) | |
2446 | result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; | |
2447 | if ((**stringPtr=='K') || (**stringPtr=='M')) { | |
2448 | result <<= 10; | |
2449 | if (**stringPtr=='M') result <<= 10; | |
2450 | (*stringPtr)++ ; | |
2451 | if (**stringPtr=='i') (*stringPtr)++; | |
2452 | if (**stringPtr=='B') (*stringPtr)++; | |
2453 | } | |
2454 | return result; | |
2455 | } | |
2456 | ||
2457 | /** longCommandWArg() : | |
2458 | * check if *stringPtr is the same as longCommand. | |
2459 | * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. | |
2460 | * @return 0 and doesn't modify *stringPtr otherwise. | |
2461 | */ | |
2462 | static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) | |
2463 | { | |
2464 | size_t const comSize = strlen(longCommand); | |
2465 | int const result = !strncmp(*stringPtr, longCommand, comSize); | |
2466 | if (result) *stringPtr += comSize; | |
2467 | return result; | |
2468 | } | |
7c673cae FG |
2469 | |
2470 | int main(int argc, const char** argv) | |
2471 | { | |
11fdf7f2 TL |
2472 | U32 seed = 0; |
2473 | int seedset = 0; | |
7c673cae FG |
2474 | int argNb; |
2475 | int nbTests = nbTestsDefault; | |
2476 | int testNb = 0; | |
9f95a23c | 2477 | int proba = FUZ_compressibility_default; |
11fdf7f2 | 2478 | int result = 0; |
7c673cae FG |
2479 | U32 mainPause = 0; |
2480 | U32 maxDuration = 0; | |
11fdf7f2 TL |
2481 | int bigTests = 1; |
2482 | U32 memTestsOnly = 0; | |
2483 | const char* const programName = argv[0]; | |
7c673cae FG |
2484 | |
2485 | /* Check command line */ | |
2486 | for (argNb=1; argNb<argc; argNb++) { | |
2487 | const char* argument = argv[argNb]; | |
2488 | if(!argument) continue; /* Protection if argument empty */ | |
2489 | ||
2490 | /* Handle commands. Aggregated commands are allowed */ | |
2491 | if (argument[0]=='-') { | |
11fdf7f2 TL |
2492 | |
2493 | if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; } | |
2494 | ||
2495 | if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; } | |
2496 | if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } | |
2497 | ||
7c673cae FG |
2498 | argument++; |
2499 | while (*argument!=0) { | |
2500 | switch(*argument) | |
2501 | { | |
2502 | case 'h': | |
2503 | return FUZ_usage(programName); | |
11fdf7f2 | 2504 | |
7c673cae FG |
2505 | case 'v': |
2506 | argument++; | |
9f95a23c | 2507 | g_displayLevel++; |
7c673cae | 2508 | break; |
11fdf7f2 | 2509 | |
7c673cae FG |
2510 | case 'q': |
2511 | argument++; | |
2512 | g_displayLevel--; | |
2513 | break; | |
11fdf7f2 | 2514 | |
7c673cae FG |
2515 | case 'p': /* pause at the end */ |
2516 | argument++; | |
2517 | mainPause = 1; | |
2518 | break; | |
2519 | ||
2520 | case 'i': | |
11fdf7f2 TL |
2521 | argument++; maxDuration = 0; |
2522 | nbTests = readU32FromChar(&argument); | |
7c673cae FG |
2523 | break; |
2524 | ||
2525 | case 'T': | |
2526 | argument++; | |
11fdf7f2 TL |
2527 | nbTests = 0; |
2528 | maxDuration = readU32FromChar(&argument); | |
2529 | if (*argument=='s') argument++; /* seconds */ | |
2530 | if (*argument=='m') maxDuration *= 60, argument++; /* minutes */ | |
7c673cae FG |
2531 | if (*argument=='n') argument++; |
2532 | break; | |
2533 | ||
2534 | case 's': | |
2535 | argument++; | |
11fdf7f2 TL |
2536 | seedset = 1; |
2537 | seed = readU32FromChar(&argument); | |
7c673cae FG |
2538 | break; |
2539 | ||
2540 | case 't': | |
2541 | argument++; | |
11fdf7f2 | 2542 | testNb = readU32FromChar(&argument); |
7c673cae FG |
2543 | break; |
2544 | ||
2545 | case 'P': /* compressibility % */ | |
2546 | argument++; | |
11fdf7f2 TL |
2547 | proba = readU32FromChar(&argument); |
2548 | if (proba>100) proba = 100; | |
7c673cae FG |
2549 | break; |
2550 | ||
2551 | default: | |
11fdf7f2 | 2552 | return (FUZ_usage(programName), 1); |
7c673cae FG |
2553 | } } } } /* for (argNb=1; argNb<argc; argNb++) */ |
2554 | ||
2555 | /* Get Seed */ | |
2556 | DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING); | |
2557 | ||
2558 | if (!seedset) { | |
2559 | time_t const t = time(NULL); | |
2560 | U32 const h = XXH32(&t, sizeof(t), 1); | |
2561 | seed = h % 10000; | |
2562 | } | |
2563 | ||
9f95a23c TL |
2564 | DISPLAY("Seed = %u\n", (unsigned)seed); |
2565 | if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba); | |
7c673cae | 2566 | |
11fdf7f2 TL |
2567 | if (memTestsOnly) { |
2568 | g_displayLevel = MAX(3, g_displayLevel); | |
2569 | return FUZ_mallocTests(seed, ((double)proba) / 100, memTestsOnly); | |
2570 | } | |
2571 | ||
7c673cae FG |
2572 | if (nbTests < testNb) nbTests = testNb; |
2573 | ||
2574 | if (testNb==0) | |
2575 | result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */ | |
2576 | if (!result) | |
11fdf7f2 | 2577 | result = fuzzerTests(seed, nbTests, testNb, maxDuration, ((double)proba) / 100, bigTests); |
7c673cae FG |
2578 | if (mainPause) { |
2579 | int unused; | |
2580 | DISPLAY("Press Enter \n"); | |
2581 | unused = getchar(); | |
2582 | (void)unused; | |
2583 | } | |
2584 | return result; | |
2585 | } |