]>
Commit | Line | Data |
---|---|---|
9f95a23c | 1 | /* |
f67539c2 | 2 | * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. |
9f95a23c TL |
3 | * All rights reserved. |
4 | * | |
5 | * This source code is licensed under both the BSD-style license (found in the | |
6 | * LICENSE file in the root directory of this source tree) and the GPLv2 (found | |
7 | * in the COPYING file in the root directory of this source tree). | |
8 | * You may select, at your option, one of the above-listed licenses. | |
9 | */ | |
10 | ||
11 | ||
12 | /* ************************************** | |
13 | * Tuning parameters | |
14 | ****************************************/ | |
15 | #ifndef BMK_TIMETEST_DEFAULT_S /* default minimum time per test */ | |
16 | #define BMK_TIMETEST_DEFAULT_S 3 | |
17 | #endif | |
18 | ||
19 | ||
20 | /* ************************************* | |
21 | * Includes | |
22 | ***************************************/ | |
23 | #include "platform.h" /* Large Files support */ | |
24 | #include "util.h" /* UTIL_getFileSize, UTIL_sleep */ | |
25 | #include <stdlib.h> /* malloc, free */ | |
26 | #include <string.h> /* memset, strerror */ | |
27 | #include <stdio.h> /* fprintf, fopen */ | |
28 | #include <errno.h> | |
29 | #include <assert.h> /* assert */ | |
30 | ||
31 | #include "timefn.h" /* UTIL_time_t */ | |
32 | #include "benchfn.h" | |
f67539c2 | 33 | #include "../lib/common/mem.h" |
9f95a23c | 34 | #define ZSTD_STATIC_LINKING_ONLY |
f67539c2 | 35 | #include "../lib/zstd.h" |
9f95a23c | 36 | #include "datagen.h" /* RDG_genBuffer */ |
f67539c2 | 37 | #include "../lib/common/xxhash.h" |
9f95a23c | 38 | #include "benchzstd.h" |
f67539c2 | 39 | #include "../lib/common/zstd_errors.h" |
9f95a23c TL |
40 | |
41 | ||
42 | /* ************************************* | |
43 | * Constants | |
44 | ***************************************/ | |
45 | #ifndef ZSTD_GIT_COMMIT | |
46 | # define ZSTD_GIT_COMMIT_STRING "" | |
47 | #else | |
48 | # define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT) | |
49 | #endif | |
50 | ||
51 | #define TIMELOOP_MICROSEC (1*1000000ULL) /* 1 second */ | |
52 | #define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ | |
53 | #define ACTIVEPERIOD_MICROSEC (70*TIMELOOP_MICROSEC) /* 70 seconds */ | |
54 | #define COOLPERIOD_SEC 10 | |
55 | ||
56 | #define KB *(1 <<10) | |
57 | #define MB *(1 <<20) | |
58 | #define GB *(1U<<30) | |
59 | ||
60 | #define BMK_RUNTEST_DEFAULT_MS 1000 | |
61 | ||
62 | static const size_t maxMemory = (sizeof(size_t)==4) ? | |
63 | /* 32-bit */ (2 GB - 64 MB) : | |
64 | /* 64-bit */ (size_t)(1ULL << ((sizeof(size_t)*8)-31)); | |
65 | ||
66 | ||
67 | /* ************************************* | |
68 | * console display | |
69 | ***************************************/ | |
70 | #define DISPLAY(...) fprintf(stderr, __VA_ARGS__) | |
71 | #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } | |
72 | /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ | |
73 | ||
74 | static const U64 g_refreshRate = SEC_TO_MICRO / 6; | |
75 | static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; | |
76 | ||
77 | #define DISPLAYUPDATE(l, ...) { if (displayLevel>=l) { \ | |
78 | if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (displayLevel>=4)) \ | |
79 | { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ | |
80 | if (displayLevel>=4) fflush(stderr); } } } | |
81 | ||
82 | ||
83 | /* ************************************* | |
84 | * Exceptions | |
85 | ***************************************/ | |
86 | #ifndef DEBUG | |
87 | # define DEBUG 0 | |
88 | #endif | |
89 | #define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } | |
90 | ||
f67539c2 | 91 | #define RETURN_ERROR_INT(errorNum, ...) { \ |
9f95a23c TL |
92 | DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \ |
93 | DISPLAYLEVEL(1, "Error %i : ", errorNum); \ | |
94 | DISPLAYLEVEL(1, __VA_ARGS__); \ | |
95 | DISPLAYLEVEL(1, " \n"); \ | |
96 | return errorNum; \ | |
97 | } | |
98 | ||
99 | #define CHECK_Z(zf) { \ | |
100 | size_t const zerr = zf; \ | |
101 | if (ZSTD_isError(zerr)) { \ | |
102 | DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \ | |
103 | DISPLAY("Error : "); \ | |
104 | DISPLAY("%s failed : %s", \ | |
105 | #zf, ZSTD_getErrorName(zerr)); \ | |
106 | DISPLAY(" \n"); \ | |
107 | exit(1); \ | |
108 | } \ | |
109 | } | |
110 | ||
111 | #define RETURN_ERROR(errorNum, retType, ...) { \ | |
112 | retType r; \ | |
113 | memset(&r, 0, sizeof(retType)); \ | |
114 | DEBUGOUTPUT("%s: %i: \n", __FILE__, __LINE__); \ | |
115 | DISPLAYLEVEL(1, "Error %i : ", errorNum); \ | |
116 | DISPLAYLEVEL(1, __VA_ARGS__); \ | |
117 | DISPLAYLEVEL(1, " \n"); \ | |
118 | r.tag = errorNum; \ | |
119 | return r; \ | |
120 | } | |
121 | ||
122 | ||
123 | /* ************************************* | |
124 | * Benchmark Parameters | |
125 | ***************************************/ | |
126 | ||
127 | BMK_advancedParams_t BMK_initAdvancedParams(void) { | |
128 | BMK_advancedParams_t const res = { | |
129 | BMK_both, /* mode */ | |
130 | BMK_TIMETEST_DEFAULT_S, /* nbSeconds */ | |
131 | 0, /* blockSize */ | |
132 | 0, /* nbWorkers */ | |
133 | 0, /* realTime */ | |
134 | 0, /* additionalParam */ | |
135 | 0, /* ldmFlag */ | |
136 | 0, /* ldmMinMatch */ | |
137 | 0, /* ldmHashLog */ | |
138 | 0, /* ldmBuckSizeLog */ | |
139 | 0, /* ldmHashRateLog */ | |
140 | ZSTD_lcm_auto /* literalCompressionMode */ | |
141 | }; | |
142 | return res; | |
143 | } | |
144 | ||
145 | ||
146 | /* ******************************************************** | |
147 | * Bench functions | |
148 | **********************************************************/ | |
149 | typedef struct { | |
150 | const void* srcPtr; | |
151 | size_t srcSize; | |
152 | void* cPtr; | |
153 | size_t cRoom; | |
154 | size_t cSize; | |
155 | void* resPtr; | |
156 | size_t resSize; | |
157 | } blockParam_t; | |
158 | ||
159 | #undef MIN | |
160 | #undef MAX | |
161 | #define MIN(a,b) ((a) < (b) ? (a) : (b)) | |
162 | #define MAX(a,b) ((a) > (b) ? (a) : (b)) | |
163 | ||
164 | static void | |
165 | BMK_initCCtx(ZSTD_CCtx* ctx, | |
166 | const void* dictBuffer, size_t dictBufferSize, | |
167 | int cLevel, | |
168 | const ZSTD_compressionParameters* comprParams, | |
169 | const BMK_advancedParams_t* adv) | |
170 | { | |
171 | ZSTD_CCtx_reset(ctx, ZSTD_reset_session_and_parameters); | |
172 | if (adv->nbWorkers==1) { | |
173 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, 0)); | |
174 | } else { | |
175 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_nbWorkers, adv->nbWorkers)); | |
176 | } | |
177 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, cLevel)); | |
178 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_enableLongDistanceMatching, adv->ldmFlag)); | |
179 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmMinMatch, adv->ldmMinMatch)); | |
180 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashLog, adv->ldmHashLog)); | |
181 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmBucketSizeLog, adv->ldmBucketSizeLog)); | |
182 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_ldmHashRateLog, adv->ldmHashRateLog)); | |
183 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_windowLog, (int)comprParams->windowLog)); | |
184 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_hashLog, (int)comprParams->hashLog)); | |
185 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_chainLog, (int)comprParams->chainLog)); | |
186 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_searchLog, (int)comprParams->searchLog)); | |
187 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_minMatch, (int)comprParams->minMatch)); | |
188 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_targetLength, (int)comprParams->targetLength)); | |
189 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_literalCompressionMode, (int)adv->literalCompressionMode)); | |
190 | CHECK_Z(ZSTD_CCtx_setParameter(ctx, ZSTD_c_strategy, comprParams->strategy)); | |
191 | CHECK_Z(ZSTD_CCtx_loadDictionary(ctx, dictBuffer, dictBufferSize)); | |
192 | } | |
193 | ||
194 | static void BMK_initDCtx(ZSTD_DCtx* dctx, | |
195 | const void* dictBuffer, size_t dictBufferSize) { | |
196 | CHECK_Z(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters)); | |
197 | CHECK_Z(ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictBufferSize)); | |
198 | } | |
199 | ||
200 | ||
201 | typedef struct { | |
202 | ZSTD_CCtx* cctx; | |
203 | const void* dictBuffer; | |
204 | size_t dictBufferSize; | |
205 | int cLevel; | |
206 | const ZSTD_compressionParameters* comprParams; | |
207 | const BMK_advancedParams_t* adv; | |
208 | } BMK_initCCtxArgs; | |
209 | ||
210 | static size_t local_initCCtx(void* payload) { | |
211 | BMK_initCCtxArgs* ag = (BMK_initCCtxArgs*)payload; | |
212 | BMK_initCCtx(ag->cctx, ag->dictBuffer, ag->dictBufferSize, ag->cLevel, ag->comprParams, ag->adv); | |
213 | return 0; | |
214 | } | |
215 | ||
216 | typedef struct { | |
217 | ZSTD_DCtx* dctx; | |
218 | const void* dictBuffer; | |
219 | size_t dictBufferSize; | |
220 | } BMK_initDCtxArgs; | |
221 | ||
222 | static size_t local_initDCtx(void* payload) { | |
223 | BMK_initDCtxArgs* ag = (BMK_initDCtxArgs*)payload; | |
224 | BMK_initDCtx(ag->dctx, ag->dictBuffer, ag->dictBufferSize); | |
225 | return 0; | |
226 | } | |
227 | ||
228 | ||
229 | /* `addArgs` is the context */ | |
230 | static size_t local_defaultCompress( | |
231 | const void* srcBuffer, size_t srcSize, | |
232 | void* dstBuffer, size_t dstSize, | |
233 | void* addArgs) | |
234 | { | |
235 | ZSTD_CCtx* const cctx = (ZSTD_CCtx*)addArgs; | |
236 | return ZSTD_compress2(cctx, dstBuffer, dstSize, srcBuffer, srcSize); | |
237 | } | |
238 | ||
239 | /* `addArgs` is the context */ | |
240 | static size_t local_defaultDecompress( | |
241 | const void* srcBuffer, size_t srcSize, | |
242 | void* dstBuffer, size_t dstCapacity, | |
243 | void* addArgs) | |
244 | { | |
245 | size_t moreToFlush = 1; | |
246 | ZSTD_DCtx* const dctx = (ZSTD_DCtx*)addArgs; | |
247 | ZSTD_inBuffer in; | |
248 | ZSTD_outBuffer out; | |
249 | in.src = srcBuffer; in.size = srcSize; in.pos = 0; | |
250 | out.dst = dstBuffer; out.size = dstCapacity; out.pos = 0; | |
251 | while (moreToFlush) { | |
252 | if(out.pos == out.size) { | |
253 | return (size_t)-ZSTD_error_dstSize_tooSmall; | |
254 | } | |
255 | moreToFlush = ZSTD_decompressStream(dctx, &out, &in); | |
256 | if (ZSTD_isError(moreToFlush)) { | |
257 | return moreToFlush; | |
258 | } | |
259 | } | |
260 | return out.pos; | |
261 | ||
262 | } | |
263 | ||
264 | ||
265 | /* ================================================================= */ | |
266 | /* Benchmark Zstandard, mem-to-mem scenarios */ | |
267 | /* ================================================================= */ | |
268 | ||
269 | int BMK_isSuccessful_benchOutcome(BMK_benchOutcome_t outcome) | |
270 | { | |
271 | return outcome.tag == 0; | |
272 | } | |
273 | ||
274 | BMK_benchResult_t BMK_extract_benchResult(BMK_benchOutcome_t outcome) | |
275 | { | |
276 | assert(outcome.tag == 0); | |
277 | return outcome.internal_never_use_directly; | |
278 | } | |
279 | ||
280 | static BMK_benchOutcome_t BMK_benchOutcome_error(void) | |
281 | { | |
282 | BMK_benchOutcome_t b; | |
283 | memset(&b, 0, sizeof(b)); | |
284 | b.tag = 1; | |
285 | return b; | |
286 | } | |
287 | ||
288 | static BMK_benchOutcome_t BMK_benchOutcome_setValidResult(BMK_benchResult_t result) | |
289 | { | |
290 | BMK_benchOutcome_t b; | |
291 | b.tag = 0; | |
292 | b.internal_never_use_directly = result; | |
293 | return b; | |
294 | } | |
295 | ||
296 | ||
297 | /* benchMem with no allocation */ | |
298 | static BMK_benchOutcome_t | |
299 | BMK_benchMemAdvancedNoAlloc( | |
300 | const void** srcPtrs, size_t* srcSizes, | |
301 | void** cPtrs, size_t* cCapacities, size_t* cSizes, | |
302 | void** resPtrs, size_t* resSizes, | |
303 | void** resultBufferPtr, void* compressedBuffer, | |
304 | size_t maxCompressedSize, | |
305 | BMK_timedFnState_t* timeStateCompress, | |
306 | BMK_timedFnState_t* timeStateDecompress, | |
307 | ||
308 | const void* srcBuffer, size_t srcSize, | |
309 | const size_t* fileSizes, unsigned nbFiles, | |
310 | const int cLevel, | |
311 | const ZSTD_compressionParameters* comprParams, | |
312 | const void* dictBuffer, size_t dictBufferSize, | |
313 | ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, | |
314 | int displayLevel, const char* displayName, | |
315 | const BMK_advancedParams_t* adv) | |
316 | { | |
317 | size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize); /* avoid div by 0 */ | |
318 | BMK_benchResult_t benchResult; | |
319 | size_t const loadedCompressedSize = srcSize; | |
320 | size_t cSize = 0; | |
321 | double ratio = 0.; | |
322 | U32 nbBlocks; | |
323 | ||
324 | assert(cctx != NULL); assert(dctx != NULL); | |
325 | ||
326 | /* init */ | |
327 | memset(&benchResult, 0, sizeof(benchResult)); | |
328 | if (strlen(displayName)>17) displayName += strlen(displayName) - 17; /* display last 17 characters */ | |
329 | if (adv->mode == BMK_decodeOnly) { /* benchmark only decompression : source must be already compressed */ | |
330 | const char* srcPtr = (const char*)srcBuffer; | |
331 | U64 totalDSize64 = 0; | |
332 | U32 fileNb; | |
333 | for (fileNb=0; fileNb<nbFiles; fileNb++) { | |
334 | U64 const fSize64 = ZSTD_findDecompressedSize(srcPtr, fileSizes[fileNb]); | |
335 | if (fSize64==0) RETURN_ERROR(32, BMK_benchOutcome_t, "Impossible to determine original size "); | |
336 | totalDSize64 += fSize64; | |
337 | srcPtr += fileSizes[fileNb]; | |
338 | } | |
339 | { size_t const decodedSize = (size_t)totalDSize64; | |
340 | assert((U64)decodedSize == totalDSize64); /* check overflow */ | |
341 | free(*resultBufferPtr); | |
342 | *resultBufferPtr = malloc(decodedSize); | |
343 | if (!(*resultBufferPtr)) { | |
344 | RETURN_ERROR(33, BMK_benchOutcome_t, "not enough memory"); | |
345 | } | |
346 | if (totalDSize64 > decodedSize) { /* size_t overflow */ | |
347 | free(*resultBufferPtr); | |
348 | RETURN_ERROR(32, BMK_benchOutcome_t, "original size is too large"); | |
349 | } | |
350 | cSize = srcSize; | |
351 | srcSize = decodedSize; | |
352 | ratio = (double)srcSize / (double)cSize; | |
353 | } | |
354 | } | |
355 | ||
356 | /* Init data blocks */ | |
357 | { const char* srcPtr = (const char*)srcBuffer; | |
358 | char* cPtr = (char*)compressedBuffer; | |
359 | char* resPtr = (char*)(*resultBufferPtr); | |
360 | U32 fileNb; | |
361 | for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) { | |
362 | size_t remaining = fileSizes[fileNb]; | |
363 | U32 const nbBlocksforThisFile = (adv->mode == BMK_decodeOnly) ? 1 : (U32)((remaining + (blockSize-1)) / blockSize); | |
364 | U32 const blockEnd = nbBlocks + nbBlocksforThisFile; | |
365 | for ( ; nbBlocks<blockEnd; nbBlocks++) { | |
366 | size_t const thisBlockSize = MIN(remaining, blockSize); | |
367 | srcPtrs[nbBlocks] = srcPtr; | |
368 | srcSizes[nbBlocks] = thisBlockSize; | |
369 | cPtrs[nbBlocks] = cPtr; | |
370 | cCapacities[nbBlocks] = (adv->mode == BMK_decodeOnly) ? thisBlockSize : ZSTD_compressBound(thisBlockSize); | |
371 | resPtrs[nbBlocks] = resPtr; | |
372 | resSizes[nbBlocks] = (adv->mode == BMK_decodeOnly) ? (size_t) ZSTD_findDecompressedSize(srcPtr, thisBlockSize) : thisBlockSize; | |
373 | srcPtr += thisBlockSize; | |
374 | cPtr += cCapacities[nbBlocks]; | |
375 | resPtr += thisBlockSize; | |
376 | remaining -= thisBlockSize; | |
377 | if (adv->mode == BMK_decodeOnly) { | |
9f95a23c TL |
378 | cSizes[nbBlocks] = thisBlockSize; |
379 | benchResult.cSize = thisBlockSize; | |
380 | } | |
381 | } | |
382 | } | |
383 | } | |
384 | ||
385 | /* warming up `compressedBuffer` */ | |
386 | if (adv->mode == BMK_decodeOnly) { | |
387 | memcpy(compressedBuffer, srcBuffer, loadedCompressedSize); | |
388 | } else { | |
389 | RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1); | |
390 | } | |
391 | ||
392 | /* Bench */ | |
393 | { U64 const crcOrig = (adv->mode == BMK_decodeOnly) ? 0 : XXH64(srcBuffer, srcSize, 0); | |
394 | # define NB_MARKS 4 | |
395 | const char* marks[NB_MARKS] = { " |", " /", " =", " \\" }; | |
396 | U32 markNb = 0; | |
397 | int compressionCompleted = (adv->mode == BMK_decodeOnly); | |
398 | int decompressionCompleted = (adv->mode == BMK_compressOnly); | |
399 | BMK_benchParams_t cbp, dbp; | |
400 | BMK_initCCtxArgs cctxprep; | |
401 | BMK_initDCtxArgs dctxprep; | |
402 | ||
f67539c2 | 403 | cbp.benchFn = local_defaultCompress; /* ZSTD_compress2 */ |
9f95a23c | 404 | cbp.benchPayload = cctx; |
f67539c2 | 405 | cbp.initFn = local_initCCtx; /* BMK_initCCtx */ |
9f95a23c TL |
406 | cbp.initPayload = &cctxprep; |
407 | cbp.errorFn = ZSTD_isError; | |
408 | cbp.blockCount = nbBlocks; | |
409 | cbp.srcBuffers = srcPtrs; | |
410 | cbp.srcSizes = srcSizes; | |
411 | cbp.dstBuffers = cPtrs; | |
412 | cbp.dstCapacities = cCapacities; | |
413 | cbp.blockResults = cSizes; | |
414 | ||
415 | cctxprep.cctx = cctx; | |
416 | cctxprep.dictBuffer = dictBuffer; | |
417 | cctxprep.dictBufferSize = dictBufferSize; | |
418 | cctxprep.cLevel = cLevel; | |
419 | cctxprep.comprParams = comprParams; | |
420 | cctxprep.adv = adv; | |
421 | ||
422 | dbp.benchFn = local_defaultDecompress; | |
423 | dbp.benchPayload = dctx; | |
424 | dbp.initFn = local_initDCtx; | |
425 | dbp.initPayload = &dctxprep; | |
426 | dbp.errorFn = ZSTD_isError; | |
427 | dbp.blockCount = nbBlocks; | |
428 | dbp.srcBuffers = (const void* const *) cPtrs; | |
429 | dbp.srcSizes = cSizes; | |
430 | dbp.dstBuffers = resPtrs; | |
431 | dbp.dstCapacities = resSizes; | |
432 | dbp.blockResults = NULL; | |
433 | ||
434 | dctxprep.dctx = dctx; | |
435 | dctxprep.dictBuffer = dictBuffer; | |
436 | dctxprep.dictBufferSize = dictBufferSize; | |
437 | ||
438 | DISPLAYLEVEL(2, "\r%70s\r", ""); /* blank line */ | |
439 | DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (unsigned)srcSize); | |
440 | ||
441 | while (!(compressionCompleted && decompressionCompleted)) { | |
442 | if (!compressionCompleted) { | |
443 | BMK_runOutcome_t const cOutcome = BMK_benchTimedFn( timeStateCompress, cbp); | |
444 | ||
445 | if (!BMK_isSuccessful_runOutcome(cOutcome)) { | |
446 | return BMK_benchOutcome_error(); | |
447 | } | |
448 | ||
449 | { BMK_runTime_t const cResult = BMK_extract_runTime(cOutcome); | |
450 | cSize = cResult.sumOfReturn; | |
451 | ratio = (double)srcSize / cSize; | |
452 | { BMK_benchResult_t newResult; | |
453 | newResult.cSpeed = (U64)((double)srcSize * TIMELOOP_NANOSEC / cResult.nanoSecPerRun); | |
454 | benchResult.cSize = cSize; | |
455 | if (newResult.cSpeed > benchResult.cSpeed) | |
456 | benchResult.cSpeed = newResult.cSpeed; | |
457 | } } | |
458 | ||
459 | { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; | |
460 | DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s\r", | |
461 | marks[markNb], displayName, | |
462 | (unsigned)srcSize, (unsigned)cSize, | |
463 | ratioAccuracy, ratio, | |
464 | benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT); | |
465 | } | |
466 | compressionCompleted = BMK_isCompleted_TimedFn(timeStateCompress); | |
467 | } | |
468 | ||
469 | if(!decompressionCompleted) { | |
470 | BMK_runOutcome_t const dOutcome = BMK_benchTimedFn(timeStateDecompress, dbp); | |
471 | ||
472 | if(!BMK_isSuccessful_runOutcome(dOutcome)) { | |
473 | return BMK_benchOutcome_error(); | |
474 | } | |
475 | ||
476 | { BMK_runTime_t const dResult = BMK_extract_runTime(dOutcome); | |
477 | U64 const newDSpeed = (U64)((double)srcSize * TIMELOOP_NANOSEC / dResult.nanoSecPerRun); | |
478 | if (newDSpeed > benchResult.dSpeed) | |
479 | benchResult.dSpeed = newDSpeed; | |
480 | } | |
481 | ||
482 | { int const ratioAccuracy = (ratio < 10.) ? 3 : 2; | |
483 | DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.*f),%6.*f MB/s ,%6.1f MB/s \r", | |
484 | marks[markNb], displayName, | |
f67539c2 | 485 | (unsigned)srcSize, (unsigned)cSize, |
9f95a23c TL |
486 | ratioAccuracy, ratio, |
487 | benchResult.cSpeed < (10 MB) ? 2 : 1, (double)benchResult.cSpeed / MB_UNIT, | |
488 | (double)benchResult.dSpeed / MB_UNIT); | |
489 | } | |
490 | decompressionCompleted = BMK_isCompleted_TimedFn(timeStateDecompress); | |
491 | } | |
492 | markNb = (markNb+1) % NB_MARKS; | |
493 | } /* while (!(compressionCompleted && decompressionCompleted)) */ | |
494 | ||
495 | /* CRC Checking */ | |
496 | { const BYTE* resultBuffer = (const BYTE*)(*resultBufferPtr); | |
497 | U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); | |
498 | if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) { | |
499 | size_t u; | |
500 | DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", | |
501 | displayName, (unsigned)crcOrig, (unsigned)crcCheck); | |
502 | for (u=0; u<srcSize; u++) { | |
503 | if (((const BYTE*)srcBuffer)[u] != resultBuffer[u]) { | |
504 | unsigned segNb, bNb, pos; | |
505 | size_t bacc = 0; | |
506 | DISPLAY("Decoding error at pos %u ", (unsigned)u); | |
507 | for (segNb = 0; segNb < nbBlocks; segNb++) { | |
508 | if (bacc + srcSizes[segNb] > u) break; | |
509 | bacc += srcSizes[segNb]; | |
510 | } | |
511 | pos = (U32)(u - bacc); | |
512 | bNb = pos / (128 KB); | |
513 | DISPLAY("(sample %u, block %u, pos %u) \n", segNb, bNb, pos); | |
514 | { size_t const lowest = (u>5) ? 5 : u; | |
515 | size_t n; | |
516 | DISPLAY("origin: "); | |
517 | for (n=lowest; n>0; n--) | |
518 | DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u-n]); | |
519 | DISPLAY(" :%02X: ", ((const BYTE*)srcBuffer)[u]); | |
520 | for (n=1; n<3; n++) | |
521 | DISPLAY("%02X ", ((const BYTE*)srcBuffer)[u+n]); | |
522 | DISPLAY(" \n"); | |
523 | DISPLAY("decode: "); | |
524 | for (n=lowest; n>0; n++) | |
525 | DISPLAY("%02X ", resultBuffer[u-n]); | |
526 | DISPLAY(" :%02X: ", resultBuffer[u]); | |
527 | for (n=1; n<3; n++) | |
528 | DISPLAY("%02X ", resultBuffer[u+n]); | |
529 | DISPLAY(" \n"); | |
530 | } | |
531 | break; | |
532 | } | |
533 | if (u==srcSize-1) { /* should never happen */ | |
534 | DISPLAY("no difference detected\n"); | |
535 | } | |
f67539c2 TL |
536 | } /* for (u=0; u<srcSize; u++) */ |
537 | } /* if ((adv->mode == BMK_both) && (crcOrig!=crcCheck)) */ | |
9f95a23c TL |
538 | } /* CRC Checking */ |
539 | ||
540 | if (displayLevel == 1) { /* hidden display mode -q, used by python speed benchmark */ | |
541 | double const cSpeed = (double)benchResult.cSpeed / MB_UNIT; | |
542 | double const dSpeed = (double)benchResult.dSpeed / MB_UNIT; | |
543 | if (adv->additionalParam) { | |
544 | DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, adv->additionalParam); | |
545 | } else { | |
546 | DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); | |
547 | } | |
548 | } | |
549 | ||
550 | DISPLAYLEVEL(2, "%2i#\n", cLevel); | |
551 | } /* Bench */ | |
552 | ||
553 | benchResult.cMem = (1ULL << (comprParams->windowLog)) + ZSTD_sizeof_CCtx(cctx); | |
554 | return BMK_benchOutcome_setValidResult(benchResult); | |
555 | } | |
556 | ||
557 | BMK_benchOutcome_t BMK_benchMemAdvanced(const void* srcBuffer, size_t srcSize, | |
558 | void* dstBuffer, size_t dstCapacity, | |
559 | const size_t* fileSizes, unsigned nbFiles, | |
560 | int cLevel, const ZSTD_compressionParameters* comprParams, | |
561 | const void* dictBuffer, size_t dictBufferSize, | |
562 | int displayLevel, const char* displayName, const BMK_advancedParams_t* adv) | |
563 | ||
564 | { | |
565 | int const dstParamsError = !dstBuffer ^ !dstCapacity; /* must be both NULL or none */ | |
566 | ||
567 | size_t const blockSize = ((adv->blockSize>=32 && (adv->mode != BMK_decodeOnly)) ? adv->blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; | |
568 | U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; | |
569 | ||
570 | /* these are the blockTable parameters, just split up */ | |
571 | const void ** const srcPtrs = (const void**)malloc(maxNbBlocks * sizeof(void*)); | |
572 | size_t* const srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); | |
573 | ||
574 | ||
575 | void ** const cPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); | |
576 | size_t* const cSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); | |
577 | size_t* const cCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); | |
578 | ||
579 | void ** const resPtrs = (void**)malloc(maxNbBlocks * sizeof(void*)); | |
580 | size_t* const resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); | |
581 | ||
582 | BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState(adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS); | |
583 | BMK_timedFnState_t* timeStateDecompress = BMK_createTimedFnState(adv->nbSeconds * 1000, BMK_RUNTEST_DEFAULT_MS); | |
584 | ||
585 | ZSTD_CCtx* const cctx = ZSTD_createCCtx(); | |
586 | ZSTD_DCtx* const dctx = ZSTD_createDCtx(); | |
587 | ||
588 | const size_t maxCompressedSize = dstCapacity ? dstCapacity : ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); | |
589 | ||
590 | void* const internalDstBuffer = dstBuffer ? NULL : malloc(maxCompressedSize); | |
591 | void* const compressedBuffer = dstBuffer ? dstBuffer : internalDstBuffer; | |
592 | ||
593 | BMK_benchOutcome_t outcome = BMK_benchOutcome_error(); /* error by default */ | |
594 | ||
595 | void* resultBuffer = srcSize ? malloc(srcSize) : NULL; | |
596 | ||
597 | int allocationincomplete = !srcPtrs || !srcSizes || !cPtrs || | |
598 | !cSizes || !cCapacities || !resPtrs || !resSizes || | |
599 | !timeStateCompress || !timeStateDecompress || | |
600 | !cctx || !dctx || | |
601 | !compressedBuffer || !resultBuffer; | |
602 | ||
603 | ||
604 | if (!allocationincomplete && !dstParamsError) { | |
605 | outcome = BMK_benchMemAdvancedNoAlloc(srcPtrs, srcSizes, | |
606 | cPtrs, cCapacities, cSizes, | |
607 | resPtrs, resSizes, | |
608 | &resultBuffer, | |
609 | compressedBuffer, maxCompressedSize, | |
610 | timeStateCompress, timeStateDecompress, | |
611 | srcBuffer, srcSize, | |
612 | fileSizes, nbFiles, | |
613 | cLevel, comprParams, | |
614 | dictBuffer, dictBufferSize, | |
615 | cctx, dctx, | |
616 | displayLevel, displayName, adv); | |
617 | } | |
618 | ||
619 | /* clean up */ | |
620 | BMK_freeTimedFnState(timeStateCompress); | |
621 | BMK_freeTimedFnState(timeStateDecompress); | |
622 | ||
623 | ZSTD_freeCCtx(cctx); | |
624 | ZSTD_freeDCtx(dctx); | |
625 | ||
626 | free(internalDstBuffer); | |
627 | free(resultBuffer); | |
628 | ||
629 | free((void*)srcPtrs); | |
630 | free(srcSizes); | |
631 | free(cPtrs); | |
632 | free(cSizes); | |
633 | free(cCapacities); | |
634 | free(resPtrs); | |
635 | free(resSizes); | |
636 | ||
637 | if(allocationincomplete) { | |
638 | RETURN_ERROR(31, BMK_benchOutcome_t, "allocation error : not enough memory"); | |
639 | } | |
640 | ||
641 | if(dstParamsError) { | |
642 | RETURN_ERROR(32, BMK_benchOutcome_t, "Dst parameters not coherent"); | |
643 | } | |
644 | return outcome; | |
645 | } | |
646 | ||
647 | BMK_benchOutcome_t BMK_benchMem(const void* srcBuffer, size_t srcSize, | |
648 | const size_t* fileSizes, unsigned nbFiles, | |
649 | int cLevel, const ZSTD_compressionParameters* comprParams, | |
650 | const void* dictBuffer, size_t dictBufferSize, | |
651 | int displayLevel, const char* displayName) { | |
652 | ||
653 | BMK_advancedParams_t const adv = BMK_initAdvancedParams(); | |
654 | return BMK_benchMemAdvanced(srcBuffer, srcSize, | |
655 | NULL, 0, | |
656 | fileSizes, nbFiles, | |
657 | cLevel, comprParams, | |
658 | dictBuffer, dictBufferSize, | |
659 | displayLevel, displayName, &adv); | |
660 | } | |
661 | ||
662 | static BMK_benchOutcome_t BMK_benchCLevel(const void* srcBuffer, size_t benchedSize, | |
663 | const size_t* fileSizes, unsigned nbFiles, | |
664 | int cLevel, const ZSTD_compressionParameters* comprParams, | |
665 | const void* dictBuffer, size_t dictBufferSize, | |
666 | int displayLevel, const char* displayName, | |
667 | BMK_advancedParams_t const * const adv) | |
668 | { | |
669 | const char* pch = strrchr(displayName, '\\'); /* Windows */ | |
670 | if (!pch) pch = strrchr(displayName, '/'); /* Linux */ | |
671 | if (pch) displayName = pch+1; | |
672 | ||
673 | if (adv->realTime) { | |
674 | DISPLAYLEVEL(2, "Note : switching to real-time priority \n"); | |
675 | SET_REALTIME_PRIORITY; | |
676 | } | |
677 | ||
678 | if (displayLevel == 1 && !adv->additionalParam) /* --quiet mode */ | |
679 | DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", | |
680 | ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, | |
681 | (unsigned)benchedSize, adv->nbSeconds, (unsigned)(adv->blockSize>>10)); | |
682 | ||
683 | return BMK_benchMemAdvanced(srcBuffer, benchedSize, | |
684 | NULL, 0, | |
685 | fileSizes, nbFiles, | |
686 | cLevel, comprParams, | |
687 | dictBuffer, dictBufferSize, | |
688 | displayLevel, displayName, adv); | |
689 | } | |
690 | ||
691 | BMK_benchOutcome_t BMK_syntheticTest(int cLevel, double compressibility, | |
692 | const ZSTD_compressionParameters* compressionParams, | |
693 | int displayLevel, const BMK_advancedParams_t* adv) | |
694 | { | |
695 | char name[20] = {0}; | |
696 | size_t const benchedSize = 10000000; | |
697 | void* srcBuffer; | |
698 | BMK_benchOutcome_t res; | |
699 | ||
700 | if (cLevel > ZSTD_maxCLevel()) { | |
701 | RETURN_ERROR(15, BMK_benchOutcome_t, "Invalid Compression Level"); | |
702 | } | |
703 | ||
704 | /* Memory allocation */ | |
705 | srcBuffer = malloc(benchedSize); | |
706 | if (!srcBuffer) RETURN_ERROR(21, BMK_benchOutcome_t, "not enough memory"); | |
707 | ||
708 | /* Fill input buffer */ | |
709 | RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); | |
710 | ||
711 | /* Bench */ | |
712 | snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); | |
713 | res = BMK_benchCLevel(srcBuffer, benchedSize, | |
714 | &benchedSize /* ? */, 1 /* ? */, | |
715 | cLevel, compressionParams, | |
716 | NULL, 0, /* dictionary */ | |
717 | displayLevel, name, adv); | |
718 | ||
719 | /* clean up */ | |
720 | free(srcBuffer); | |
721 | ||
722 | return res; | |
723 | } | |
724 | ||
725 | ||
726 | ||
727 | static size_t BMK_findMaxMem(U64 requiredMem) | |
728 | { | |
729 | size_t const step = 64 MB; | |
730 | BYTE* testmem = NULL; | |
731 | ||
732 | requiredMem = (((requiredMem >> 26) + 1) << 26); | |
733 | requiredMem += step; | |
734 | if (requiredMem > maxMemory) requiredMem = maxMemory; | |
735 | ||
736 | do { | |
737 | testmem = (BYTE*)malloc((size_t)requiredMem); | |
738 | requiredMem -= step; | |
739 | } while (!testmem && requiredMem > 0); | |
740 | ||
741 | free(testmem); | |
742 | return (size_t)(requiredMem); | |
743 | } | |
744 | ||
745 | /*! BMK_loadFiles() : | |
746 | * Loads `buffer` with content of files listed within `fileNamesTable`. | |
747 | * At most, fills `buffer` entirely. */ | |
748 | static int BMK_loadFiles(void* buffer, size_t bufferSize, | |
749 | size_t* fileSizes, | |
750 | const char* const * fileNamesTable, unsigned nbFiles, | |
751 | int displayLevel) | |
752 | { | |
753 | size_t pos = 0, totalSize = 0; | |
754 | unsigned n; | |
755 | for (n=0; n<nbFiles; n++) { | |
f67539c2 | 756 | U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); /* last file may be shortened */ |
9f95a23c TL |
757 | if (UTIL_isDirectory(fileNamesTable[n])) { |
758 | DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]); | |
759 | fileSizes[n] = 0; | |
760 | continue; | |
761 | } | |
762 | if (fileSize == UTIL_FILESIZE_UNKNOWN) { | |
763 | DISPLAYLEVEL(2, "Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]); | |
764 | fileSizes[n] = 0; | |
765 | continue; | |
766 | } | |
f67539c2 TL |
767 | { FILE* const f = fopen(fileNamesTable[n], "rb"); |
768 | if (f==NULL) RETURN_ERROR_INT(10, "impossible to open file %s", fileNamesTable[n]); | |
769 | DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); | |
770 | if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ | |
771 | { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); | |
772 | if (readSize != (size_t)fileSize) RETURN_ERROR_INT(11, "could not read %s", fileNamesTable[n]); | |
773 | pos += readSize; | |
774 | } | |
775 | fileSizes[n] = (size_t)fileSize; | |
776 | totalSize += (size_t)fileSize; | |
777 | fclose(f); | |
778 | } } | |
9f95a23c | 779 | |
f67539c2 | 780 | if (totalSize == 0) RETURN_ERROR_INT(12, "no data to bench"); |
9f95a23c TL |
781 | return 0; |
782 | } | |
783 | ||
784 | BMK_benchOutcome_t BMK_benchFilesAdvanced( | |
785 | const char* const * fileNamesTable, unsigned nbFiles, | |
786 | const char* dictFileName, int cLevel, | |
787 | const ZSTD_compressionParameters* compressionParams, | |
788 | int displayLevel, const BMK_advancedParams_t* adv) | |
789 | { | |
790 | void* srcBuffer = NULL; | |
791 | size_t benchedSize; | |
792 | void* dictBuffer = NULL; | |
793 | size_t dictBufferSize = 0; | |
794 | size_t* fileSizes = NULL; | |
795 | BMK_benchOutcome_t res; | |
796 | U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); | |
797 | ||
798 | if (!nbFiles) { | |
799 | RETURN_ERROR(14, BMK_benchOutcome_t, "No Files to Benchmark"); | |
800 | } | |
801 | ||
802 | if (cLevel > ZSTD_maxCLevel()) { | |
803 | RETURN_ERROR(15, BMK_benchOutcome_t, "Invalid Compression Level"); | |
804 | } | |
805 | ||
806 | fileSizes = (size_t*)calloc(nbFiles, sizeof(size_t)); | |
807 | if (!fileSizes) RETURN_ERROR(12, BMK_benchOutcome_t, "not enough memory for fileSizes"); | |
808 | ||
809 | /* Load dictionary */ | |
810 | if (dictFileName != NULL) { | |
811 | U64 const dictFileSize = UTIL_getFileSize(dictFileName); | |
812 | if (dictFileSize == UTIL_FILESIZE_UNKNOWN) { | |
813 | DISPLAYLEVEL(1, "error loading %s : %s \n", dictFileName, strerror(errno)); | |
814 | free(fileSizes); | |
815 | RETURN_ERROR(9, BMK_benchOutcome_t, "benchmark aborted"); | |
816 | } | |
817 | if (dictFileSize > 64 MB) { | |
818 | free(fileSizes); | |
819 | RETURN_ERROR(10, BMK_benchOutcome_t, "dictionary file %s too large", dictFileName); | |
820 | } | |
821 | dictBufferSize = (size_t)dictFileSize; | |
822 | dictBuffer = malloc(dictBufferSize); | |
823 | if (dictBuffer==NULL) { | |
824 | free(fileSizes); | |
825 | RETURN_ERROR(11, BMK_benchOutcome_t, "not enough memory for dictionary (%u bytes)", | |
826 | (unsigned)dictBufferSize); | |
827 | } | |
828 | ||
829 | { int const errorCode = BMK_loadFiles(dictBuffer, dictBufferSize, | |
830 | fileSizes, &dictFileName /*?*/, | |
831 | 1 /*?*/, displayLevel); | |
832 | if (errorCode) { | |
833 | res = BMK_benchOutcome_error(); | |
834 | goto _cleanUp; | |
835 | } } | |
836 | } | |
837 | ||
838 | /* Memory allocation & restrictions */ | |
839 | benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; | |
840 | if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; | |
841 | if (benchedSize < totalSizeToLoad) | |
842 | DISPLAY("Not enough memory; testing %u MB only...\n", (unsigned)(benchedSize >> 20)); | |
843 | ||
844 | srcBuffer = benchedSize ? malloc(benchedSize) : NULL; | |
845 | if (!srcBuffer) { | |
846 | free(dictBuffer); | |
847 | free(fileSizes); | |
848 | RETURN_ERROR(12, BMK_benchOutcome_t, "not enough memory"); | |
849 | } | |
850 | ||
851 | /* Load input buffer */ | |
852 | { int const errorCode = BMK_loadFiles(srcBuffer, benchedSize, | |
853 | fileSizes, fileNamesTable, nbFiles, | |
854 | displayLevel); | |
855 | if (errorCode) { | |
856 | res = BMK_benchOutcome_error(); | |
857 | goto _cleanUp; | |
858 | } } | |
859 | ||
860 | /* Bench */ | |
861 | { char mfName[20] = {0}; | |
862 | snprintf (mfName, sizeof(mfName), " %u files", nbFiles); | |
863 | { const char* const displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; | |
864 | res = BMK_benchCLevel(srcBuffer, benchedSize, | |
865 | fileSizes, nbFiles, | |
866 | cLevel, compressionParams, | |
867 | dictBuffer, dictBufferSize, | |
868 | displayLevel, displayName, | |
869 | adv); | |
870 | } } | |
871 | ||
872 | _cleanUp: | |
873 | free(srcBuffer); | |
874 | free(dictBuffer); | |
875 | free(fileSizes); | |
876 | return res; | |
877 | } | |
878 | ||
879 | ||
880 | BMK_benchOutcome_t BMK_benchFiles( | |
881 | const char* const * fileNamesTable, unsigned nbFiles, | |
882 | const char* dictFileName, | |
883 | int cLevel, const ZSTD_compressionParameters* compressionParams, | |
884 | int displayLevel) | |
885 | { | |
886 | BMK_advancedParams_t const adv = BMK_initAdvancedParams(); | |
887 | return BMK_benchFilesAdvanced(fileNamesTable, nbFiles, dictFileName, cLevel, compressionParams, displayLevel, &adv); | |
888 | } |