]> git.proxmox.com Git - ceph.git/blob - ceph/src/isa-l/igzip/igzip_check.c
projects / ceph.git / blob
? search:


0ef79d9582bb11b7f77e063543489652190b6691
[ceph.git] / ceph / src / isa-l / igzip / igzip_check.c
1 /**********************************************************************
2 Copyright(c) 2011-2016 Intel Corporation All rights reserved.
3
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions
6 are met:
7 * Redistributions of source code must retain the above copyright
8 notice, this list of conditions and the following disclaimer.
9 * Redistributions in binary form must reproduce the above copyright
10 notice, this list of conditions and the following disclaimer in
11 the documentation and/or other materials provided with the
12 distribution.
13 * Neither the name of Intel Corporation nor the names of its
14 contributors may be used to endorse or promote products derived
15 from this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 **********************************************************************/
29
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <string.h>
33 #include "igzip_lib.h"
34 #include "igzip_inflate_ref.h"
35 #include "crc_inflate.h"
36 #include <math.h>
37
38 #ifndef RANDOMS
39 # define RANDOMS 50
40 #endif
41 #ifndef TEST_SEED
42 # define TEST_SEED 0x1234
43 #endif
44
45 #define IBUF_SIZE (1024*1024)
46
47 #ifndef IGZIP_USE_GZIP_FORMAT
48 # define DEFLATE 1
49 #endif
50
51 #define str1 "Short test string"
52 #define str2 "one two three four five six seven eight nine ten eleven twelve " \
53 "thirteen fourteen fifteen sixteen"
54
55 #define TYPE0_HDR_SIZE 5 /* Size of a type 0 blocks header in bytes */
56 #define TYPE0_MAX_SIZE 65535 /* Max length of a type 0 block in bytes (excludes the header) */
57
58 #define MAX_LOOPS 20
59 /* Defines for the possible error conditions */
60 enum IGZIP_TEST_ERROR_CODES {
61 IGZIP_COMP_OK,
62
63 MALLOC_FAILED,
64 FILE_READ_FAILED,
65
66 COMPRESS_INCORRECT_STATE,
67 COMPRESS_INPUT_STREAM_INTEGRITY_ERROR,
68 COMPRESS_OUTPUT_STREAM_INTEGRITY_ERROR,
69 COMPRESS_END_OF_STREAM_NOT_SET,
70 COMPRESS_ALL_INPUT_FAIL,
71 COMPRESS_OUT_BUFFER_OVERFLOW,
72 COMPRESS_LOOP_COUNT_OVERFLOW,
73 COMPRESS_GENERAL_ERROR,
74
75 INFLATE_END_OF_INPUT,
76 INFLATE_INVALID_BLOCK_HEADER,
77 INFLATE_INVALID_SYMBOL,
78 INFLATE_OUT_BUFFER_OVERFLOW,
79 INFLATE_INVALID_NON_COMPRESSED_BLOCK_LENGTH,
80 INFLATE_LEFTOVER_INPUT,
81 INFLATE_INCORRECT_OUTPUT_SIZE,
82 INFLATE_INVALID_LOOK_BACK_DISTANCE,
83 INVALID_GZIP_HEADER,
84 INCORRECT_GZIP_TRAILER,
85 INFLATE_GENERAL_ERROR,
86
87 INVALID_FLUSH_ERROR,
88
89 OVERFLOW_TEST_ERROR,
90 RESULT_ERROR
91 };
92
93 const int hdr_bytes = 300;
94
95 #ifndef DEFLATE
96 const uint8_t gzip_hdr[10] = {
97 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
98 0x00, 0xff
99 };
100
101 const uint32_t gzip_hdr_bytes = 10;
102 const uint32_t gzip_trl_bytes = 8;
103
104 const int trl_bytes = 8;
105 const int gzip_extra_bytes = 18;
106
107 #else
108 const int trl_bytes = 0;
109 const int gzip_extra_bytes = 0;
110
111 #endif
112
113 #define HISTORY_SIZE 32*1024
114 #define MIN_LENGTH 3
115 #define MIN_DIST 1
116
117 /* Create random compressible data. This is achieved by randomly choosing a
118 * random character, or to repeat previous data in the stream for a random
119 * length and look back distance. The probability of a random character or a
120 * repeat being chosen is semi-randomly chosen by setting max_repeat_data to be
121 * differing values */
122 void create_rand_repeat_data(uint8_t * data, int size)
123 {
124 uint32_t next_data;
125 uint8_t *data_start = data;
126 uint32_t length, distance;
127 uint32_t max_repeat_data = 256;
128 uint32_t power = rand() % 32;
129 /* An array of the powers of 2 (except the final element which is 0) */
130 const uint32_t power_of_2_array[] = {
131 0x00000001, 0x00000002, 0x00000004, 0x00000008,
132 0x00000010, 0x00000020, 0x00000040, 0x00000080,
133 0x00000100, 0x00000200, 0x00000400, 0x00000800,
134 0x00001000, 0x00002000, 0x00004000, 0x00008000,
135 0x00010000, 0x00020000, 0x00040000, 0x00080000,
136 0x00100000, 0x00200000, 0x00400000, 0x00800000,
137 0x01000000, 0x02000000, 0x04000000, 0x08000000,
138 0x10000000, 0x20000000, 0x40000000, 0x00000000
139 };
140
141 max_repeat_data += power_of_2_array[power];
142
143 if (size-- > 0)
144 *data++ = rand();
145
146 while (size > 0) {
147 next_data = rand() % max_repeat_data;
148 if (next_data < 256) {
149 *data++ = next_data;
150 size--;
151 } else if (size < 3) {
152 *data++ = rand() % 256;
153 size--;
154 } else {
155 length = (rand() % 256) + MIN_LENGTH;
156 if (length > size)
157 length = (rand() % (size - 2)) + MIN_LENGTH;
158
159 distance = (rand() % HISTORY_SIZE) + MIN_DIST;
160 if (distance > data - data_start)
161 distance = (rand() % (data - data_start)) + MIN_DIST;
162
163 size -= length;
164 if (distance <= length) {
165 while (length-- > 0) {
166 *data = *(data - distance);
167 data++;
168 }
169 } else
170 memcpy(data, data - distance, length);
171 }
172 }
173 }
174
175 void print_error(int error_code)
176 {
177 switch (error_code) {
178 case IGZIP_COMP_OK:
179 break;
180 case MALLOC_FAILED:
181 printf("error: failed to allocate memory\n");
182 break;
183 case FILE_READ_FAILED:
184 printf("error: failed to read in file\n");
185 break;
186 case COMPRESS_INCORRECT_STATE:
187 printf("error: incorrect stream internal state\n");
188 break;
189 case COMPRESS_INPUT_STREAM_INTEGRITY_ERROR:
190 printf("error: inconsistent stream input buffer\n");
191 break;
192 case COMPRESS_OUTPUT_STREAM_INTEGRITY_ERROR:
193 printf("error: inconsistent stream output buffer\n");
194 break;
195 case COMPRESS_END_OF_STREAM_NOT_SET:
196 printf("error: end of stream not set\n");
197 break;
198 case COMPRESS_ALL_INPUT_FAIL:
199 printf("error: not all input data compressed\n");
200 break;
201 case COMPRESS_OUT_BUFFER_OVERFLOW:
202 printf("error: output buffer overflow while compressing data\n");
203 break;
204 case COMPRESS_GENERAL_ERROR:
205 printf("error: compression failed\n");
206 break;
207 case INFLATE_END_OF_INPUT:
208 printf("error: did not decompress all input\n");
209 break;
210 case INFLATE_INVALID_BLOCK_HEADER:
211 printf("error: invalid header\n");
212 break;
213 case INFLATE_INVALID_SYMBOL:
214 printf("error: invalid symbol found when decompressing input\n");
215 break;
216 case INFLATE_OUT_BUFFER_OVERFLOW:
217 printf("error: output buffer overflow while decompressing data\n");
218 break;
219 case INFLATE_INVALID_NON_COMPRESSED_BLOCK_LENGTH:
220 printf("error: invalid length bits in non-compressed block\n");
221 break;
222 case INFLATE_GENERAL_ERROR:
223 printf("error: decompression failed\n");
224 break;
225 case INFLATE_LEFTOVER_INPUT:
226 printf("error: the trailer of igzip output contains junk\n");
227 break;
228 case INFLATE_INCORRECT_OUTPUT_SIZE:
229 printf("error: incorrect amount of data was decompressed\n");
230 break;
231 case INFLATE_INVALID_LOOK_BACK_DISTANCE:
232 printf("error: invalid look back distance found while decompressing\n");
233 break;
234 case INVALID_GZIP_HEADER:
235 printf("error: incorrect gzip header found when inflating data\n");
236 break;
237 case INCORRECT_GZIP_TRAILER:
238 printf("error: incorrect gzip trailer found when inflating data\n");
239 break;
240 case INVALID_FLUSH_ERROR:
241 printf("error: invalid flush did not cause compression to error\n");
242 break;
243 case RESULT_ERROR:
244 printf("error: decompressed data is not the same as the compressed data\n");
245 break;
246 case OVERFLOW_TEST_ERROR:
247 printf("error: overflow undetected\n");
248 break;
249 default:
250 printf("error: unknown error code\n");
251 }
252 }
253
254 void print_uint8_t(uint8_t * array, uint64_t length)
255 {
256 int i;
257
258 const int line_size = 16;
259 printf("Length = %lu", length);
260 for (i = 0; i < length; i++) {
261 if ((i % line_size) == 0)
262 printf("\n0x%08x\t", i);
263 else
264 printf(" ");
265 printf("0x%02x,", array[i]);
266 }
267 printf("\n");
268 }
269
270 #ifndef DEFLATE
271 uint32_t check_gzip_header(uint8_t * z_buf)
272 {
273 /* These values are defined in RFC 1952 page 4 */
274 const uint8_t ID1 = 0x1f, ID2 = 0x8b, CM = 0x08, FLG = 0;
275 uint32_t ret = 0;
276 int i;
277 /* Verify that the gzip header is the one used in hufftables_c.c */
278 for (i = 0; i < gzip_hdr_bytes; i++)
279 if (z_buf[i] != gzip_hdr[i])
280 ret = INVALID_GZIP_HEADER;
281
282 /* Verify that the gzip header is a valid gzip header */
283 if (*z_buf++ != ID1)
284 ret = INVALID_GZIP_HEADER;
285
286 if (*z_buf++ != ID2)
287 ret = INVALID_GZIP_HEADER;
288
289 /* Verfiy compression method is Deflate */
290 if (*z_buf++ != CM)
291 ret = INVALID_GZIP_HEADER;
292
293 /* The following comparison is specific to how gzip headers are written in igzip */
294 /* Verify no extra flags are set */
295 if (*z_buf != FLG)
296 ret = INVALID_GZIP_HEADER;
297
298 /* The last 6 bytes in the gzip header do not contain any information
299 * important to decomrpessing the data */
300
301 return ret;
302 }
303
304 uint32_t check_gzip_trl(struct inflate_state * gstream)
305 {
306 uint8_t *index = NULL;
307 uint32_t crc, ret = 0;
308
309 index = gstream->out_buffer.next_out - gstream->out_buffer.total_out;
310 crc = find_crc(index, gstream->out_buffer.total_out);
311
312 if (gstream->out_buffer.total_out != *(uint32_t *) (gstream->in_buffer.next_in + 4) ||
313 crc != *(uint32_t *) gstream->in_buffer.next_in)
314 ret = INCORRECT_GZIP_TRAILER;
315
316 return ret;
317 }
318 #endif
319
320 /* Inflate the compressed data and check that the decompressed data agrees with the input data */
321 int inflate_check(uint8_t * z_buf, int z_size, uint8_t * in_buf, int in_size)
322 {
323 /* Test inflate with reference inflate */
324
325 int ret = 0;
326 struct inflate_state gstream;
327 uint32_t test_size = in_size;
328 uint8_t *test_buf = NULL;
329 int mem_result = 0;
330
331 assert(in_buf != NULL);
332
333 if (in_size > 0) {
334 test_buf = malloc(test_size);
335
336 if (test_buf == NULL)
337 return MALLOC_FAILED;
338 }
339 if (test_buf != NULL)
340 memset(test_buf, 0xff, test_size);
341
342 #ifndef DEFLATE
343 int gzip_hdr_result, gzip_trl_result;
344
345 gzip_hdr_result = check_gzip_header(z_buf);
346 z_buf += gzip_hdr_bytes;
347 z_size -= gzip_hdr_bytes;
348 #endif
349
350 igzip_inflate_init(&gstream, z_buf, z_size, test_buf, test_size);
351 ret = igzip_inflate(&gstream);
352
353 if (test_buf != NULL)
354 mem_result = memcmp(in_buf, test_buf, in_size);
355
356 #ifdef VERBOSE
357 int i;
358 if (mem_result)
359 for (i = 0; i < in_size; i++) {
360 if (in_buf[i] != test_buf[i]) {
361 printf("First incorrect data at 0x%x of 0x%x, 0x%x != 0x%x\n",
362 i, in_size, in_buf[i], test_buf[i]);
363 break;
364 }
365 }
366 #endif
367
368 #ifndef DEFLATE
369 gzip_trl_result = check_gzip_trl(&gstream);
370 gstream.in_buffer.avail_in -= gzip_trl_bytes;
371 gstream.in_buffer.next_in += gzip_trl_bytes;
372 #endif
373
374 if (test_buf != NULL)
375 free(test_buf);
376
377 switch (ret) {
378 case 0:
379 break;
380 case END_OF_INPUT:
381 return INFLATE_END_OF_INPUT;
382 break;
383 case INVALID_BLOCK_HEADER:
384 return INFLATE_INVALID_BLOCK_HEADER;
385 break;
386 case INVALID_SYMBOL:
387 return INFLATE_INVALID_SYMBOL;
388 break;
389 case OUT_BUFFER_OVERFLOW:
390 return INFLATE_OUT_BUFFER_OVERFLOW;
391 break;
392 case INVALID_NON_COMPRESSED_BLOCK_LENGTH:
393 return INFLATE_INVALID_NON_COMPRESSED_BLOCK_LENGTH;
394 break;
395 case INVALID_LOOK_BACK_DISTANCE:
396 return INFLATE_INVALID_LOOK_BACK_DISTANCE;
397 break;
398 default:
399 return INFLATE_GENERAL_ERROR;
400 break;
401 }
402
403 if (gstream.in_buffer.avail_in != 0)
404 return INFLATE_LEFTOVER_INPUT;
405
406 if (gstream.out_buffer.total_out != in_size)
407 return INFLATE_INCORRECT_OUTPUT_SIZE;
408
409 if (mem_result)
410 return RESULT_ERROR;
411
412 #ifndef DEFLATE
413 if (gzip_hdr_result)
414 return INVALID_GZIP_HEADER;
415
416 if (gzip_trl_result)
417 return INCORRECT_GZIP_TRAILER;
418 #endif
419
420 return 0;
421 }
422
423 /* Check if that the state of the data stream is consistent */
424 int stream_valid_check(struct isal_zstream *stream, uint8_t * in_buf, uint32_t in_size,
425 uint8_t * out_buf, uint32_t out_size, uint32_t in_processed,
426 uint32_t out_processed, uint32_t data_size)
427 {
428 uint32_t total_in, in_buffer_size, total_out, out_buffer_size;
429
430 total_in =
431 (in_size ==
432 0) ? in_processed : (in_processed - in_size) + (stream->next_in - in_buf);
433 in_buffer_size = (in_size == 0) ? 0 : stream->next_in - in_buf + stream->avail_in;
434
435 /* Check for a consistent amount of data processed */
436 if (total_in != stream->total_in || in_buffer_size != in_size)
437 return COMPRESS_INPUT_STREAM_INTEGRITY_ERROR;
438
439 total_out =
440 (out_size == 0) ? out_processed : out_processed + (stream->next_out - out_buf);
441 out_buffer_size = (out_size == 0) ? 0 : stream->next_out - out_buf + stream->avail_out;
442
443 /* Check for a consistent amount of data compressed */
444 if (total_out != stream->total_out || out_buffer_size != out_size) {
445 return COMPRESS_OUTPUT_STREAM_INTEGRITY_ERROR;
446 }
447
448 return 0;
449 }
450
451 /* Performs compression with checks to discover and verify the state of the
452 * stream
453 * stream: compress data structure which has been initialized to use
454 * in_buf and out_buf as the buffers
455 * data_size: size of all input data
456 * compressed_size: size of all available output buffers
457 * in_buf: next buffer of data to be compressed
458 * in_size: size of in_buf
459 * out_buf: next out put buffer where data is stored
460 * out_size: size of out_buf
461 * in_processed: the amount of input data which has been loaded into buffers
462 * to be compressed, this includes the data in in_buf
463 * out_processed: the amount of output data which has been compressed and stored,
464 * this does not include the data in the current out_buf
465 */
466 int isal_deflate_with_checks(struct isal_zstream *stream, uint32_t data_size,
467 uint32_t compressed_size, uint8_t * in_buf, uint32_t in_size,
468 uint32_t in_processed, uint8_t * out_buf, uint32_t out_size,
469 uint32_t out_processed)
470 {
471 int ret, stream_check;
472 struct isal_zstate *state = &stream->internal_state;
473
474 #ifdef VERBOSE
475 printf("Pre compression\n");
476 printf
477 ("data_size = 0x%05x, in_processed = 0x%05x, in_size = 0x%05x, avail_in = 0x%05x, total_in = 0x%05x\n",
478 data_size, in_processed, in_size, stream->avail_in, stream->total_in);
479 printf
480 ("compressed_size = 0x%05x, out_processed = 0x%05x, out_size = 0x%05x, avail_out = 0x%05x, total_out = 0x%05x\n",
481 compressed_size, out_processed, out_size, stream->avail_out, stream->total_out);
482 #endif
483
484 ret = isal_deflate(stream);
485
486 #ifdef VERBOSE
487 printf("Post compression\n");
488 printf
489 ("data_size = 0x%05x, in_processed = 0x%05x, in_size = 0x%05x, avail_in = 0x%05x, total_in = 0x%05x\n",
490 data_size, in_processed, in_size, stream->avail_in, stream->total_in);
491 printf
492 ("compressed_size = 0x%05x, out_processed = 0x%05x, out_size = 0x%05x, avail_out = 0x%05x, total_out = 0x%05x\n",
493 compressed_size, out_processed, out_size, stream->avail_out, stream->total_out);
494 printf("\n\n");
495 #endif
496
497 /* Verify the stream is in a valid state */
498 stream_check = stream_valid_check(stream, in_buf, in_size, out_buf, out_size,
499 in_processed, out_processed, data_size);
500
501 if (stream_check != 0)
502 return stream_check;
503
504 if (ret != IGZIP_COMP_OK)
505 return COMPRESS_GENERAL_ERROR;
506
507 /* Check if the compression is completed */
508 if (state->state != ZSTATE_END)
509 if (compressed_size - out_processed - (out_size - stream->avail_out) <= 0)
510 return COMPRESS_OUT_BUFFER_OVERFLOW;
511
512 return ret;
513
514 }
515
516 /* Compress the input data into the output buffer where the input buffer and
517 * output buffer are randomly segmented to test state information for the
518 * compression*/
519 int compress_multi_pass(uint8_t * data, uint32_t data_size, uint8_t * compressed_buf,
520 uint32_t * compressed_size, uint32_t flush_type)
521 {
522 int ret = IGZIP_COMP_OK;
523 uint8_t *in_buf = NULL, *out_buf = NULL;
524 uint32_t in_size = 0, out_size = 0;
525 uint32_t in_processed = 0, out_processed = 0;
526 struct isal_zstream stream;
527 struct isal_zstate *state = &stream.internal_state;
528 uint32_t loop_count = 0;
529
530 #ifdef VERBOSE
531 printf("Starting Compress Multi Pass\n");
532 #endif
533
534 create_rand_repeat_data((uint8_t *) & stream, sizeof(stream));
535
536 isal_deflate_init(&stream);
537
538 if (state->state != ZSTATE_NEW_HDR)
539 return COMPRESS_INCORRECT_STATE;
540
541 stream.flush = flush_type;
542 stream.end_of_stream = 0;
543
544 /* These are set here to allow the loop to run correctly */
545 stream.avail_in = 0;
546 stream.avail_out = 0;
547
548 while (1) {
549 loop_count++;
550
551 /* Setup in buffer for next round of compression */
552 if (stream.avail_in == 0) {
553 if (flush_type != SYNC_FLUSH || state->state == ZSTATE_NEW_HDR) {
554 /* Randomly choose size of the next out buffer */
555 in_size = rand() % (data_size + 1);
556
557 /* Limit size of buffer to be smaller than maximum */
558 if (in_size >= data_size - in_processed) {
559 in_size = data_size - in_processed;
560 stream.end_of_stream = 1;
561 }
562
563 if (in_size != 0) {
564 if (in_buf != NULL) {
565 free(in_buf);
566 in_buf = NULL;
567 }
568
569 in_buf = malloc(in_size);
570 if (in_buf == NULL) {
571 ret = MALLOC_FAILED;
572 break;
573 }
574 memcpy(in_buf, data + in_processed, in_size);
575 in_processed += in_size;
576
577 stream.avail_in = in_size;
578 stream.next_in = in_buf;
579 }
580 }
581 }
582
583 /* Setup out buffer for next round of compression */
584 if (stream.avail_out == 0) {
585 /* Save compressed data inot compressed_buf */
586 if (out_buf != NULL) {
587 memcpy(compressed_buf + out_processed, out_buf,
588 out_size - stream.avail_out);
589 out_processed += out_size - stream.avail_out;
590 }
591
592 /* Randomly choose size of the next out buffer */
593 out_size = rand() % (*compressed_size + 1);
594
595 /* Limit size of buffer to be smaller than maximum */
596 if (out_size > *compressed_size - out_processed)
597 out_size = *compressed_size - out_processed;
598
599 if (out_size != 0) {
600 if (out_buf != NULL) {
601 free(out_buf);
602 out_buf = NULL;
603 }
604
605 out_buf = malloc(out_size);
606 if (out_buf == NULL) {
607 ret = MALLOC_FAILED;
608 break;
609 }
610
611 stream.avail_out = out_size;
612 stream.next_out = out_buf;
613 }
614 }
615
616 ret =
617 isal_deflate_with_checks(&stream, data_size, *compressed_size, in_buf,
618 in_size, in_processed, out_buf, out_size,
619 out_processed);
620
621 if (ret) {
622 if (ret == COMPRESS_OUT_BUFFER_OVERFLOW
623 || ret == COMPRESS_INCORRECT_STATE)
624 memcpy(compressed_buf + out_processed, out_buf, out_size);
625 break;
626 }
627
628 /* Check if the compression is completed */
629 if (state->state == ZSTATE_END) {
630 memcpy(compressed_buf + out_processed, out_buf, out_size);
631 *compressed_size = stream.total_out;
632 break;
633 }
634
635 }
636
637 if (in_buf != NULL)
638 free(in_buf);
639 if (out_buf != NULL)
640 free(out_buf);
641
642 if (ret == COMPRESS_OUT_BUFFER_OVERFLOW && flush_type == SYNC_FLUSH
643 && loop_count >= MAX_LOOPS)
644 ret = COMPRESS_LOOP_COUNT_OVERFLOW;
645
646 return ret;
647
648 }
649
650 /* Compress the input data into the outbuffer in one call to isal_deflate */
651 int compress_single_pass(uint8_t * data, uint32_t data_size, uint8_t * compressed_buf,
652 uint32_t * compressed_size, uint32_t flush_type)
653 {
654 int ret = IGZIP_COMP_OK;
655 struct isal_zstream stream;
656 struct isal_zstate *state = &stream.internal_state;
657
658 #ifdef VERBOSE
659 printf("Starting Compress Single Pass\n");
660 #endif
661
662 create_rand_repeat_data((uint8_t *) & stream, sizeof(stream));
663
664 isal_deflate_init(&stream);
665
666 if (state->state != ZSTATE_NEW_HDR)
667 return COMPRESS_INCORRECT_STATE;
668
669 stream.flush = flush_type;
670 stream.avail_in = data_size;
671 stream.next_in = data;
672 stream.avail_out = *compressed_size;
673 stream.next_out = compressed_buf;
674 stream.end_of_stream = 1;
675
676 ret =
677 isal_deflate_with_checks(&stream, data_size, *compressed_size, data, data_size,
678 data_size, compressed_buf, *compressed_size, 0);
679
680 /* Check if the compression is completed */
681 if (state->state == ZSTATE_END)
682 *compressed_size = stream.total_out;
683 else if (flush_type == SYNC_FLUSH && stream.avail_out < 16)
684 ret = COMPRESS_OUT_BUFFER_OVERFLOW;
685
686 return ret;
687
688 }
689
690 /* Statelessly compress the input buffer into the output buffer */
691 int compress_stateless(uint8_t * data, uint32_t data_size, uint8_t * compressed_buf,
692 uint32_t * compressed_size)
693 {
694 int ret = IGZIP_COMP_OK;
695 struct isal_zstream stream;
696
697 create_rand_repeat_data((uint8_t *) & stream, sizeof(stream));
698
699 isal_deflate_init(&stream);
700
701 stream.avail_in = data_size;
702 stream.end_of_stream = 1;
703 stream.next_in = data;
704 stream.flush = NO_FLUSH;
705
706 stream.avail_out = *compressed_size;
707 stream.next_out = compressed_buf;
708
709 ret = isal_deflate_stateless(&stream);
710
711 /* verify the stream */
712 if (stream.next_in - data != stream.total_in ||
713 stream.total_in + stream.avail_in != data_size)
714 return COMPRESS_INPUT_STREAM_INTEGRITY_ERROR;
715
716 if (stream.next_out - compressed_buf != stream.total_out ||
717 stream.total_out + stream.avail_out != *compressed_size)
718 return COMPRESS_OUTPUT_STREAM_INTEGRITY_ERROR;
719
720 if (ret != IGZIP_COMP_OK) {
721 if (ret == STATELESS_OVERFLOW)
722 return COMPRESS_OUT_BUFFER_OVERFLOW;
723 else
724 return COMPRESS_GENERAL_ERROR;
725 }
726
727 if (!stream.end_of_stream) {
728 return COMPRESS_END_OF_STREAM_NOT_SET;
729 }
730
731 if (stream.avail_in != 0)
732 return COMPRESS_ALL_INPUT_FAIL;
733
734 *compressed_size = stream.total_out;
735
736 return ret;
737
738 }
739
740 /*Compress the input buffer into the output buffer, but switch the flush type in
741 * the middle of the compression to test what happens*/
742 int compress_swap_flush(uint8_t * data, uint32_t data_size, uint8_t * compressed_buf,
743 uint32_t * compressed_size, uint32_t flush_type)
744 {
745 int ret = IGZIP_COMP_OK;
746 struct isal_zstream stream;
747 struct isal_zstate *state = &stream.internal_state;
748 uint32_t partial_size;
749
750 #ifdef VERBOSE
751 printf("Starting Compress Swap Flush\n");
752 #endif
753
754 isal_deflate_init(&stream);
755
756 if (state->state != ZSTATE_NEW_HDR)
757 return COMPRESS_INCORRECT_STATE;
758
759 partial_size = rand() % (data_size + 1);
760
761 stream.flush = flush_type;
762 stream.avail_in = partial_size;
763 stream.next_in = data;
764 stream.avail_out = *compressed_size;
765 stream.next_out = compressed_buf;
766 stream.end_of_stream = 0;
767
768 ret =
769 isal_deflate_with_checks(&stream, data_size, *compressed_size, data, partial_size,
770 partial_size, compressed_buf, *compressed_size, 0);
771
772 if (ret)
773 return ret;
774
775 if (flush_type == NO_FLUSH)
776 flush_type = SYNC_FLUSH;
777 else
778 flush_type = NO_FLUSH;
779
780 stream.flush = flush_type;
781 stream.avail_in = data_size - partial_size;
782 stream.next_in = data + partial_size;
783 stream.end_of_stream = 1;
784
785 ret =
786 isal_deflate_with_checks(&stream, data_size, *compressed_size, data + partial_size,
787 data_size - partial_size, data_size, compressed_buf,
788 *compressed_size, 0);
789
790 if (ret == COMPRESS_GENERAL_ERROR)
791 return INVALID_FLUSH_ERROR;
792
793 *compressed_size = stream.total_out;
794
795 return ret;
796 }
797
798 /* Test isal_deflate_stateless */
799 int test_compress_stateless(uint8_t * in_buf, uint32_t in_size)
800 {
801 int ret = IGZIP_COMP_OK;
802 uint32_t z_size, overflow;
803 uint8_t *z_buf = NULL;
804
805 /* Test non-overflow case where a type 0 block is not written */
806 z_size = 2 * in_size + hdr_bytes + trl_bytes;
807
808 z_buf = malloc(z_size);
809
810 if (z_buf == NULL)
811 return MALLOC_FAILED;
812 create_rand_repeat_data(z_buf, z_size);
813
814 ret = compress_stateless(in_buf, in_size, z_buf, &z_size);
815
816 if (!ret)
817 ret = inflate_check(z_buf, z_size, in_buf, in_size);
818
819 if (z_buf != NULL) {
820 free(z_buf);
821 z_buf = NULL;
822 }
823
824 print_error(ret);
825
826 /*Test non-overflow case where a type 0 block is possible to be written */
827 z_size =
828 TYPE0_HDR_SIZE * ((in_size + TYPE0_MAX_SIZE - 1) / TYPE0_MAX_SIZE) + in_size +
829 gzip_extra_bytes;
830
831 if (z_size == gzip_extra_bytes)
832 z_size += TYPE0_HDR_SIZE;
833
834 if (z_size < 8)
835 z_size = 8;
836
837 z_buf = malloc(z_size);
838
839 if (z_buf == NULL)
840 return MALLOC_FAILED;
841
842 create_rand_repeat_data(z_buf, z_size);
843
844 ret = compress_stateless(in_buf, in_size, z_buf, &z_size);
845 if (!ret)
846 ret = inflate_check(z_buf, z_size, in_buf, in_size);
847 #ifdef VERBOSE
848 if (ret) {
849 printf("Compressed array: ");
850 print_uint8_t(z_buf, z_size);
851 printf("\n");
852 printf("Data: ");
853 print_uint8_t(in_buf, in_size);
854 }
855 #endif
856
857 if (!ret) {
858 free(z_buf);
859 z_buf = NULL;
860
861 /* Test random overflow case */
862 z_size = rand() % z_size;
863
864 if (z_size > in_size)
865 z_size = rand() & in_size;
866
867 if (z_size > 0) {
868 z_buf = malloc(z_size);
869
870 if (z_buf == NULL)
871 return MALLOC_FAILED;
872 }
873
874 overflow = compress_stateless(in_buf, in_size, z_buf, &z_size);
875
876 if (overflow != COMPRESS_OUT_BUFFER_OVERFLOW) {
877 #ifdef VERBOSE
878 printf("overflow error = %d\n", overflow);
879 print_error(overflow);
880 if (overflow == 0) {
881 overflow = inflate_check(z_buf, z_size, in_buf, in_size);
882 printf("inflate ret = %d\n", overflow);
883 print_error(overflow);
884 }
885 printf("Compressed array: ");
886 print_uint8_t(z_buf, z_size);
887 printf("\n");
888 printf("Data: ");
889 print_uint8_t(in_buf, in_size);
890 #endif
891 ret = OVERFLOW_TEST_ERROR;
892 }
893 }
894
895 print_error(ret);
896
897 if (z_buf != NULL)
898 free(z_buf);
899
900 return ret;
901 }
902
903 /* Test isal_deflate */
904 int test_compress(uint8_t * in_buf, uint32_t in_size, uint32_t flush_type)
905 {
906 int ret = IGZIP_COMP_OK, fin_ret = IGZIP_COMP_OK;
907 uint32_t overflow = 0;
908 uint32_t z_size, z_size_max, z_compressed_size;
909 uint8_t *z_buf = NULL;
910
911 /* Test a non overflow case */
912 if (flush_type == NO_FLUSH)
913 z_size_max = 2 * in_size + hdr_bytes + trl_bytes + 2;
914 else if (flush_type == SYNC_FLUSH)
915 z_size_max = 2 * in_size + MAX_LOOPS * (hdr_bytes + trl_bytes + 5);
916 else {
917 printf("Invalid Flush Parameter\n");
918 return COMPRESS_GENERAL_ERROR;
919 }
920
921 z_size = z_size_max;
922
923 z_buf = malloc(z_size);
924 if (z_buf == NULL) {
925 print_error(MALLOC_FAILED);
926 return MALLOC_FAILED;
927 }
928 create_rand_repeat_data(z_buf, z_size_max);
929
930 ret = compress_single_pass(in_buf, in_size, z_buf, &z_size, flush_type);
931
932 if (!ret)
933 ret = inflate_check(z_buf, z_size, in_buf, in_size);
934
935 if (ret) {
936 #ifdef VERBOSE
937 printf("Compressed array: ");
938 print_uint8_t(z_buf, z_size);
939 printf("\n");
940 printf("Data: ");
941 print_uint8_t(in_buf, in_size);
942 #endif
943 printf("Failed on compress single pass\n");
944 print_error(ret);
945 }
946
947 fin_ret |= ret;
948
949 z_compressed_size = z_size;
950 z_size = z_size_max;
951 create_rand_repeat_data(z_buf, z_size_max);
952
953 ret = compress_multi_pass(in_buf, in_size, z_buf, &z_size, flush_type);
954
955 if (!ret)
956 ret = inflate_check(z_buf, z_size, in_buf, in_size);
957
958 if (ret) {
959 #ifdef VERBOSE
960 printf("Compressed array: ");
961 print_uint8_t(z_buf, z_size);
962 printf("\n");
963 printf("Data: ");
964 print_uint8_t(in_buf, in_size);
965 #endif
966 printf("Failed on compress multi pass\n");
967 print_error(ret);
968 }
969
970 fin_ret |= ret;
971
972 ret = 0;
973
974 /* Test random overflow case */
975 if (flush_type == SYNC_FLUSH && z_compressed_size > in_size)
976 z_compressed_size = in_size + 1;
977
978 z_size = rand() % z_compressed_size;
979 create_rand_repeat_data(z_buf, z_size_max);
980
981 overflow = compress_single_pass(in_buf, in_size, z_buf, &z_size, flush_type);
982
983 if (overflow != COMPRESS_OUT_BUFFER_OVERFLOW) {
984 if (overflow == 0)
985 ret = inflate_check(z_buf, z_size, in_buf, in_size);
986
987 /* Rarely single pass overflow will compresses data
988 * better than the initial run. This is to stop that
989 * case from erroring. */
990 if (overflow != 0 || ret != 0) {
991 #ifdef VERBOSE
992 printf("overflow error = %d\n", overflow);
993 print_error(overflow);
994 printf("inflate ret = %d\n", ret);
995 print_error(overflow);
996
997 printf("Compressed array: ");
998 print_uint8_t(z_buf, z_size);
999 printf("\n");
1000 printf("Data: ");
1001 print_uint8_t(in_buf, in_size);
1002 #endif
1003 printf("Failed on compress multi pass overflow\n");
1004 print_error(ret);
1005 ret = OVERFLOW_TEST_ERROR;
1006 }
1007 }
1008
1009 fin_ret |= ret;
1010
1011 if (flush_type == NO_FLUSH) {
1012 create_rand_repeat_data(z_buf, z_size_max);
1013
1014 overflow = compress_multi_pass(in_buf, in_size, z_buf, &z_size, flush_type);
1015
1016 if (overflow != COMPRESS_OUT_BUFFER_OVERFLOW) {
1017 if (overflow == 0)
1018 ret = inflate_check(z_buf, z_size, in_buf, in_size);
1019
1020 /* Rarely multi pass overflow will compresses data
1021 * better than the initial run. This is to stop that
1022 * case from erroring */
1023 if (overflow != 0 || ret != 0) {
1024 #ifdef VERBOSE
1025 printf("overflow error = %d\n", overflow);
1026 print_error(overflow);
1027 printf("inflate ret = %d\n", ret);
1028 print_error(overflow);
1029
1030 printf("Compressed array: ");
1031 print_uint8_t(z_buf, z_size);
1032 printf("\n");
1033 printf("Data: ");
1034 print_uint8_t(in_buf, in_size);
1035 #endif
1036 printf("Failed on compress multi pass overflow\n");
1037 print_error(ret);
1038 ret = OVERFLOW_TEST_ERROR;
1039 }
1040 }
1041 fin_ret |= ret;
1042 }
1043
1044 free(z_buf);
1045
1046 return fin_ret;
1047 }
1048
1049 /* Test swapping flush types in the middle of compression */
1050 int test_flush(uint8_t * in_buf, uint32_t in_size)
1051 {
1052 int fin_ret = IGZIP_COMP_OK, ret;
1053 uint32_t z_size, flush_type = 0;
1054 uint8_t *z_buf = NULL;
1055
1056 z_size = 2 * in_size + 2 * (hdr_bytes + trl_bytes) + 8;
1057
1058 z_buf = malloc(z_size);
1059
1060 if (z_buf == NULL)
1061 return MALLOC_FAILED;
1062
1063 create_rand_repeat_data(z_buf, z_size);
1064
1065 while (flush_type == NO_FLUSH || flush_type == SYNC_FLUSH)
1066 flush_type = rand();
1067
1068 /* Test invalid flush */
1069 ret = compress_single_pass(in_buf, in_size, z_buf, &z_size, flush_type);
1070
1071 if (ret == COMPRESS_GENERAL_ERROR)
1072 ret = 0;
1073 else {
1074 printf("Failed when passing invalid flush parameter\n");
1075 ret = INVALID_FLUSH_ERROR;
1076 }
1077
1078 fin_ret |= ret;
1079 print_error(ret);
1080
1081 create_rand_repeat_data(z_buf, z_size);
1082
1083 /* Test the valid case of SYNC_FLUSH followed by NO_FLUSH */
1084 ret = compress_swap_flush(in_buf, in_size, z_buf, &z_size, rand() % 2);
1085
1086 if (!ret)
1087 ret = inflate_check(z_buf, z_size, in_buf, in_size);
1088
1089 if (ret) {
1090 #ifdef VERBOSE
1091 printf("Compressed array: ");
1092 print_uint8_t(z_buf, z_size);
1093 printf("\n");
1094 printf("Data: ");
1095 print_uint8_t(in_buf, in_size);
1096 #endif
1097 printf("Failed on swapping from SYNC_FLUSH to NO_FLUSH\n");
1098 print_error(ret);
1099 }
1100
1101 fin_ret |= ret;
1102 print_error(ret);
1103
1104 return fin_ret;
1105 }
1106
1107 int get_filesize(FILE * f)
1108 {
1109 int curr, end;
1110
1111 curr = ftell(f); /* Save current position */
1112 fseek(f, 0L, SEEK_END);
1113 end = ftell(f);
1114 fseek(f, curr, SEEK_SET); /* Restore position */
1115 return end;
1116 }
1117
1118 /* Run multiple compression tests on data stored in a file */
1119 int test_compress_file(char *file_name)
1120 {
1121 int ret = IGZIP_COMP_OK;
1122 uint32_t in_size;
1123 uint8_t *in_buf = NULL;
1124 FILE *in_file = NULL;
1125
1126 in_file = fopen(file_name, "rb");
1127 if (!in_file)
1128 return FILE_READ_FAILED;
1129
1130 in_size = get_filesize(in_file);
1131 if (in_size != 0) {
1132 in_buf = malloc(in_size);
1133 if (in_buf == NULL)
1134 return MALLOC_FAILED;
1135 fread(in_buf, 1, in_size, in_file);
1136 }
1137
1138 ret |= test_compress_stateless(in_buf, in_size);
1139 ret |= test_compress(in_buf, in_size, NO_FLUSH);
1140 ret |= test_compress(in_buf, in_size, SYNC_FLUSH);
1141 ret |= test_flush(in_buf, in_size);
1142
1143 if (ret)
1144 printf("Failed on file %s\n", file_name);
1145
1146 if (in_buf != NULL)
1147 free(in_buf);
1148
1149 return ret;
1150 }
1151
1152 int main(int argc, char *argv[])
1153 {
1154 int i = 0, ret = 0, fin_ret = 0;
1155 uint32_t in_size = 0, offset = 0;
1156 uint8_t *in_buf;
1157
1158 #ifndef VERBOSE
1159 setbuf(stdout, NULL);
1160 #endif
1161
1162 printf("Window Size: %d K\n", HIST_SIZE);
1163 printf("Test Seed : %d\n", TEST_SEED);
1164 printf("Randoms : %d\n", RANDOMS);
1165 srand(TEST_SEED);
1166
1167 in_buf = malloc(IBUF_SIZE);
1168 if (in_buf == NULL) {
1169 fprintf(stderr, "Can't allocate in_buf memory\n");
1170 return -1;
1171 }
1172
1173 if (argc > 1) {
1174 printf("igzip_rand_test files: ");
1175
1176 for (i = 1; i < argc; i++) {
1177 ret |= test_compress_file(argv[i]);
1178 if (ret)
1179 return ret;
1180 }
1181
1182 printf("................");
1183 printf("%s\n", ret ? "Fail" : "Pass");
1184 fin_ret |= ret;
1185 }
1186
1187 printf("igzip_rand_test stateless: ");
1188
1189 ret = test_compress_stateless((uint8_t *) str1, sizeof(str1));
1190 if (ret)
1191 return ret;
1192
1193 ret |= test_compress_stateless((uint8_t *) str2, sizeof(str2));
1194 if (ret)
1195 return ret;
1196
1197 for (i = 0; i < RANDOMS; i++) {
1198 in_size = rand() % (IBUF_SIZE + 1);
1199 offset = rand() % (IBUF_SIZE + 1 - in_size);
1200 in_buf += offset;
1201
1202 create_rand_repeat_data(in_buf, in_size);
1203
1204 ret |= test_compress_stateless(in_buf, in_size);
1205
1206 in_buf -= offset;
1207
1208 if (i % (RANDOMS / 16) == 0)
1209 printf(".");
1210
1211 if (ret)
1212 return ret;
1213 }
1214
1215 fin_ret |= ret;
1216
1217 printf("%s\n", ret ? "Fail" : "Pass");
1218
1219 printf("igzip_rand_test NO_FLUSH: ");
1220
1221 ret = test_compress((uint8_t *) str1, sizeof(str1), NO_FLUSH);
1222 if (ret)
1223 return ret;
1224
1225 ret |= test_compress((uint8_t *) str2, sizeof(str2), NO_FLUSH);
1226 if (ret)
1227 return ret;
1228
1229 for (i = 0; i < RANDOMS; i++) {
1230 in_size = rand() % (IBUF_SIZE + 1);
1231 offset = rand() % (IBUF_SIZE + 1 - in_size);
1232 in_buf += offset;
1233
1234 create_rand_repeat_data(in_buf, in_size);
1235
1236 ret |= test_compress(in_buf, in_size, NO_FLUSH);
1237
1238 in_buf -= offset;
1239
1240 if (i % (RANDOMS / 16) == 0)
1241 printf(".");
1242 if (ret)
1243 return ret;
1244 }
1245
1246 fin_ret |= ret;
1247
1248 printf("%s\n", ret ? "Fail" : "Pass");
1249
1250 printf("igzip_rand_test SYNC_FLUSH: ");
1251
1252 ret = test_compress((uint8_t *) str1, sizeof(str1), SYNC_FLUSH);
1253 if (ret)
1254 return ret;
1255
1256 ret |= test_compress((uint8_t *) str2, sizeof(str2), SYNC_FLUSH);
1257 if (ret)
1258 return ret;
1259
1260 for (i = 0; i < RANDOMS; i++) {
1261 in_size = rand() % (IBUF_SIZE + 1);
1262 offset = rand() % (IBUF_SIZE + 1 - in_size);
1263 in_buf += offset;
1264
1265 create_rand_repeat_data(in_buf, in_size);
1266
1267 ret |= test_compress(in_buf, in_size, SYNC_FLUSH);
1268
1269 in_buf -= offset;
1270
1271 if (i % (RANDOMS / 16) == 0)
1272 printf(".");
1273 if (ret)
1274 return ret;
1275 }
1276
1277 fin_ret |= ret;
1278
1279 printf("%s\n", ret ? "Fail" : "Pass");
1280
1281 printf("igzip rand test finished: %s\n",
1282 fin_ret ? "Some tests failed" : "All tests passed");
1283
1284 return fin_ret != IGZIP_COMP_OK;
1285 }
Ceph