ceph/src/isa-l/igzip/igzip_semi_dyn_file_perf.c

   1 /**********************************************************************
   2   Copyright(c) 2011-2016 Intel Corporation All rights reserved.
   3
   4   Redistribution and use in source and binary forms, with or without
   5   modification, are permitted provided that the following conditions
   6   are met:
   7     * Redistributions of source code must retain the above copyright
   8       notice, this list of conditions and the following disclaimer.
   9     * Redistributions in binary form must reproduce the above copyright
  10       notice, this list of conditions and the following disclaimer in
  11       the documentation and/or other materials provided with the
  12       distribution.
  13     * Neither the name of Intel Corporation nor the names of its
  14       contributors may be used to endorse or promote products derived
  15       from this software without specific prior written permission.
  16
  17   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 **********************************************************************/
  29
  30 #include <stdio.h>
  31 #include <stdlib.h>
  32 #include <string.h>
  33 #include <getopt.h>
  34 #include "igzip_lib.h"
  35 #include "test.h"
  36
  37 #define MIN_BUF_SIZE    (4 * 1024)
  38 #define MIN_TEST_LOOPS   10
  39 #ifndef RUN_MEM_SIZE
  40 # define RUN_MEM_SIZE 500000000
  41 #endif
  42
  43 #define DEFAULT_SEG_SIZE    (512 * 1024)
  44 #define DEFAULT_SAMPLE_SIZE (32 * 1024)
  45
  46 int usage(void)
  47 {
  48         fprintf(stderr,
  49                 "Usage: igzip_semi_dynamic [options] <infile>\n"
  50                 "  -h        help\n"
  51                 "  -v        (don't) validate output by inflate and compare\n"
  52                 "  -i <iter> iterations\n"
  53                 "  -t <type> 1:stateless 0:(default)stateful\n"
  54                 "  -c <size> chunk size default=%d\n"
  55                 "  -s <size> sample size default=%d\n"
  56                 "  -o <file> output file\n", DEFAULT_SEG_SIZE, DEFAULT_SAMPLE_SIZE);
  57         exit(0);
  58 }
  59
  60 int str_to_i(char *s)
  61 {
  62 #define ARG_MAX 32
  63
  64         int i = atoi(s);
  65         int len = strnlen(s, ARG_MAX);
  66         if (len < 2 || len == ARG_MAX)
  67                 return i;
  68
  69         switch (s[len - 1]) {
  70         case 'k':
  71                 i *= 1024;
  72                 break;
  73         case 'K':
  74                 i *= 1000;
  75                 break;
  76         case 'm':
  77                 i *= (1024 * 1024);
  78                 break;
  79         case 'M':
  80                 i *= (1000 * 1000);
  81                 break;
  82         case 'g':
  83                 i *= (1024 * 1024 * 1024);
  84                 break;
  85         case 'G':
  86                 i *= (1000 * 1000 * 1000);
  87                 break;
  88         }
  89         return i;
  90 }
  91
  92 int get_filesize(FILE * f)
  93 {
  94         int curr, end;
  95
  96         curr = ftell(f);        /* Save current position */
  97         fseek(f, 0L, SEEK_END);
  98         end = ftell(f);
  99         fseek(f, curr, SEEK_SET);       /* Restore position */
 100         return end;
 101 }
 102
 103 int main(int argc, char *argv[])
 104 {
 105         FILE *in = stdin, *out = NULL;
 106         unsigned char *inbuf, *outbuf;
 107         int i = 0, c, infile_size, outbuf_size;
 108         int segment_size = DEFAULT_SEG_SIZE;
 109         int sample_size = DEFAULT_SAMPLE_SIZE;
 110         int check_output = 1;
 111         int iterations = 0, do_stateless = 0, do_stateful = 1;
 112         int ret = 0;
 113         char *out_file_name = NULL;
 114         struct isal_zstream stream;
 115         struct isal_huff_histogram histogram;
 116         struct isal_hufftables hufftable;
 117
 118         while ((c = getopt(argc, argv, "vht:c:s:o:i:")) != -1) {
 119                 switch (c) {
 120                 case 'v':
 121                         check_output ^= 1;
 122                         break;
 123                 case 't':
 124                         if (atoi(optarg) == 1) {
 125                                 do_stateful = 0;
 126                                 do_stateless = 1;
 127                         }
 128                         break;
 129                 case 'c':
 130                         segment_size = str_to_i(optarg);
 131                         break;
 132                 case 's':
 133                         sample_size = str_to_i(optarg);
 134                         break;
 135                 case 'o':
 136                         out_file_name = optarg;
 137                         break;
 138                 case 'i':
 139                         iterations = str_to_i(optarg);
 140                         break;
 141                 case 'h':
 142                 default:
 143                         usage();
 144                         break;
 145                 }
 146         }
 147
 148         // Open input file
 149         if (optind < argc) {
 150                 if (!(in = fopen(argv[optind], "rb"))) {
 151                         fprintf(stderr, "Can't open %s for reading\n", argv[optind]);
 152                         exit(1);
 153                 }
 154         } else
 155                 usage();
 156
 157         // Optionally open output file
 158         if (out_file_name != NULL) {
 159                 if (!(out = fopen(out_file_name, "wb"))) {
 160                         fprintf(stderr, "Can't open %s for writing\n", out_file_name);
 161                         exit(1);
 162                 }
 163         }
 164
 165         printf("Window Size: %d K\n", IGZIP_HIST_SIZE / 1024);
 166
 167         /*
 168          * Allocate space for entire input file and output
 169          * (assuming some possible expansion on output size)
 170          */
 171         infile_size = get_filesize(in);
 172         if (infile_size == 0) {
 173                 printf("Input file has zero length\n");
 174                 usage();
 175         }
 176
 177         if (iterations == 0) {
 178                 iterations = RUN_MEM_SIZE / infile_size;
 179                 if (iterations < MIN_TEST_LOOPS)
 180                         iterations = MIN_TEST_LOOPS;
 181         }
 182
 183         outbuf_size = infile_size * 1.30 > MIN_BUF_SIZE ? infile_size * 1.30 : MIN_BUF_SIZE;
 184
 185         if (NULL == (inbuf = malloc(infile_size))) {
 186                 fprintf(stderr, "Can't allocate input buffer memory\n");
 187                 exit(0);
 188         }
 189         if (NULL == (outbuf = malloc(outbuf_size))) {
 190                 fprintf(stderr, "Can't allocate output buffer memory\n");
 191                 exit(0);
 192         }
 193
 194         int hist_size = sample_size > segment_size ? segment_size : sample_size;
 195
 196         printf("semi-dynamic sample=%d segment=%d %s\n", hist_size, segment_size,
 197                do_stateful ? "stateful" : "stateless");
 198         printf("igzip_file_perf: %s %d iterations\n", argv[optind], iterations);
 199
 200         // Read complete input file into buffer
 201         stream.avail_in = (uint32_t) fread(inbuf, 1, infile_size, in);
 202         if (stream.avail_in != infile_size) {
 203                 fprintf(stderr, "Couldn't fit all of input file into buffer\n");
 204                 exit(0);
 205         }
 206
 207         struct perf start, stop;
 208
 209         if (do_stateful) {
 210                 perf_start(&start);
 211
 212                 for (i = 0; i < iterations; i++) {
 213                         isal_deflate_init(&stream);
 214                         stream.end_of_stream = 0;
 215                         stream.flush = SYNC_FLUSH;
 216                         stream.next_in = inbuf;
 217                         stream.next_out = outbuf;
 218                         stream.avail_out = outbuf_size;
 219                         int remaining = infile_size;
 220                         int chunk_size = segment_size;
 221
 222                         while (remaining > 0) {
 223                                 // Generate custom hufftables on sample
 224                                 memset(&histogram, 0, sizeof(struct isal_huff_histogram));
 225                                 if (remaining < segment_size * 2) {
 226                                         chunk_size = remaining;
 227                                         stream.end_of_stream = 1;
 228                                 }
 229                                 int hist_rem =
 230                                     (hist_size > chunk_size) ? chunk_size : hist_size;
 231                                 isal_update_histogram(stream.next_in, hist_rem, &histogram);
 232
 233                                 if (hist_rem == chunk_size)
 234                                         isal_create_hufftables_subset(&hufftable, &histogram);
 235                                 else
 236                                         isal_create_hufftables(&hufftable, &histogram);
 237
 238                                 // Compress with custom table
 239                                 stream.avail_in = chunk_size;
 240                                 stream.hufftables = &hufftable;
 241                                 remaining -= chunk_size;
 242                                 isal_deflate(&stream);
 243                                 if (stream.internal_state.state != ZSTATE_NEW_HDR)
 244                                         break;
 245                         }
 246                 }
 247                 perf_stop(&stop);
 248         }
 249
 250         if (do_stateless) {
 251                 perf_start(&start);
 252
 253                 for (i = 0; i < iterations; i++) {
 254                         isal_deflate_stateless_init(&stream);
 255                         stream.end_of_stream = 0;
 256                         stream.flush = FULL_FLUSH;
 257                         stream.next_in = inbuf;
 258                         stream.next_out = outbuf;
 259                         int remaining = infile_size;
 260                         int chunk_size = segment_size;
 261
 262                         while (remaining > 0) {
 263                                 // Generate custom hufftables on sample
 264                                 memset(&histogram, 0, sizeof(struct isal_huff_histogram));
 265                                 if (remaining < segment_size * 2) {
 266                                         chunk_size = remaining;
 267                                         stream.end_of_stream = 1;
 268                                 }
 269                                 int hist_rem =
 270                                     (hist_size > chunk_size) ? chunk_size : hist_size;
 271                                 isal_update_histogram(stream.next_in, hist_rem, &histogram);
 272
 273                                 if (hist_rem == chunk_size)
 274                                         isal_create_hufftables_subset(&hufftable, &histogram);
 275                                 else
 276                                         isal_create_hufftables(&hufftable, &histogram);
 277
 278                                 // Compress with custom table
 279                                 stream.avail_in = chunk_size;
 280                                 stream.avail_out = chunk_size + 8 * (1 + (chunk_size >> 16));
 281                                 stream.hufftables = &hufftable;
 282                                 remaining -= chunk_size;
 283                                 isal_deflate_stateless(&stream);
 284                                 if (stream.avail_in != 0)
 285                                         break;
 286                         }
 287                 }
 288                 perf_stop(&stop);
 289         }
 290
 291         if (stream.avail_in != 0) {
 292                 printf("Could not compress all of inbuf\n");
 293                 ret = 1;
 294         }
 295
 296         printf("  file %s - in_size=%d out_size=%d iter=%d ratio=%3.1f%%\n", argv[optind],
 297                infile_size, stream.total_out, i, 100.0 * stream.total_out / infile_size);
 298
 299         printf("igzip_file: ");
 300         perf_print(stop, start, (long long)infile_size * i);
 301
 302         if (out != NULL) {
 303                 printf("writing %s\n", out_file_name);
 304                 fwrite(outbuf, 1, stream.total_out, out);
 305                 fclose(out);
 306         }
 307
 308         fclose(in);
 309
 310         if (check_output) {
 311                 unsigned char *inflate_buf;
 312                 struct inflate_state istate;
 313
 314                 if (NULL == (inflate_buf = malloc(infile_size))) {
 315                         fprintf(stderr, "Can't allocate reconstruct buffer memory\n");
 316                         exit(0);
 317                 }
 318                 isal_inflate_init(&istate);
 319                 istate.next_in = outbuf;
 320                 istate.avail_in = stream.total_out;
 321                 istate.next_out = inflate_buf;
 322                 istate.avail_out = infile_size;
 323                 int check = isal_inflate(&istate);
 324
 325                 if (memcmp(inflate_buf, inbuf, infile_size)) {
 326                         printf("inflate check Fail\n");
 327                         printf(" ret %d total_inflate=%d\n", check, istate.total_out);
 328                         for (i = 0; i < infile_size; i++) {
 329                                 if (inbuf[i] != inflate_buf[i]) {
 330                                         printf("  first diff at offset=%d\n", i);
 331                                         break;
 332                                 }
 333                         }
 334                         ret = 1;
 335                 } else
 336                         printf("inflate check Pass\n");
 337                 free(inflate_buf);
 338         }
 339
 340         printf("End of igzip_semi_dyn_file_perf\n\n");
 341         return ret;
 342 }