ceph/src/spdk/isa-l/igzip/igzip_semi_dyn_file_perf.c

   1 /**********************************************************************
   2   Copyright(c) 2011-2016 Intel Corporation All rights reserved.
   3
   4   Redistribution and use in source and binary forms, with or without
   5   modification, are permitted provided that the following conditions
   6   are met:
   7     * Redistributions of source code must retain the above copyright
   8       notice, this list of conditions and the following disclaimer.
   9     * Redistributions in binary form must reproduce the above copyright
  10       notice, this list of conditions and the following disclaimer in
  11       the documentation and/or other materials provided with the
  12       distribution.
  13     * Neither the name of Intel Corporation nor the names of its
  14       contributors may be used to endorse or promote products derived
  15       from this software without specific prior written permission.
  16
  17   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 **********************************************************************/
  29
  30 #define _FILE_OFFSET_BITS 64
  31 #include <stdio.h>
  32 #include <stdlib.h>
  33 #include <string.h>
  34 #include <getopt.h>
  35 #include "igzip_lib.h"
  36 #include "test.h"
  37
  38 #define MIN_BUF_SIZE    (4 * 1024)
  39 #define MIN_TEST_LOOPS   10
  40 #ifndef RUN_MEM_SIZE
  41 # define RUN_MEM_SIZE 500000000
  42 #endif
  43
  44 #define DEFAULT_SEG_SIZE    (512 * 1024)
  45 #define DEFAULT_SAMPLE_SIZE (32 * 1024)
  46
  47 int usage(void)
  48 {
  49         fprintf(stderr,
  50                 "Usage: igzip_semi_dynamic [options] <infile>\n"
  51                 "  -h        help\n"
  52                 "  -v        (don't) validate output by inflate and compare\n"
  53                 "  -i <iter> iterations\n"
  54                 "  -t <type> 1:stateless 0:(default)stateful\n"
  55                 "  -c <size> chunk size default=%d\n"
  56                 "  -s <size> sample size default=%d\n"
  57                 "  -o <file> output file\n", DEFAULT_SEG_SIZE, DEFAULT_SAMPLE_SIZE);
  58         exit(0);
  59 }
  60
  61 int str_to_i(char *s)
  62 {
  63 #define ARG_MAX 32
  64
  65         int i = atoi(s);
  66         int len = strnlen(s, ARG_MAX);
  67         if (len < 2 || len == ARG_MAX)
  68                 return i;
  69
  70         switch (s[len - 1]) {
  71         case 'k':
  72                 i *= 1024;
  73                 break;
  74         case 'K':
  75                 i *= 1000;
  76                 break;
  77         case 'm':
  78                 i *= (1024 * 1024);
  79                 break;
  80         case 'M':
  81                 i *= (1000 * 1000);
  82                 break;
  83         case 'g':
  84                 i *= (1024 * 1024 * 1024);
  85                 break;
  86         case 'G':
  87                 i *= (1000 * 1000 * 1000);
  88                 break;
  89         }
  90         return i;
  91 }
  92
  93 int main(int argc, char *argv[])
  94 {
  95         FILE *in = stdin, *out = NULL;
  96         unsigned char *inbuf, *outbuf;
  97         int i = 0, c;
  98         uint64_t infile_size, outbuf_size;
  99         int segment_size = DEFAULT_SEG_SIZE;
 100         int sample_size = DEFAULT_SAMPLE_SIZE;
 101         int check_output = 1;
 102         int iterations = 0, do_stateless = 0, do_stateful = 1;
 103         int ret = 0;
 104         char *out_file_name = NULL;
 105         struct isal_zstream stream;
 106         struct isal_huff_histogram histogram;
 107         struct isal_hufftables hufftable;
 108
 109         while ((c = getopt(argc, argv, "vht:c:s:o:i:")) != -1) {
 110                 switch (c) {
 111                 case 'v':
 112                         check_output ^= 1;
 113                         break;
 114                 case 't':
 115                         if (atoi(optarg) == 1) {
 116                                 do_stateful = 0;
 117                                 do_stateless = 1;
 118                         }
 119                         break;
 120                 case 'c':
 121                         segment_size = str_to_i(optarg);
 122                         break;
 123                 case 's':
 124                         sample_size = str_to_i(optarg);
 125                         break;
 126                 case 'o':
 127                         out_file_name = optarg;
 128                         break;
 129                 case 'i':
 130                         iterations = str_to_i(optarg);
 131                         break;
 132                 case 'h':
 133                 default:
 134                         usage();
 135                         break;
 136                 }
 137         }
 138
 139         // Open input file
 140         if (optind < argc) {
 141                 if (!(in = fopen(argv[optind], "rb"))) {
 142                         fprintf(stderr, "Can't open %s for reading\n", argv[optind]);
 143                         exit(1);
 144                 }
 145         } else
 146                 usage();
 147
 148         // Optionally open output file
 149         if (out_file_name != NULL) {
 150                 if (!(out = fopen(out_file_name, "wb"))) {
 151                         fprintf(stderr, "Can't open %s for writing\n", out_file_name);
 152                         exit(1);
 153                 }
 154         }
 155
 156         printf("Window Size: %d K\n", IGZIP_HIST_SIZE / 1024);
 157
 158         /*
 159          * Allocate space for entire input file and output
 160          * (assuming some possible expansion on output size)
 161          */
 162         infile_size = get_filesize(in);
 163         if (infile_size == 0) {
 164                 printf("Input file has zero length\n");
 165                 usage();
 166         }
 167
 168         if (iterations == 0) {
 169                 iterations = RUN_MEM_SIZE / infile_size;
 170                 if (iterations < MIN_TEST_LOOPS)
 171                         iterations = MIN_TEST_LOOPS;
 172         }
 173
 174         outbuf_size = infile_size * 1.30 > MIN_BUF_SIZE ? infile_size * 1.30 : MIN_BUF_SIZE;
 175
 176         if (NULL == (inbuf = malloc(infile_size))) {
 177                 fprintf(stderr, "Can't allocate input buffer memory\n");
 178                 exit(0);
 179         }
 180         if (NULL == (outbuf = malloc(outbuf_size))) {
 181                 fprintf(stderr, "Can't allocate output buffer memory\n");
 182                 exit(0);
 183         }
 184
 185         int hist_size = sample_size > segment_size ? segment_size : sample_size;
 186
 187         printf("semi-dynamic sample=%d segment=%d %s\n", hist_size, segment_size,
 188                do_stateful ? "stateful" : "stateless");
 189         printf("igzip_file_perf: %s %d iterations\n", argv[optind], iterations);
 190
 191         // Read complete input file into buffer
 192         stream.avail_in = (uint32_t) fread(inbuf, 1, infile_size, in);
 193         if (stream.avail_in != infile_size) {
 194                 fprintf(stderr, "Couldn't fit all of input file into buffer\n");
 195                 exit(0);
 196         }
 197
 198         struct perf start, stop;
 199
 200         if (do_stateful) {
 201                 perf_start(&start);
 202
 203                 for (i = 0; i < iterations; i++) {
 204                         isal_deflate_init(&stream);
 205                         stream.end_of_stream = 0;
 206                         stream.flush = SYNC_FLUSH;
 207                         stream.next_in = inbuf;
 208                         stream.next_out = outbuf;
 209                         stream.avail_out = outbuf_size;
 210                         int remaining = infile_size;
 211                         int chunk_size = segment_size;
 212
 213                         while (remaining > 0) {
 214                                 // Generate custom hufftables on sample
 215                                 memset(&histogram, 0, sizeof(struct isal_huff_histogram));
 216                                 if (remaining < segment_size * 2) {
 217                                         chunk_size = remaining;
 218                                         stream.end_of_stream = 1;
 219                                 }
 220                                 int hist_rem =
 221                                     (hist_size > chunk_size) ? chunk_size : hist_size;
 222                                 isal_update_histogram(stream.next_in, hist_rem, &histogram);
 223
 224                                 if (hist_rem == chunk_size)
 225                                         isal_create_hufftables_subset(&hufftable, &histogram);
 226                                 else
 227                                         isal_create_hufftables(&hufftable, &histogram);
 228
 229                                 // Compress with custom table
 230                                 stream.avail_in = chunk_size;
 231                                 stream.hufftables = &hufftable;
 232                                 remaining -= chunk_size;
 233                                 isal_deflate(&stream);
 234                                 if (stream.internal_state.state != ZSTATE_NEW_HDR)
 235                                         break;
 236                         }
 237                 }
 238                 perf_stop(&stop);
 239         }
 240
 241         if (do_stateless) {
 242                 perf_start(&start);
 243
 244                 for (i = 0; i < iterations; i++) {
 245                         isal_deflate_stateless_init(&stream);
 246                         stream.end_of_stream = 0;
 247                         stream.flush = FULL_FLUSH;
 248                         stream.next_in = inbuf;
 249                         stream.next_out = outbuf;
 250                         int remaining = infile_size;
 251                         int chunk_size = segment_size;
 252
 253                         while (remaining > 0) {
 254                                 // Generate custom hufftables on sample
 255                                 memset(&histogram, 0, sizeof(struct isal_huff_histogram));
 256                                 if (remaining < segment_size * 2) {
 257                                         chunk_size = remaining;
 258                                         stream.end_of_stream = 1;
 259                                 }
 260                                 int hist_rem =
 261                                     (hist_size > chunk_size) ? chunk_size : hist_size;
 262                                 isal_update_histogram(stream.next_in, hist_rem, &histogram);
 263
 264                                 if (hist_rem == chunk_size)
 265                                         isal_create_hufftables_subset(&hufftable, &histogram);
 266                                 else
 267                                         isal_create_hufftables(&hufftable, &histogram);
 268
 269                                 // Compress with custom table
 270                                 stream.avail_in = chunk_size;
 271                                 stream.avail_out = chunk_size + 8 * (1 + (chunk_size >> 16));
 272                                 stream.hufftables = &hufftable;
 273                                 remaining -= chunk_size;
 274                                 isal_deflate_stateless(&stream);
 275                                 if (stream.avail_in != 0)
 276                                         break;
 277                         }
 278                 }
 279                 perf_stop(&stop);
 280         }
 281
 282         if (stream.avail_in != 0) {
 283                 printf("Could not compress all of inbuf\n");
 284                 ret = 1;
 285         }
 286
 287         printf("  file %s - in_size=%lu out_size=%d iter=%d ratio=%3.1f%%\n", argv[optind],
 288                infile_size, stream.total_out, i, 100.0 * stream.total_out / infile_size);
 289
 290         printf("igzip_file: ");
 291         perf_print(stop, start, (long long)infile_size * i);
 292
 293         if (out != NULL) {
 294                 printf("writing %s\n", out_file_name);
 295                 fwrite(outbuf, 1, stream.total_out, out);
 296                 fclose(out);
 297         }
 298
 299         fclose(in);
 300
 301         if (check_output) {
 302                 unsigned char *inflate_buf;
 303                 struct inflate_state istate;
 304
 305                 if (NULL == (inflate_buf = malloc(infile_size))) {
 306                         fprintf(stderr, "Can't allocate reconstruct buffer memory\n");
 307                         exit(0);
 308                 }
 309                 isal_inflate_init(&istate);
 310                 istate.next_in = outbuf;
 311                 istate.avail_in = stream.total_out;
 312                 istate.next_out = inflate_buf;
 313                 istate.avail_out = infile_size;
 314                 int check = isal_inflate(&istate);
 315
 316                 if (memcmp(inflate_buf, inbuf, infile_size)) {
 317                         printf("inflate check Fail\n");
 318                         printf(" ret %d total_inflate=%d\n", check, istate.total_out);
 319                         for (i = 0; i < infile_size; i++) {
 320                                 if (inbuf[i] != inflate_buf[i]) {
 321                                         printf("  first diff at offset=%d\n", i);
 322                                         break;
 323                                 }
 324                         }
 325                         ret = 1;
 326                 } else
 327                         printf("inflate check Pass\n");
 328                 free(inflate_buf);
 329         }
 330
 331         printf("End of igzip_semi_dyn_file_perf\n\n");
 332         return ret;
 333 }