ceph/src/spdk/isa-l/igzip/generate_custom_hufftables.c

   1 /**********************************************************************
   2   Copyright(c) 2011-2016 Intel Corporation All rights reserved.
   3
   4   Redistribution and use in source and binary forms, with or without
   5   modification, are permitted provided that the following conditions
   6   are met:
   7     * Redistributions of source code must retain the above copyright
   8       notice, this list of conditions and the following disclaimer.
   9     * Redistributions in binary form must reproduce the above copyright
  10       notice, this list of conditions and the following disclaimer in
  11       the documentation and/or other materials provided with the
  12       distribution.
  13     * Neither the name of Intel Corporation nor the names of its
  14       contributors may be used to endorse or promote products derived
  15       from this software without specific prior written permission.
  16
  17   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 **********************************************************************/
  29
  30 /* This program can be used to generate custom a custom huffman encoding to get
  31  * better data compression. This is most useful when the type of data being
  32  * compressed is well known.
  33  *
  34  * To use generate_custom_hufftables, pass a sequence of files to the program
  35  * that together form an accurate representation of the data that is being
  36  * compressed. Generate_custom_hufftables will then produce the file
  37  * hufftables_c.c, which should be moved to replace its counterpart in the igzip
  38  * source folder. After recompiling the Isa-l library, the igzip compression
  39  * functions will use the new hufftables.
  40  *
  41  * Generate_custom_hufftables should be compiled with the same compile time
  42  * parameters as the igzip source code. Generating custom hufftables with
  43  * different compile time parameters may cause igzip to produce invalid output
  44  * for the reasons described below. The default parameters used by
  45  * generate_custom_hufftables are the same as the default parameters used by
  46  * igzip.
  47  *
  48  * *WARNING* generate custom hufftables must be compiled with a IGZIP_HIST_SIZE
  49  * that is at least as large as the IGZIP_HIST_SIZE used by igzip. By default
  50  * IGZIP_HIST_SIZE is 32K, the maximum usable IGZIP_HIST_SIZE is 32K. The reason
  51  * for this is to generate better compression. Igzip cannot produce look back
  52  * distances with sizes larger than the IGZIP_HIST_SIZE igzip was compiled with,
  53  * so look back distances with sizes larger than IGZIP_HIST_SIZE are not
  54  * assigned a huffman code. The definition of LONGER_HUFFTABLES must be
  55  * consistent as well since that definition changes the size of the structures
  56  * printed by this tool.
  57  *
  58  */
  59
  60 #include <stdint.h>
  61 #include <stdio.h>
  62 #include <inttypes.h>
  63 #include <string.h>
  64 #include <stdlib.h>
  65 #include "igzip_lib.h"
  66
  67 /*These max code lengths are limited by how the data is stored in
  68  * hufftables.asm. The deflate standard max is 15.*/
  69
  70 #define MAX_HEADER_SIZE ISAL_DEF_MAX_HDR_SIZE
  71
  72 #define GZIP_HEADER_SIZE 10
  73 #define GZIP_TRAILER_SIZE 8
  74 #define ZLIB_HEADER_SIZE 2
  75 #define ZLIB_TRAILER_SIZE 4
  76
  77 /**
  78  * @brief Prints a table of uint8_t elements to a file.
  79  * @param outfile: the file the table is printed to.
  80  * @param table: the table to be printed.
  81  * @param length: number of elements to be printed.
  82  * @param header: header to append in front of the table.
  83  * @param footer: footer to append at the end of the table.
  84  * @param begin_line: string printed at beginning of new line
  85  */
  86 void fprint_uint8_table(FILE * outfile, uint8_t * table, uint64_t length, char *header,
  87                         char *footer, char *begin_line)
  88 {
  89         int i;
  90         fprintf(outfile, "%s", header);
  91         for (i = 0; i < length - 1; i++) {
  92                 if ((i & 7) == 0)
  93                         fprintf(outfile, "\n%s", begin_line);
  94                 else
  95                         fprintf(outfile, " ");
  96                 fprintf(outfile, "0x%02x,", table[i]);
  97         }
  98
  99         if ((i & 7) == 0)
 100                 fprintf(outfile, "\n%s", begin_line);
 101         else
 102                 fprintf(outfile, " ");
 103         fprintf(outfile, "0x%02x", table[i]);
 104         fprintf(outfile, "%s", footer);
 105
 106 }
 107
 108 /**
 109  * @brief Prints a table of uint16_t elements to a file.
 110  * @param outfile: the file the table is printed to.
 111  * @param table: the table to be printed.
 112  * @param length: number of elements to be printed.
 113  * @param header: header to append in front of the table.
 114  * @param footer: footer to append at the end of the table.
 115  * @param begin_line: string printed at beginning of new line
 116  */
 117 void fprint_uint16_table(FILE * outfile, uint16_t * table, uint64_t length, char *header,
 118                          char *footer, char *begin_line)
 119 {
 120         int i;
 121         fprintf(outfile, "%s", header);
 122         for (i = 0; i < length - 1; i++) {
 123                 if ((i & 7) == 0)
 124                         fprintf(outfile, "\n%s", begin_line);
 125                 else
 126                         fprintf(outfile, " ");
 127                 fprintf(outfile, "0x%04x,", table[i]);
 128         }
 129
 130         if ((i & 7) == 0)
 131                 fprintf(outfile, "\n%s", begin_line);
 132         else
 133                 fprintf(outfile, " ");
 134         fprintf(outfile, "0x%04x", table[i]);
 135         fprintf(outfile, "%s", footer);
 136
 137 }
 138
 139 /**
 140  * @brief Prints a table of uint32_t elements to a file.
 141  * @param outfile: the file the table is printed to.
 142  * @param table: the table to be printed.
 143  * @param length: number of elements to be printed.
 144  * @param header: header to append in front of the table.
 145  * @param footer: footer to append at the end of the table.
 146  * @param begin_line: string printed at beginning of new line
 147  */
 148 void fprint_uint32_table(FILE * outfile, uint32_t * table, uint64_t length, char *header,
 149                          char *footer, char *begin_line)
 150 {
 151         int i;
 152         fprintf(outfile, "%s", header);
 153         for (i = 0; i < length - 1; i++) {
 154                 if ((i & 3) == 0)
 155                         fprintf(outfile, "\n%s", begin_line);
 156                 else
 157                         fprintf(outfile, " ");
 158                 fprintf(outfile, "0x%08x,", table[i]);
 159         }
 160
 161         if ((i & 3) == 0)
 162                 fprintf(outfile, "%s", begin_line);
 163         else
 164                 fprintf(outfile, " ");
 165         fprintf(outfile, "0x%08x", table[i]);
 166         fprintf(outfile, "%s", footer);
 167
 168 }
 169
 170 void fprint_hufftables(FILE * output_file, char *hufftables_name,
 171                        struct isal_hufftables *hufftables)
 172 {
 173         fprintf(output_file, "struct isal_hufftables %s = {\n\n", hufftables_name);
 174
 175         fprint_uint8_table(output_file, hufftables->deflate_hdr,
 176                            hufftables->deflate_hdr_count +
 177                            (hufftables->deflate_hdr_extra_bits + 7) / 8,
 178                            "\t.deflate_hdr = {", "},\n\n", "\t\t");
 179
 180         fprintf(output_file, "\t.deflate_hdr_count = %d,\n", hufftables->deflate_hdr_count);
 181         fprintf(output_file, "\t.deflate_hdr_extra_bits = %d,\n\n",
 182                 hufftables->deflate_hdr_extra_bits);
 183
 184         fprint_uint32_table(output_file, hufftables->dist_table, IGZIP_DIST_TABLE_SIZE,
 185                             "\t.dist_table = {", "},\n\n", "\t\t");
 186
 187         fprint_uint32_table(output_file, hufftables->len_table, IGZIP_LEN_TABLE_SIZE,
 188                             "\t.len_table = {", "},\n\n", "\t\t");
 189
 190         fprint_uint16_table(output_file, hufftables->lit_table, IGZIP_LIT_TABLE_SIZE,
 191                             "\t.lit_table = {", "},\n\n", "\t\t");
 192         fprint_uint8_table(output_file, hufftables->lit_table_sizes, IGZIP_LIT_TABLE_SIZE,
 193                            "\t.lit_table_sizes = {", "},\n\n", "\t\t");
 194
 195         fprint_uint16_table(output_file, hufftables->dcodes,
 196                             ISAL_DEF_DIST_SYMBOLS - IGZIP_DECODE_OFFSET,
 197                             "\t.dcodes = {", "},\n\n", "\t\t");
 198         fprint_uint8_table(output_file, hufftables->dcodes_sizes,
 199                            ISAL_DEF_DIST_SYMBOLS - IGZIP_DECODE_OFFSET,
 200                            "\t.dcodes_sizes = {", "}\n", "\t\t");
 201         fprintf(output_file, "};\n");
 202 }
 203
 204 void fprint_header(FILE * output_file)
 205 {
 206
 207         fprintf(output_file, "#include <stdint.h>\n");
 208         fprintf(output_file, "#include <igzip_lib.h>\n\n");
 209
 210         fprintf(output_file, "#if IGZIP_HIST_SIZE > %d\n"
 211                 "# error \"Invalid history size for the custom hufftable\"\n"
 212                 "#endif\n", IGZIP_HIST_SIZE);
 213
 214 #ifdef LONGER_HUFFTABLE
 215         fprintf(output_file, "#ifndef LONGER_HUFFTABLE\n"
 216                 "# error \"Custom hufftable requires LONGER_HUFFTABLE to be defined \"\n"
 217                 "#endif\n");
 218 #else
 219         fprintf(output_file, "#ifdef LONGER_HUFFTABLE\n"
 220                 "# error \"Custom hufftable requires LONGER_HUFFTABLE to not be defined \"\n"
 221                 "#endif\n");
 222 #endif
 223         fprintf(output_file, "\n");
 224
 225         fprintf(output_file, "const uint8_t gzip_hdr[] = {\n"
 226                 "\t0x1f, 0x8b, 0x08, 0x00, 0x00,\n" "\t0x00, 0x00, 0x00, 0x00, 0xff\t};\n\n");
 227
 228         fprintf(output_file, "const uint32_t gzip_hdr_bytes = %d;\n", GZIP_HEADER_SIZE);
 229         fprintf(output_file, "const uint32_t gzip_trl_bytes = %d;\n\n", GZIP_TRAILER_SIZE);
 230
 231         fprintf(output_file, "const uint8_t zlib_hdr[] = { 0x78, 0x01 };\n\n");
 232         fprintf(output_file, "const uint32_t zlib_hdr_bytes = %d;\n", ZLIB_HEADER_SIZE);
 233         fprintf(output_file, "const uint32_t zlib_trl_bytes = %d;\n", ZLIB_TRAILER_SIZE);
 234 }
 235
 236 int main(int argc, char *argv[])
 237 {
 238         long int file_length;
 239         uint8_t *stream = NULL;
 240         struct isal_hufftables hufftables;
 241         struct isal_huff_histogram histogram;
 242         struct isal_zstream tmp_stream;
 243         FILE *file;
 244
 245         if (argc == 1) {
 246                 printf("Error, no input file.\n");
 247                 return 1;
 248         }
 249
 250         memset(&histogram, 0, sizeof(histogram));       /* Initialize histograms. */
 251
 252         while (argc > 1) {
 253                 printf("Processing %s\n", argv[argc - 1]);
 254                 file = fopen(argv[argc - 1], "r");
 255                 if (file == NULL) {
 256                         printf("Error opening file\n");
 257                         return 1;
 258                 }
 259                 fseek(file, 0, SEEK_END);
 260                 file_length = ftell(file);
 261                 fseek(file, 0, SEEK_SET);
 262                 file_length -= ftell(file);
 263                 stream = malloc(file_length);
 264                 if (stream == NULL) {
 265                         printf("Failed to allocate memory to read in file\n");
 266                         fclose(file);
 267                         return 1;
 268                 }
 269                 fread(stream, 1, file_length, file);
 270                 if (ferror(file)) {
 271                         printf("Error occurred when reading file");
 272                         fclose(file);
 273                         free(stream);
 274                         return 1;
 275                 }
 276
 277                 /* Create a histogram of frequency of symbols found in stream to
 278                  * generate the huffman tree.*/
 279                 isal_update_histogram(stream, file_length, &histogram);
 280
 281                 fclose(file);
 282                 free(stream);
 283                 argc--;
 284         }
 285
 286         isal_create_hufftables(&hufftables, &histogram);
 287
 288         file = fopen("hufftables_c.c", "w");
 289         if (file == NULL) {
 290                 printf("Error creating file hufftables_c.c\n");
 291                 return 1;
 292         }
 293
 294         fprint_header(file);
 295
 296         fprintf(file, "\n");
 297
 298         fprint_hufftables(file, "hufftables_default", &hufftables);
 299
 300         fprintf(file, "\n");
 301
 302         isal_deflate_stateless_init(&tmp_stream);
 303         isal_deflate_set_hufftables(&tmp_stream, NULL, IGZIP_HUFFTABLE_STATIC);
 304         fprint_hufftables(file, "hufftables_static", tmp_stream.hufftables);
 305
 306         fclose(file);
 307
 308         return 0;
 309 }