ceph/src/isa-l/igzip/generate_custom_hufftables.c

   1 /**********************************************************************
   2   Copyright(c) 2011-2016 Intel Corporation All rights reserved.
   3
   4   Redistribution and use in source and binary forms, with or without
   5   modification, are permitted provided that the following conditions
   6   are met:
   7     * Redistributions of source code must retain the above copyright
   8       notice, this list of conditions and the following disclaimer.
   9     * Redistributions in binary form must reproduce the above copyright
  10       notice, this list of conditions and the following disclaimer in
  11       the documentation and/or other materials provided with the
  12       distribution.
  13     * Neither the name of Intel Corporation nor the names of its
  14       contributors may be used to endorse or promote products derived
  15       from this software without specific prior written permission.
  16
  17   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 **********************************************************************/
  29
  30 /* This program can be used to generate custom a custom huffman encoding to get
  31  * better data compression. This is most useful when the type of data being
  32  * compressed is well known.
  33  *
  34  * To use generate_custom_hufftables, pass a sequence of files to the program
  35  * that together form an accurate representation of the data that is being
  36  * compressed. Generate_custom_hufftables will then produce the file
  37  * hufftables_c.c, which should be moved to replace its counterpart in the igzip
  38  * source folder. After recompiling the Isa-l library, the igzip compression
  39  * functions will use the new hufftables.
  40  *
  41  * Generate_custom_hufftables should be compiled with the same compile time
  42  * parameters as the igzip source code. Generating custom hufftables with
  43  * different compile time parameters may cause igzip to produce invalid output
  44  * for the reasons described below. The default parameters used by
  45  * generate_custom_hufftables are the same as the default parameters used by
  46  * igzip.
  47  *
  48  * *WARNING* generate custom hufftables must be compiled with a IGZIP_HIST_SIZE
  49  * that is at least as large as the IGZIP_HIST_SIZE used by igzip. By default
  50  * IGZIP_HIST_SIZE is 32K, the maximum usable IGZIP_HIST_SIZE is 32K. The reason
  51  * for this is to generate better compression. Igzip cannot produce look back
  52  * distances with sizes larger than the IGZIP_HIST_SIZE igzip was compiled with,
  53  * so look back distances with sizes larger than IGZIP_HIST_SIZE are not
  54  * assigned a huffman code. The definition of LONGER_HUFFTABLES must be
  55  * consistent as well since that definition changes the size of the structures
  56  * printed by this tool.
  57  *
  58  */
  59
  60 #include <stdint.h>
  61 #include <stdio.h>
  62 #include <inttypes.h>
  63 #include <string.h>
  64 #include <stdlib.h>
  65 #include "igzip_lib.h"
  66
  67 /*These max code lengths are limited by how the data is stored in
  68  * hufftables.asm. The deflate standard max is 15.*/
  69
  70 #define MAX_HEADER_SIZE ISAL_DEF_MAX_HDR_SIZE
  71
  72 #define GZIP_HEADER_SIZE 10
  73 #define GZIP_TRAILER_SIZE 8
  74
  75 /**
  76  * @brief Prints a table of uint8_t elements to a file.
  77  * @param outfile: the file the table is printed to.
  78  * @param table: the table to be printed.
  79  * @param length: number of elements to be printed.
  80  * @param header: header to append in front of the table.
  81  * @param footer: footer to append at the end of the table.
  82  * @param begin_line: string printed at beginning of new line
  83  */
  84 void fprint_uint8_table(FILE * outfile, uint8_t * table, uint64_t length, char *header,
  85                         char *footer, char *begin_line)
  86 {
  87         int i;
  88         fprintf(outfile, "%s", header);
  89         for (i = 0; i < length - 1; i++) {
  90                 if ((i & 7) == 0)
  91                         fprintf(outfile, "\n%s", begin_line);
  92                 else
  93                         fprintf(outfile, " ");
  94                 fprintf(outfile, "0x%02x,", table[i]);
  95         }
  96
  97         if ((i & 7) == 0)
  98                 fprintf(outfile, "\n%s", begin_line);
  99         else
 100                 fprintf(outfile, " ");
 101         fprintf(outfile, "0x%02x", table[i]);
 102         fprintf(outfile, "%s", footer);
 103
 104 }
 105
 106 /**
 107  * @brief Prints a table of uint16_t elements to a file.
 108  * @param outfile: the file the table is printed to.
 109  * @param table: the table to be printed.
 110  * @param length: number of elements to be printed.
 111  * @param header: header to append in front of the table.
 112  * @param footer: footer to append at the end of the table.
 113  * @param begin_line: string printed at beginning of new line
 114  */
 115 void fprint_uint16_table(FILE * outfile, uint16_t * table, uint64_t length, char *header,
 116                          char *footer, char *begin_line)
 117 {
 118         int i;
 119         fprintf(outfile, "%s", header);
 120         for (i = 0; i < length - 1; i++) {
 121                 if ((i & 7) == 0)
 122                         fprintf(outfile, "\n%s", begin_line);
 123                 else
 124                         fprintf(outfile, " ");
 125                 fprintf(outfile, "0x%04x,", table[i]);
 126         }
 127
 128         if ((i & 7) == 0)
 129                 fprintf(outfile, "\n%s", begin_line);
 130         else
 131                 fprintf(outfile, " ");
 132         fprintf(outfile, "0x%04x", table[i]);
 133         fprintf(outfile, "%s", footer);
 134
 135 }
 136
 137 /**
 138  * @brief Prints a table of uint32_t elements to a file.
 139  * @param outfile: the file the table is printed to.
 140  * @param table: the table to be printed.
 141  * @param length: number of elements to be printed.
 142  * @param header: header to append in front of the table.
 143  * @param footer: footer to append at the end of the table.
 144  * @param begin_line: string printed at beginning of new line
 145  */
 146 void fprint_uint32_table(FILE * outfile, uint32_t * table, uint64_t length, char *header,
 147                          char *footer, char *begin_line)
 148 {
 149         int i;
 150         fprintf(outfile, "%s", header);
 151         for (i = 0; i < length - 1; i++) {
 152                 if ((i & 3) == 0)
 153                         fprintf(outfile, "\n%s", begin_line);
 154                 else
 155                         fprintf(outfile, " ");
 156                 fprintf(outfile, "0x%08x,", table[i]);
 157         }
 158
 159         if ((i & 3) == 0)
 160                 fprintf(outfile, "%s", begin_line);
 161         else
 162                 fprintf(outfile, " ");
 163         fprintf(outfile, "0x%08x", table[i]);
 164         fprintf(outfile, "%s", footer);
 165
 166 }
 167
 168 void fprint_hufftables(FILE * output_file, char *hufftables_name,
 169                        struct isal_hufftables *hufftables)
 170 {
 171         fprintf(output_file, "struct isal_hufftables %s = {\n\n", hufftables_name);
 172
 173         fprint_uint8_table(output_file, hufftables->deflate_hdr,
 174                            hufftables->deflate_hdr_count +
 175                            (hufftables->deflate_hdr_extra_bits + 7) / 8,
 176                            "\t.deflate_hdr = {", "},\n\n", "\t\t");
 177
 178         fprintf(output_file, "\t.deflate_hdr_count = %d,\n", hufftables->deflate_hdr_count);
 179         fprintf(output_file, "\t.deflate_hdr_extra_bits = %d,\n\n",
 180                 hufftables->deflate_hdr_extra_bits);
 181
 182         fprint_uint32_table(output_file, hufftables->dist_table, IGZIP_DIST_TABLE_SIZE,
 183                             "\t.dist_table = {", "},\n\n", "\t\t");
 184
 185         fprint_uint32_table(output_file, hufftables->len_table, IGZIP_LEN_TABLE_SIZE,
 186                             "\t.len_table = {", "},\n\n", "\t\t");
 187
 188         fprint_uint16_table(output_file, hufftables->lit_table, IGZIP_LIT_TABLE_SIZE,
 189                             "\t.lit_table = {", "},\n\n", "\t\t");
 190         fprint_uint8_table(output_file, hufftables->lit_table_sizes, IGZIP_LIT_TABLE_SIZE,
 191                            "\t.lit_table_sizes = {", "},\n\n", "\t\t");
 192
 193         fprint_uint16_table(output_file, hufftables->dcodes,
 194                             ISAL_DEF_DIST_SYMBOLS - IGZIP_DECODE_OFFSET,
 195                             "\t.dcodes = {", "},\n\n", "\t\t");
 196         fprint_uint8_table(output_file, hufftables->dcodes_sizes,
 197                            ISAL_DEF_DIST_SYMBOLS - IGZIP_DECODE_OFFSET,
 198                            "\t.dcodes_sizes = {", "}\n", "\t\t");
 199         fprintf(output_file, "};\n");
 200 }
 201
 202 void fprint_header(FILE * output_file)
 203 {
 204
 205         fprintf(output_file, "#include <stdint.h>\n");
 206         fprintf(output_file, "#include <igzip_lib.h>\n\n");
 207
 208         fprintf(output_file, "#if IGZIP_HIST_SIZE > %d\n"
 209                 "# error \"Invalid history size for the custom hufftable\"\n"
 210                 "#endif\n", IGZIP_HIST_SIZE);
 211
 212 #ifdef LONGER_HUFFTABLE
 213         fprintf(output_file, "#ifndef LONGER_HUFFTABLE\n"
 214                 "# error \"Custom hufftable requires LONGER_HUFFTABLE to be defined \"\n"
 215                 "#endif\n");
 216 #else
 217         fprintf(output_file, "#ifdef LONGER_HUFFTABLE\n"
 218                 "# error \"Custom hufftable requires LONGER_HUFFTABLE to not be defined \"\n"
 219                 "#endif\n");
 220 #endif
 221         fprintf(output_file, "\n");
 222
 223         fprintf(output_file, "const uint8_t gzip_hdr[] = {\n"
 224                 "\t0x1f, 0x8b, 0x08, 0x00, 0x00,\n" "\t0x00, 0x00, 0x00, 0x00, 0xff\t};\n\n");
 225
 226         fprintf(output_file, "const uint32_t gzip_hdr_bytes = %d;\n", GZIP_HEADER_SIZE);
 227         fprintf(output_file, "const uint32_t gzip_trl_bytes = %d;\n", GZIP_TRAILER_SIZE);
 228 }
 229
 230 int main(int argc, char *argv[])
 231 {
 232         long int file_length;
 233         uint8_t *stream = NULL;
 234         struct isal_hufftables hufftables;
 235         struct isal_huff_histogram histogram;
 236         struct isal_zstream tmp_stream;
 237         FILE *file;
 238
 239         if (argc == 1) {
 240                 printf("Error, no input file.\n");
 241                 return 1;
 242         }
 243
 244         memset(&histogram, 0, sizeof(histogram));       /* Initialize histograms. */
 245
 246         while (argc > 1) {
 247                 printf("Processing %s\n", argv[argc - 1]);
 248                 file = fopen(argv[argc - 1], "r");
 249                 if (file == NULL) {
 250                         printf("Error opening file\n");
 251                         return 1;
 252                 }
 253                 fseek(file, 0, SEEK_END);
 254                 file_length = ftell(file);
 255                 fseek(file, 0, SEEK_SET);
 256                 file_length -= ftell(file);
 257                 stream = malloc(file_length);
 258                 if (stream == NULL) {
 259                         printf("Failed to allocate memory to read in file\n");
 260                         fclose(file);
 261                         return 1;
 262                 }
 263                 fread(stream, 1, file_length, file);
 264                 if (ferror(file)) {
 265                         printf("Error occurred when reading file");
 266                         fclose(file);
 267                         free(stream);
 268                         return 1;
 269                 }
 270
 271                 /* Create a histogram of frequency of symbols found in stream to
 272                  * generate the huffman tree.*/
 273                 isal_update_histogram(stream, file_length, &histogram);
 274
 275                 fclose(file);
 276                 free(stream);
 277                 argc--;
 278         }
 279
 280         isal_create_hufftables(&hufftables, &histogram);
 281
 282         file = fopen("hufftables_c.c", "w");
 283         if (file == NULL) {
 284                 printf("Error creating file hufftables_c.c\n");
 285                 return 1;
 286         }
 287
 288         fprint_header(file);
 289
 290         fprintf(file, "\n");
 291
 292         fprint_hufftables(file, "hufftables_default", &hufftables);
 293
 294         fprintf(file, "\n");
 295
 296         isal_deflate_stateless_init(&tmp_stream);
 297         isal_deflate_set_hufftables(&tmp_stream, NULL, IGZIP_HUFFTABLE_STATIC);
 298         fprint_hufftables(file, "hufftables_static", tmp_stream.hufftables);
 299
 300         fclose(file);
 301
 302         return 0;
 303 }