]>
Commit | Line | Data |
---|---|---|
9f95a23c TL |
1 | /********************************************************************** |
2 | Copyright(c) 2011-2016 Intel Corporation All rights reserved. | |
3 | ||
4 | Redistribution and use in source and binary forms, with or without | |
5 | modification, are permitted provided that the following conditions | |
6 | are met: | |
7 | * Redistributions of source code must retain the above copyright | |
8 | notice, this list of conditions and the following disclaimer. | |
9 | * Redistributions in binary form must reproduce the above copyright | |
10 | notice, this list of conditions and the following disclaimer in | |
11 | the documentation and/or other materials provided with the | |
12 | distribution. | |
13 | * Neither the name of Intel Corporation nor the names of its | |
14 | contributors may be used to endorse or promote products derived | |
15 | from this software without specific prior written permission. | |
16 | ||
17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
28 | **********************************************************************/ | |
29 | ||
30 | /* This program can be used to generate custom a custom huffman encoding to get | |
31 | * better data compression. This is most useful when the type of data being | |
32 | * compressed is well known. | |
33 | * | |
34 | * To use generate_custom_hufftables, pass a sequence of files to the program | |
35 | * that together form an accurate representation of the data that is being | |
36 | * compressed. Generate_custom_hufftables will then produce the file | |
37 | * hufftables_c.c, which should be moved to replace its counterpart in the igzip | |
38 | * source folder. After recompiling the Isa-l library, the igzip compression | |
39 | * functions will use the new hufftables. | |
40 | * | |
41 | * Generate_custom_hufftables should be compiled with the same compile time | |
42 | * parameters as the igzip source code. Generating custom hufftables with | |
43 | * different compile time parameters may cause igzip to produce invalid output | |
44 | * for the reasons described below. The default parameters used by | |
45 | * generate_custom_hufftables are the same as the default parameters used by | |
46 | * igzip. | |
47 | * | |
48 | * *WARNING* generate custom hufftables must be compiled with a IGZIP_HIST_SIZE | |
49 | * that is at least as large as the IGZIP_HIST_SIZE used by igzip. By default | |
50 | * IGZIP_HIST_SIZE is 32K, the maximum usable IGZIP_HIST_SIZE is 32K. The reason | |
51 | * for this is to generate better compression. Igzip cannot produce look back | |
52 | * distances with sizes larger than the IGZIP_HIST_SIZE igzip was compiled with, | |
53 | * so look back distances with sizes larger than IGZIP_HIST_SIZE are not | |
54 | * assigned a huffman code. The definition of LONGER_HUFFTABLES must be | |
55 | * consistent as well since that definition changes the size of the structures | |
56 | * printed by this tool. | |
57 | * | |
58 | */ | |
59 | ||
60 | #include <stdint.h> | |
61 | #include <stdio.h> | |
62 | #include <inttypes.h> | |
63 | #include <string.h> | |
64 | #include <stdlib.h> | |
65 | #include "igzip_lib.h" | |
66 | ||
67 | /*These max code lengths are limited by how the data is stored in | |
68 | * hufftables.asm. The deflate standard max is 15.*/ | |
69 | ||
70 | #define MAX_HEADER_SIZE ISAL_DEF_MAX_HDR_SIZE | |
71 | ||
72 | #define GZIP_HEADER_SIZE 10 | |
73 | #define GZIP_TRAILER_SIZE 8 | |
74 | #define ZLIB_HEADER_SIZE 2 | |
75 | #define ZLIB_TRAILER_SIZE 4 | |
76 | ||
77 | /** | |
78 | * @brief Prints a table of uint8_t elements to a file. | |
79 | * @param outfile: the file the table is printed to. | |
80 | * @param table: the table to be printed. | |
81 | * @param length: number of elements to be printed. | |
82 | * @param header: header to append in front of the table. | |
83 | * @param footer: footer to append at the end of the table. | |
84 | * @param begin_line: string printed at beginning of new line | |
85 | */ | |
86 | void fprint_uint8_table(FILE * outfile, uint8_t * table, uint64_t length, char *header, | |
87 | char *footer, char *begin_line) | |
88 | { | |
89 | int i; | |
90 | fprintf(outfile, "%s", header); | |
91 | for (i = 0; i < length - 1; i++) { | |
92 | if ((i & 7) == 0) | |
93 | fprintf(outfile, "\n%s", begin_line); | |
94 | else | |
95 | fprintf(outfile, " "); | |
96 | fprintf(outfile, "0x%02x,", table[i]); | |
97 | } | |
98 | ||
99 | if ((i & 7) == 0) | |
100 | fprintf(outfile, "\n%s", begin_line); | |
101 | else | |
102 | fprintf(outfile, " "); | |
103 | fprintf(outfile, "0x%02x", table[i]); | |
104 | fprintf(outfile, "%s", footer); | |
105 | ||
106 | } | |
107 | ||
108 | /** | |
109 | * @brief Prints a table of uint16_t elements to a file. | |
110 | * @param outfile: the file the table is printed to. | |
111 | * @param table: the table to be printed. | |
112 | * @param length: number of elements to be printed. | |
113 | * @param header: header to append in front of the table. | |
114 | * @param footer: footer to append at the end of the table. | |
115 | * @param begin_line: string printed at beginning of new line | |
116 | */ | |
117 | void fprint_uint16_table(FILE * outfile, uint16_t * table, uint64_t length, char *header, | |
118 | char *footer, char *begin_line) | |
119 | { | |
120 | int i; | |
121 | fprintf(outfile, "%s", header); | |
122 | for (i = 0; i < length - 1; i++) { | |
123 | if ((i & 7) == 0) | |
124 | fprintf(outfile, "\n%s", begin_line); | |
125 | else | |
126 | fprintf(outfile, " "); | |
127 | fprintf(outfile, "0x%04x,", table[i]); | |
128 | } | |
129 | ||
130 | if ((i & 7) == 0) | |
131 | fprintf(outfile, "\n%s", begin_line); | |
132 | else | |
133 | fprintf(outfile, " "); | |
134 | fprintf(outfile, "0x%04x", table[i]); | |
135 | fprintf(outfile, "%s", footer); | |
136 | ||
137 | } | |
138 | ||
139 | /** | |
140 | * @brief Prints a table of uint32_t elements to a file. | |
141 | * @param outfile: the file the table is printed to. | |
142 | * @param table: the table to be printed. | |
143 | * @param length: number of elements to be printed. | |
144 | * @param header: header to append in front of the table. | |
145 | * @param footer: footer to append at the end of the table. | |
146 | * @param begin_line: string printed at beginning of new line | |
147 | */ | |
148 | void fprint_uint32_table(FILE * outfile, uint32_t * table, uint64_t length, char *header, | |
149 | char *footer, char *begin_line) | |
150 | { | |
151 | int i; | |
152 | fprintf(outfile, "%s", header); | |
153 | for (i = 0; i < length - 1; i++) { | |
154 | if ((i & 3) == 0) | |
155 | fprintf(outfile, "\n%s", begin_line); | |
156 | else | |
157 | fprintf(outfile, " "); | |
158 | fprintf(outfile, "0x%08x,", table[i]); | |
159 | } | |
160 | ||
161 | if ((i & 3) == 0) | |
162 | fprintf(outfile, "%s", begin_line); | |
163 | else | |
164 | fprintf(outfile, " "); | |
165 | fprintf(outfile, "0x%08x", table[i]); | |
166 | fprintf(outfile, "%s", footer); | |
167 | ||
168 | } | |
169 | ||
170 | void fprint_hufftables(FILE * output_file, char *hufftables_name, | |
171 | struct isal_hufftables *hufftables) | |
172 | { | |
173 | fprintf(output_file, "struct isal_hufftables %s = {\n\n", hufftables_name); | |
174 | ||
175 | fprint_uint8_table(output_file, hufftables->deflate_hdr, | |
176 | hufftables->deflate_hdr_count + | |
177 | (hufftables->deflate_hdr_extra_bits + 7) / 8, | |
178 | "\t.deflate_hdr = {", "},\n\n", "\t\t"); | |
179 | ||
180 | fprintf(output_file, "\t.deflate_hdr_count = %d,\n", hufftables->deflate_hdr_count); | |
181 | fprintf(output_file, "\t.deflate_hdr_extra_bits = %d,\n\n", | |
182 | hufftables->deflate_hdr_extra_bits); | |
183 | ||
184 | fprint_uint32_table(output_file, hufftables->dist_table, IGZIP_DIST_TABLE_SIZE, | |
185 | "\t.dist_table = {", "},\n\n", "\t\t"); | |
186 | ||
187 | fprint_uint32_table(output_file, hufftables->len_table, IGZIP_LEN_TABLE_SIZE, | |
188 | "\t.len_table = {", "},\n\n", "\t\t"); | |
189 | ||
190 | fprint_uint16_table(output_file, hufftables->lit_table, IGZIP_LIT_TABLE_SIZE, | |
191 | "\t.lit_table = {", "},\n\n", "\t\t"); | |
192 | fprint_uint8_table(output_file, hufftables->lit_table_sizes, IGZIP_LIT_TABLE_SIZE, | |
193 | "\t.lit_table_sizes = {", "},\n\n", "\t\t"); | |
194 | ||
195 | fprint_uint16_table(output_file, hufftables->dcodes, | |
196 | ISAL_DEF_DIST_SYMBOLS - IGZIP_DECODE_OFFSET, | |
197 | "\t.dcodes = {", "},\n\n", "\t\t"); | |
198 | fprint_uint8_table(output_file, hufftables->dcodes_sizes, | |
199 | ISAL_DEF_DIST_SYMBOLS - IGZIP_DECODE_OFFSET, | |
200 | "\t.dcodes_sizes = {", "}\n", "\t\t"); | |
201 | fprintf(output_file, "};\n"); | |
202 | } | |
203 | ||
204 | void fprint_header(FILE * output_file) | |
205 | { | |
206 | ||
207 | fprintf(output_file, "#include <stdint.h>\n"); | |
208 | fprintf(output_file, "#include <igzip_lib.h>\n\n"); | |
209 | ||
210 | fprintf(output_file, "#if IGZIP_HIST_SIZE > %d\n" | |
211 | "# error \"Invalid history size for the custom hufftable\"\n" | |
212 | "#endif\n", IGZIP_HIST_SIZE); | |
213 | ||
214 | #ifdef LONGER_HUFFTABLE | |
215 | fprintf(output_file, "#ifndef LONGER_HUFFTABLE\n" | |
216 | "# error \"Custom hufftable requires LONGER_HUFFTABLE to be defined \"\n" | |
217 | "#endif\n"); | |
218 | #else | |
219 | fprintf(output_file, "#ifdef LONGER_HUFFTABLE\n" | |
220 | "# error \"Custom hufftable requires LONGER_HUFFTABLE to not be defined \"\n" | |
221 | "#endif\n"); | |
222 | #endif | |
223 | fprintf(output_file, "\n"); | |
224 | ||
225 | fprintf(output_file, "const uint8_t gzip_hdr[] = {\n" | |
226 | "\t0x1f, 0x8b, 0x08, 0x00, 0x00,\n" "\t0x00, 0x00, 0x00, 0x00, 0xff\t};\n\n"); | |
227 | ||
228 | fprintf(output_file, "const uint32_t gzip_hdr_bytes = %d;\n", GZIP_HEADER_SIZE); | |
229 | fprintf(output_file, "const uint32_t gzip_trl_bytes = %d;\n\n", GZIP_TRAILER_SIZE); | |
230 | ||
231 | fprintf(output_file, "const uint8_t zlib_hdr[] = { 0x78, 0x01 };\n\n"); | |
232 | fprintf(output_file, "const uint32_t zlib_hdr_bytes = %d;\n", ZLIB_HEADER_SIZE); | |
233 | fprintf(output_file, "const uint32_t zlib_trl_bytes = %d;\n", ZLIB_TRAILER_SIZE); | |
234 | } | |
235 | ||
236 | int main(int argc, char *argv[]) | |
237 | { | |
238 | long int file_length; | |
239 | uint8_t *stream = NULL; | |
240 | struct isal_hufftables hufftables; | |
241 | struct isal_huff_histogram histogram; | |
242 | struct isal_zstream tmp_stream; | |
243 | FILE *file; | |
244 | ||
245 | if (argc == 1) { | |
246 | printf("Error, no input file.\n"); | |
247 | return 1; | |
248 | } | |
249 | ||
250 | memset(&histogram, 0, sizeof(histogram)); /* Initialize histograms. */ | |
251 | ||
252 | while (argc > 1) { | |
253 | printf("Processing %s\n", argv[argc - 1]); | |
254 | file = fopen(argv[argc - 1], "r"); | |
255 | if (file == NULL) { | |
256 | printf("Error opening file\n"); | |
257 | return 1; | |
258 | } | |
259 | fseek(file, 0, SEEK_END); | |
260 | file_length = ftell(file); | |
261 | fseek(file, 0, SEEK_SET); | |
262 | file_length -= ftell(file); | |
263 | stream = malloc(file_length); | |
264 | if (stream == NULL) { | |
265 | printf("Failed to allocate memory to read in file\n"); | |
266 | fclose(file); | |
267 | return 1; | |
268 | } | |
269 | fread(stream, 1, file_length, file); | |
270 | if (ferror(file)) { | |
271 | printf("Error occurred when reading file"); | |
272 | fclose(file); | |
273 | free(stream); | |
274 | return 1; | |
275 | } | |
276 | ||
277 | /* Create a histogram of frequency of symbols found in stream to | |
278 | * generate the huffman tree.*/ | |
279 | isal_update_histogram(stream, file_length, &histogram); | |
280 | ||
281 | fclose(file); | |
282 | free(stream); | |
283 | argc--; | |
284 | } | |
285 | ||
286 | isal_create_hufftables(&hufftables, &histogram); | |
287 | ||
288 | file = fopen("hufftables_c.c", "w"); | |
289 | if (file == NULL) { | |
290 | printf("Error creating file hufftables_c.c\n"); | |
291 | return 1; | |
292 | } | |
293 | ||
294 | fprint_header(file); | |
295 | ||
296 | fprintf(file, "\n"); | |
297 | ||
298 | fprint_hufftables(file, "hufftables_default", &hufftables); | |
299 | ||
300 | fprintf(file, "\n"); | |
301 | ||
302 | isal_deflate_stateless_init(&tmp_stream); | |
303 | isal_deflate_set_hufftables(&tmp_stream, NULL, IGZIP_HUFFTABLE_STATIC); | |
304 | fprint_hufftables(file, "hufftables_static", tmp_stream.hufftables); | |
305 | ||
306 | fclose(file); | |
307 | ||
308 | return 0; | |
309 | } |