1 /* Copyright 2014 Google Inc. All Rights Reserved.
3 Distributed under MIT license.
4 See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
7 /* Example main() function for Brotli library. */
14 #include <sys/types.h>
17 #include "../dec/decode.h"
18 #include "../enc/encode.h"
26 #define MAKE_BINARY(FILENO) (_setmode((FILENO), _O_BINARY), (FILENO))
28 #if !defined(__MINGW32__)
29 #define STDIN_FILENO MAKE_BINARY(_fileno(stdin))
30 #define STDOUT_FILENO MAKE_BINARY(_fileno(stdout))
31 #define S_IRUSR S_IREAD
32 #define S_IWUSR S_IWRITE
34 #define fdopen _fdopen
35 #define unlink _unlink
37 #define fopen ms_fopen
40 #if defined(_MSC_VER) && (_MSC_VER >= 1400)
41 #define fseek _fseeki64
42 #define ftell _ftelli64
45 static FILE* ms_fopen(const char *filename
, const char *mode
) {
47 fopen_s(&result
, filename
, mode
);
51 static int ms_open(const char *filename
, int oflag
, int pmode
) {
53 _sopen_s(&result
, filename
, oflag
| O_BINARY
, _SH_DENYNO
, pmode
);
58 static int ParseQuality(const char* s
, int* quality
) {
59 if (s
[0] >= '0' && s
[0] <= '9') {
60 *quality
= s
[0] - '0';
61 if (s
[1] >= '0' && s
[1] <= '9') {
62 *quality
= *quality
* 10 + s
[1] - '0';
63 return (s
[2] == 0) ? 1 : 0;
65 return (s
[1] == 0) ? 1 : 0;
70 static void ParseArgv(int argc
, char **argv
,
73 char **dictionary_path
,
89 size_t argv0_len
= strlen(argv
[0]);
91 argv0_len
>= 5 && strcmp(&argv
[0][argv0_len
- 5], "unbro") == 0;
93 for (k
= 1; k
< argc
; ++k
) {
94 if (!strcmp("--force", argv
[k
]) ||
95 !strcmp("-f", argv
[k
])) {
101 } else if (!strcmp("--decompress", argv
[k
]) ||
102 !strcmp("--uncompress", argv
[k
]) ||
103 !strcmp("-d", argv
[k
])) {
106 } else if (!strcmp("--verbose", argv
[k
]) ||
107 !strcmp("-v", argv
[k
])) {
115 if (!strcmp("--input", argv
[k
]) ||
116 !strcmp("--in", argv
[k
]) ||
117 !strcmp("-i", argv
[k
])) {
118 if (*input_path
!= 0) {
121 *input_path
= argv
[k
+ 1];
124 } else if (!strcmp("--output", argv
[k
]) ||
125 !strcmp("--out", argv
[k
]) ||
126 !strcmp("-o", argv
[k
])) {
127 if (*output_path
!= 0) {
130 *output_path
= argv
[k
+ 1];
133 } else if (!strcmp("--custom-dictionary", argv
[k
])) {
134 if (*dictionary_path
!= 0) {
137 *dictionary_path
= argv
[k
+ 1];
140 } else if (!strcmp("--quality", argv
[k
]) ||
141 !strcmp("-q", argv
[k
])) {
142 if (!ParseQuality(argv
[k
+ 1], quality
)) {
147 } else if (!strcmp("--repeat", argv
[k
]) ||
148 !strcmp("-r", argv
[k
])) {
149 if (!ParseQuality(argv
[k
+ 1], repeat
)) {
154 } else if (!strcmp("--window", argv
[k
]) ||
155 !strcmp("-w", argv
[k
])) {
156 if (!ParseQuality(argv
[k
+ 1], lgwin
)) {
159 if (*lgwin
< 10 || *lgwin
>= 25) {
164 } else if (!strcmp("--gap", argv
[k
]) ||
165 !strcmp("-g", argv
[k
])) {
166 if (!ParseQuality(argv
[k
+ 1], gapmem
)) {
178 "Usage: %s [--force] [--quality n] [--gap n] [--decompress]"
179 " [--input filename] [--output filename] [--repeat iters]"
180 " [--verbose] [--window n] [--custom-dictionary filename]\n",
185 static FILE* OpenInputFile(const char* input_path
) {
187 if (input_path
== 0) {
188 return fdopen(STDIN_FILENO
, "rb");
190 f
= fopen(input_path
, "rb");
198 static FILE *OpenOutputFile(const char *output_path
, const int force
) {
200 if (output_path
== 0) {
201 return fdopen(STDOUT_FILENO
, "wb");
203 fd
= open(output_path
, O_CREAT
| (force
? 0 : O_EXCL
) | O_WRONLY
| O_TRUNC
,
208 if (stat(output_path
, &statbuf
) == 0) {
209 fprintf(stderr
, "output file exists\n");
216 return fdopen(fd
, "wb");
219 static int64_t FileSize(const char *path
) {
220 FILE *f
= fopen(path
, "rb");
225 if (fseek(f
, 0L, SEEK_END
) != 0) {
230 if (fclose(f
) != 0) {
236 /* Brotli specified memory allocate function */
237 static void *BrAlloc (void *memsize
, size_t size
) {
238 *(int64_t *)memsize
+= size
;
242 /* Brotli specified memory free function */
243 static void BrFree (void *memsize
, void *ptr
) {
247 /* Result ownersip is passed to caller.
248 |*dictionary_size| is set to resulting buffer size. */
249 static uint8_t* ReadDictionary(const char* path
, size_t* dictionary_size
, void *memsize
) {
250 static const int kMaxDictionarySize
= (1 << 24) - 16;
251 FILE *f
= fopen(path
, "rb");
252 int64_t file_size_64
;
261 file_size_64
= FileSize(path
);
262 if (file_size_64
== -1) {
263 fprintf(stderr
, "could not get size of dictionary file");
267 if (file_size_64
> kMaxDictionarySize
) {
268 fprintf(stderr
, "dictionary is larger than maximum allowed: %d\n",
272 *dictionary_size
= (size_t)file_size_64
;
274 buffer
= (uint8_t *)BrAlloc(memsize
, *dictionary_size
);
276 fprintf(stderr
, "could not read dictionary: out of memory\n");
279 bytes_read
= fread(buffer
, sizeof(uint8_t), *dictionary_size
, f
);
280 if (bytes_read
!= *dictionary_size
) {
281 fprintf(stderr
, "could not read dictionary\n");
288 static const size_t kFileBufferSize
= 65536;
290 static int Decompress(FILE* fin
, FILE* fout
, const char* dictionary_path
, void *memsize
) {
291 /* Dictionary should be kept during first rounds of decompression. */
292 uint8_t* dictionary
= NULL
;
297 const uint8_t* next_in
;
298 size_t available_out
= kFileBufferSize
;
300 BrotliResult result
= BROTLI_RESULT_ERROR
;
301 BrotliState
* s
= BrotliCreateState(BrAlloc
, BrFree
, memsize
);
303 fprintf(stderr
, "out of memory\n");
306 if (dictionary_path
!= NULL
) {
307 size_t dictionary_size
= 0;
308 dictionary
= ReadDictionary(dictionary_path
, &dictionary_size
, memsize
);
309 BrotliSetCustomDictionary(dictionary_size
, dictionary
, s
);
311 input
= (uint8_t*)BrAlloc(memsize
, kFileBufferSize
);
312 output
= (uint8_t*)BrAlloc(memsize
, kFileBufferSize
);
313 if (!input
|| !output
) {
314 fprintf(stderr
, "out of memory\n");
318 result
= BROTLI_RESULT_NEEDS_MORE_INPUT
;
320 if (result
== BROTLI_RESULT_NEEDS_MORE_INPUT
) {
324 available_in
= fread(input
, 1, kFileBufferSize
, fin
);
329 } else if (result
== BROTLI_RESULT_NEEDS_MORE_OUTPUT
) {
330 fwrite(output
, 1, kFileBufferSize
, fout
);
334 available_out
= kFileBufferSize
;
337 break; /* Error or success. */
339 result
= BrotliDecompressStream(&available_in
, &next_in
,
340 &available_out
, &next_out
, &total_out
, s
);
342 if (next_out
!= output
) {
343 fwrite(output
, 1, (size_t)(next_out
- output
), fout
);
346 if ((result
== BROTLI_RESULT_NEEDS_MORE_OUTPUT
) || ferror(fout
)) {
347 fprintf(stderr
, "failed to write output\n");
348 } else if (result
!= BROTLI_RESULT_SUCCESS
) { /* Error or needs more input. */
349 fprintf(stderr
, "corrupt input\n");
353 BrFree(memsize
, dictionary
);
354 BrFree(memsize
, input
);
355 BrFree(memsize
, output
);
356 BrotliDestroyState(s
);
357 return (result
== BROTLI_RESULT_SUCCESS
) ? 1 : 0;
360 static int Compress(int quality
, int lgwin
, FILE* fin
, FILE* fout
,
361 const char *dictionary_path
, void *memsize
) {
362 BrotliEncoderState
* s
= BrotliEncoderCreateInstance(BrAlloc
, BrFree
, memsize
);
363 uint8_t* buffer
= (uint8_t*)BrAlloc(memsize
, kFileBufferSize
<< 1);
364 uint8_t* input
= buffer
;
365 uint8_t* output
= buffer
+ kFileBufferSize
;
366 size_t available_in
= 0;
367 const uint8_t* next_in
= NULL
;
368 size_t available_out
= kFileBufferSize
;
369 uint8_t* next_out
= output
;
378 BrotliEncoderSetParameter(s
, BROTLI_PARAM_QUALITY
, (uint32_t)quality
);
379 BrotliEncoderSetParameter(s
, BROTLI_PARAM_LGWIN
, (uint32_t)lgwin
);
380 if (dictionary_path
!= NULL
) {
381 size_t dictionary_size
= 0;
382 uint8_t* dictionary
= ReadDictionary(dictionary_path
, &dictionary_size
, memsize
);
383 BrotliEncoderSetCustomDictionary(s
, dictionary_size
, dictionary
);
384 BrFree(memsize
, dictionary
);
388 if (available_in
== 0 && !is_eof
) {
389 available_in
= fread(input
, 1, kFileBufferSize
, fin
);
391 if (ferror(fin
)) break;
395 if (!BrotliEncoderCompressStream(s
,
396 is_eof
? BROTLI_OPERATION_FINISH
: BROTLI_OPERATION_PROCESS
,
397 &available_in
, &next_in
, &available_out
, &next_out
, NULL
)) {
402 if (available_out
!= kFileBufferSize
) {
403 size_t out_size
= kFileBufferSize
- available_out
;
404 fwrite(output
, 1, out_size
, fout
);
405 if (ferror(fout
)) break;
406 available_out
= kFileBufferSize
;
410 if (BrotliEncoderIsFinished(s
)) break;
414 BrFree(memsize
, buffer
);
415 BrotliEncoderDestroyInstance(s
);
418 /* Should detect OOM? */
419 fprintf(stderr
, "failed to compress data\n");
421 } else if (ferror(fout
)) {
422 fprintf(stderr
, "failed to write output\n");
424 } else if (ferror(fin
)) {
425 fprintf(stderr
, "failed to read input\n");
431 #define GAP_MEM_BLOCK 4096
433 int main(int argc
, char** argv
) {
434 char *input_path
= 0;
435 char *output_path
= 0;
436 char *dictionary_path
= 0;
446 int64_t originsize
= 0;
448 ParseArgv(argc
, argv
, &input_path
, &output_path
, &dictionary_path
, &force
,
449 &quality
, &gmem
, &decompress
, &repeat
, &verbose
, &lgwin
);
450 clock_start
= clock();
451 for (i
= 0; i
< repeat
; ++i
) {
452 FILE* fin
= OpenInputFile(input_path
);
453 FILE* fout
= OpenOutputFile(output_path
, force
|| repeat
);
456 if (fseek(fin
, 16, SEEK_SET
) != 0) {
460 is_ok
= Decompress(fin
, fout
, dictionary_path
, (void *)&msize
);
462 originsize
= FileSize(input_path
); /* get original file size */
463 fwrite(&originsize
, 1, sizeof(int64_t), fout
); /* add in original binary file size */
464 fwrite(&msize
, 1, sizeof(int64_t), fout
); /* add in dummy decompression required memory size */
465 is_ok
= Compress(quality
, lgwin
, fin
, fout
, dictionary_path
, (void *)&msize
);
471 if (fclose(fin
) != 0) {
475 if (fclose(fout
) != 0) {
479 /* after compression operation then execute decompression operation
480 to get decompression required memory size. */
481 if (decompress
== 0) {
482 fin
= OpenInputFile(output_path
);
485 if (fseek(fin
, 16, SEEK_SET
) != 0) {
489 is_ok
= Decompress(fin
, fout
, dictionary_path
, (void *)&msize
);
493 if (fclose(fin
) != 0) {
497 if (fclose(fout
) != 0) {
501 fout
= fopen(output_path
, "rb+"); /* open output_path file and add in head info */
502 /* seek to the offset of decompression required memory size */
503 if (fseek(fout
, 8, SEEK_SET
) != 0) {
507 msize
+= gmem
* GAP_MEM_BLOCK
; /* there is a memory gap between IA32 and X64 environment*/
508 fwrite(&msize
, 1, sizeof(int64_t), fout
); /* update final decompression required memory size */
509 if (fclose(fout
) != 0) {
516 clock_t clock_end
= clock();
517 double duration
= (double)(clock_end
- clock_start
) / CLOCKS_PER_SEC
;
518 int64_t uncompressed_size
;
519 double uncompressed_bytes_in_MB
;
520 if (duration
< 1e-9) {
523 uncompressed_size
= FileSize(decompress
? output_path
: input_path
);
524 if (uncompressed_size
== -1) {
525 fprintf(stderr
, "failed to determine uncompressed file size\n");
528 uncompressed_bytes_in_MB
=
529 (double)(repeat
* uncompressed_size
) / (1024.0 * 1024.0);
531 printf("Brotli decompression speed: ");
533 printf("Brotli compression speed: ");
535 printf("%g MB/s\n", uncompressed_bytes_in_MB
/ duration
);