BaseTools/Source/C/BrotliCompress/tools/bro.c

   1 /* Copyright 2014 Google Inc. All Rights Reserved.
   2
   3    Distributed under MIT license.
   4    See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
   5 */
   6
   7 /* Example main() function for Brotli library. */
   8
   9 #include <fcntl.h>
  10 #include <stdio.h>
  11 #include <stdlib.h>
  12 #include <string.h>
  13 #include <sys/stat.h>
  14 #include <sys/types.h>
  15 #include <time.h>
  16 #include <Common/BuildVersion.h>
  17
  18 #include "../dec/decode.h"
  19 #include "../enc/encode.h"
  20
  21 #if !defined(_WIN32)
  22 #include <unistd.h>
  23 #else
  24 #include <io.h>
  25 #include <share.h>
  26
  27 #define MAKE_BINARY(FILENO) (_setmode((FILENO), _O_BINARY), (FILENO))
  28
  29 #if !defined(__MINGW32__)
  30 #define STDIN_FILENO MAKE_BINARY(_fileno(stdin))
  31 #define STDOUT_FILENO MAKE_BINARY(_fileno(stdout))
  32 #define S_IRUSR S_IREAD
  33 #define S_IWUSR S_IWRITE
  34 #endif
  35 #define fdopen _fdopen
  36 #define unlink _unlink
  37
  38 #define fopen ms_fopen
  39 #define open ms_open
  40
  41 #if defined(_MSC_VER) && (_MSC_VER >= 1400)
  42 #define fseek _fseeki64
  43 #define ftell _ftelli64
  44 #endif
  45
  46 static FILE* ms_fopen(const char *filename, const char *mode) {
  47   FILE* result = 0;
  48   fopen_s(&result, filename, mode);
  49   return result;
  50 }
  51
  52 static int ms_open(const char *filename, int oflag, int pmode) {
  53   int result = -1;
  54   _sopen_s(&result, filename, oflag | O_BINARY, _SH_DENYNO, pmode);
  55   return result;
  56 }
  57 #endif  /* WIN32 */
  58
  59 static int ParseQuality(const char* s, int* quality) {
  60   if (s[0] >= '0' && s[0] <= '9') {
  61     *quality = s[0] - '0';
  62     if (s[1] >= '0' && s[1] <= '9') {
  63       *quality = *quality * 10 + s[1] - '0';
  64       return (s[2] == 0) ? 1 : 0;
  65     }
  66     return (s[1] == 0) ? 1 : 0;
  67   }
  68   return 0;
  69 }
  70
  71 #define UTILITY_NAME          "Brotli"
  72 #define UTILITY_MAJOR_VERSION 0
  73 #define UTILITY_MINOR_VERSION 5
  74 #define UTILITY_REVERSION     2
  75
  76 static void ParseArgv(int argc, char **argv,
  77                       char **input_path,
  78                       char **output_path,
  79                       char **dictionary_path,
  80                       int *force,
  81                       int *quality,
  82                       int *gapmem,
  83                       int *decompress,
  84                       int *repeat,
  85                       int *verbose,
  86                       int *lgwin) {
  87   int k;
  88   *force = 0;
  89   *input_path = 0;
  90   *output_path = 0;
  91   *repeat = 1;
  92   *verbose = 0;
  93   *lgwin = 22;
  94   {
  95     size_t argv0_len = strlen(argv[0]);
  96     *decompress =
  97         argv0_len >= 5 && strcmp(&argv[0][argv0_len - 5], "unbro") == 0;
  98   }
  99   for (k = 1; k < argc; ++k) {
 100     if (!strcmp("--force", argv[k]) ||
 101         !strcmp("-f", argv[k])) {
 102       if (*force != 0) {
 103         goto error;
 104       }
 105       *force = 1;
 106       continue;
 107     } else if (!strcmp("--decompress", argv[k]) ||
 108                !strcmp("--uncompress", argv[k]) ||
 109                !strcmp("-d", argv[k])) {
 110       *decompress = 1;
 111       continue;
 112     } else if (!strcmp("--verbose", argv[k]) ||
 113                !strcmp("-v", argv[k])) {
 114       if (*verbose != 0) {
 115         goto error;
 116       }
 117       *verbose = 1;
 118       continue;
 119     } else if (!strcmp("--version", argv[k])) {
 120       fprintf(stderr,
 121               "%s Version %d.%d.%d %s\n",
 122               UTILITY_NAME,
 123               UTILITY_MAJOR_VERSION,
 124               UTILITY_MINOR_VERSION,
 125               UTILITY_REVERSION,
 126               __BUILD_VERSION);
 127       exit(1);
 128     }
 129     if (k < argc - 1) {
 130       if (!strcmp("--input", argv[k]) ||
 131           !strcmp("--in", argv[k]) ||
 132           !strcmp("-i", argv[k])) {
 133         if (*input_path != 0) {
 134           goto error;
 135         }
 136         *input_path = argv[k + 1];
 137         ++k;
 138         continue;
 139       } else if (!strcmp("--output", argv[k]) ||
 140                  !strcmp("--out", argv[k]) ||
 141                  !strcmp("-o", argv[k])) {
 142         if (*output_path != 0) {
 143           goto error;
 144         }
 145         *output_path = argv[k + 1];
 146         ++k;
 147         continue;
 148       } else if (!strcmp("--custom-dictionary", argv[k])) {
 149         if (*dictionary_path != 0) {
 150           goto error;
 151         }
 152         *dictionary_path = argv[k + 1];
 153         ++k;
 154         continue;
 155       } else if (!strcmp("--quality", argv[k]) ||
 156                  !strcmp("-q", argv[k])) {
 157         if (!ParseQuality(argv[k + 1], quality)) {
 158           goto error;
 159         }
 160         ++k;
 161         continue;
 162       } else if (!strcmp("--repeat", argv[k]) ||
 163                  !strcmp("-r", argv[k])) {
 164         if (!ParseQuality(argv[k + 1], repeat)) {
 165           goto error;
 166         }
 167         ++k;
 168         continue;
 169       }  else if (!strcmp("--window", argv[k]) ||
 170                   !strcmp("-w", argv[k])) {
 171         if (!ParseQuality(argv[k + 1], lgwin)) {
 172           goto error;
 173         }
 174         if (*lgwin < 10 || *lgwin >= 25) {
 175           goto error;
 176         }
 177         ++k;
 178         continue;
 179       } else if (!strcmp("--gap", argv[k]) ||
 180                   !strcmp("-g", argv[k])) {
 181         if (!ParseQuality(argv[k + 1], gapmem)) {
 182           goto error;
 183         }
 184         ++k;
 185         continue;
 186       }
 187     }
 188     goto error;
 189   }
 190   return;
 191 error:
 192   fprintf(stderr,
 193           "Usage: %s [--force] [--quality n] [--gap n] [--decompress]"
 194           " [--input filename] [--output filename] [--repeat iters]"
 195           " [--verbose] [--window n] [--custom-dictionary filename]"
 196           " [--version]\n",
 197           argv[0]);
 198   exit(1);
 199 }
 200
 201 static FILE* OpenInputFile(const char* input_path) {
 202   FILE* f;
 203   if (input_path == 0) {
 204     return fdopen(STDIN_FILENO, "rb");
 205   }
 206   f = fopen(input_path, "rb");
 207   if (f == 0) {
 208     perror("fopen");
 209     exit(1);
 210   }
 211   return f;
 212 }
 213
 214 static FILE *OpenOutputFile(const char *output_path, const int force) {
 215   int fd;
 216   if (output_path == 0) {
 217     return fdopen(STDOUT_FILENO, "wb");
 218   }
 219   fd = open(output_path, O_CREAT | (force ? 0 : O_EXCL) | O_WRONLY | O_TRUNC,
 220             S_IRUSR | S_IWUSR);
 221   if (fd < 0) {
 222     if (!force) {
 223       struct stat statbuf;
 224       if (stat(output_path, &statbuf) == 0) {
 225         fprintf(stderr, "output file exists\n");
 226         exit(1);
 227       }
 228     }
 229     perror("open");
 230     exit(1);
 231   }
 232   return fdopen(fd, "wb");
 233 }
 234
 235 static int64_t FileSize(const char *path) {
 236   FILE *f = fopen(path, "rb");
 237   int64_t retval;
 238   if (f == NULL) {
 239     return -1;
 240   }
 241   if (fseek(f, 0L, SEEK_END) != 0) {
 242     fclose(f);
 243     return -1;
 244   }
 245   retval = ftell(f);
 246   if (fclose(f) != 0) {
 247     return -1;
 248   }
 249   return retval;
 250 }
 251
 252 /* Brotli specified memory allocate function */
 253 static void *BrAlloc (void *memsize, size_t size) {
 254   *(int64_t *)memsize += size;
 255   return malloc(size);
 256 }
 257
 258 /* Brotli specified memory free function */
 259 static void BrFree (void *memsize, void *ptr) {
 260   free(ptr);
 261 }
 262
 263 /* Result ownersip is passed to caller.
 264    |*dictionary_size| is set to resulting buffer size. */
 265 static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size, void *memsize) {
 266   static const int kMaxDictionarySize = (1 << 24) - 16;
 267   FILE *f = fopen(path, "rb");
 268   int64_t file_size_64;
 269   uint8_t* buffer;
 270   size_t bytes_read;
 271
 272   if (f == NULL) {
 273     perror("fopen");
 274     exit(1);
 275   }
 276
 277   file_size_64 = FileSize(path);
 278   if (file_size_64 == -1) {
 279     fprintf(stderr, "could not get size of dictionary file");
 280     exit(1);
 281   }
 282
 283   if (file_size_64 > kMaxDictionarySize) {
 284     fprintf(stderr, "dictionary is larger than maximum allowed: %d\n",
 285             kMaxDictionarySize);
 286     exit(1);
 287   }
 288   *dictionary_size = (size_t)file_size_64;
 289
 290   buffer = (uint8_t *)BrAlloc(memsize, *dictionary_size);
 291   if (!buffer) {
 292     fprintf(stderr, "could not read dictionary: out of memory\n");
 293     exit(1);
 294   }
 295   bytes_read = fread(buffer, sizeof(uint8_t), *dictionary_size, f);
 296   if (bytes_read != *dictionary_size) {
 297     fprintf(stderr, "could not read dictionary\n");
 298     exit(1);
 299   }
 300   fclose(f);
 301   return buffer;
 302 }
 303
 304 static const size_t kFileBufferSize = 65536;
 305
 306 static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path, void *memsize) {
 307   /* Dictionary should be kept during first rounds of decompression. */
 308   uint8_t* dictionary = NULL;
 309   uint8_t* input;
 310   uint8_t* output;
 311   size_t total_out;
 312   size_t available_in;
 313   const uint8_t* next_in;
 314   size_t available_out = kFileBufferSize;
 315   uint8_t* next_out;
 316   BrotliResult result = BROTLI_RESULT_ERROR;
 317   BrotliState* s = BrotliCreateState(BrAlloc, BrFree, memsize);
 318   if (!s) {
 319     fprintf(stderr, "out of memory\n");
 320     return 0;
 321   }
 322   if (dictionary_path != NULL) {
 323     size_t dictionary_size = 0;
 324     dictionary = ReadDictionary(dictionary_path, &dictionary_size, memsize);
 325     BrotliSetCustomDictionary(dictionary_size, dictionary, s);
 326   }
 327   input = (uint8_t*)BrAlloc(memsize, kFileBufferSize);
 328   output = (uint8_t*)BrAlloc(memsize, kFileBufferSize);
 329   if (!input || !output) {
 330     fprintf(stderr, "out of memory\n");
 331     goto end;
 332   }
 333   next_out = output;
 334   result = BROTLI_RESULT_NEEDS_MORE_INPUT;
 335   while (1) {
 336     if (result == BROTLI_RESULT_NEEDS_MORE_INPUT) {
 337       if (feof(fin)) {
 338         break;
 339       }
 340       available_in = fread(input, 1, kFileBufferSize, fin);
 341       next_in = input;
 342       if (ferror(fin)) {
 343         break;
 344       }
 345     } else if (result == BROTLI_RESULT_NEEDS_MORE_OUTPUT) {
 346       fwrite(output, 1, kFileBufferSize, fout);
 347       if (ferror(fout)) {
 348         break;
 349       }
 350       available_out = kFileBufferSize;
 351       next_out = output;
 352     } else {
 353       break; /* Error or success. */
 354     }
 355     result = BrotliDecompressStream(&available_in, &next_in,
 356         &available_out, &next_out, &total_out, s);
 357   }
 358   if (next_out != output) {
 359     fwrite(output, 1, (size_t)(next_out - output), fout);
 360   }
 361
 362   if ((result == BROTLI_RESULT_NEEDS_MORE_OUTPUT) || ferror(fout)) {
 363     fprintf(stderr, "failed to write output\n");
 364   } else if (result != BROTLI_RESULT_SUCCESS) { /* Error or needs more input. */
 365     fprintf(stderr, "corrupt input\n");
 366   }
 367
 368 end:
 369   BrFree(memsize, dictionary);
 370   BrFree(memsize, input);
 371   BrFree(memsize, output);
 372   BrotliDestroyState(s);
 373   return (result == BROTLI_RESULT_SUCCESS) ? 1 : 0;
 374 }
 375
 376 static int Compress(int quality, int lgwin, FILE* fin, FILE* fout,
 377     const char *dictionary_path, void *memsize) {
 378   BrotliEncoderState* s = BrotliEncoderCreateInstance(BrAlloc, BrFree, memsize);
 379   uint8_t* buffer = (uint8_t*)BrAlloc(memsize, kFileBufferSize << 1);
 380   uint8_t* input = buffer;
 381   uint8_t* output = buffer + kFileBufferSize;
 382   size_t available_in = 0;
 383   const uint8_t* next_in = NULL;
 384   size_t available_out = kFileBufferSize;
 385   uint8_t* next_out = output;
 386   int is_eof = 0;
 387   int is_ok = 1;
 388
 389   if (!s || !buffer) {
 390     is_ok = 0;
 391     goto finish;
 392   }
 393
 394   BrotliEncoderSetParameter(s, BROTLI_PARAM_QUALITY, (uint32_t)quality);
 395   BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, (uint32_t)lgwin);
 396   if (dictionary_path != NULL) {
 397     size_t dictionary_size = 0;
 398     uint8_t* dictionary = ReadDictionary(dictionary_path, &dictionary_size, memsize);
 399     BrotliEncoderSetCustomDictionary(s, dictionary_size, dictionary);
 400     BrFree(memsize, dictionary);
 401   }
 402
 403   while (1) {
 404     if (available_in == 0 && !is_eof) {
 405       available_in = fread(input, 1, kFileBufferSize, fin);
 406       next_in = input;
 407       if (ferror(fin)) break;
 408       is_eof = feof(fin);
 409     }
 410
 411     if (!BrotliEncoderCompressStream(s,
 412         is_eof ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS,
 413         &available_in, &next_in, &available_out, &next_out, NULL)) {
 414       is_ok = 0;
 415       break;
 416     }
 417
 418     if (available_out != kFileBufferSize) {
 419       size_t out_size = kFileBufferSize - available_out;
 420       fwrite(output, 1, out_size, fout);
 421       if (ferror(fout)) break;
 422       available_out = kFileBufferSize;
 423       next_out = output;
 424     }
 425
 426     if (BrotliEncoderIsFinished(s)) break;
 427   }
 428
 429 finish:
 430   BrFree(memsize, buffer);
 431   BrotliEncoderDestroyInstance(s);
 432
 433   if (!is_ok) {
 434     /* Should detect OOM? */
 435     fprintf(stderr, "failed to compress data\n");
 436     return 0;
 437   } else if (ferror(fout)) {
 438     fprintf(stderr, "failed to write output\n");
 439     return 0;
 440   } else if (ferror(fin)) {
 441     fprintf(stderr, "failed to read input\n");
 442     return 0;
 443   }
 444   return 1;
 445 }
 446
 447 #define GAP_MEM_BLOCK  4096
 448
 449 int main(int argc, char** argv) {
 450   char *input_path = 0;
 451   char *output_path = 0;
 452   char *dictionary_path = 0;
 453   int force = 0;
 454   int quality = 11;
 455   int gmem = 1;
 456   int decompress = 0;
 457   int repeat = 1;
 458   int verbose = 0;
 459   int lgwin = 0;
 460   clock_t clock_start;
 461   int i;
 462   int64_t originsize = 0;
 463   int64_t msize = 0;
 464   ParseArgv(argc, argv, &input_path, &output_path, &dictionary_path, &force,
 465             &quality, &gmem, &decompress, &repeat, &verbose, &lgwin);
 466   clock_start = clock();
 467   for (i = 0; i < repeat; ++i) {
 468     FILE* fin = OpenInputFile(input_path);
 469     FILE* fout = OpenOutputFile(output_path, force || repeat);
 470     int is_ok = 0;
 471     if (decompress) {
 472       if (fseek(fin, 16, SEEK_SET) != 0) {
 473         fclose(fin);
 474         return -1;
 475       }
 476       is_ok = Decompress(fin, fout, dictionary_path, (void *)&msize);
 477     } else {
 478       originsize = FileSize(input_path);  /* get original file size */
 479       fwrite(&originsize, 1, sizeof(int64_t), fout); /* add in original binary file size */
 480       fwrite(&msize, 1, sizeof(int64_t), fout); /* add in dummy decompression required memory size */
 481       is_ok = Compress(quality, lgwin, fin, fout, dictionary_path, (void *)&msize);
 482     }
 483     if (!is_ok) {
 484       unlink(output_path);
 485       exit(1);
 486     }
 487     if (fclose(fin) != 0) {
 488       perror("fclose");
 489       exit(1);
 490     }
 491     if (fclose(fout) != 0) {
 492       perror("fclose");
 493       exit(1);
 494     }
 495     /* after compression operation then execute decompression operation
 496        to get decompression required memory size. */
 497     if (decompress == 0) {
 498       fin = OpenInputFile(output_path);
 499       fout = tmpfile ();
 500       msize = 0;
 501       if (fseek(fin, 16, SEEK_SET) != 0) {
 502         fclose(fin);
 503         return -1;
 504       }
 505       is_ok = Decompress(fin, fout, dictionary_path, (void *)&msize);
 506       if (!is_ok) {
 507         exit(1);
 508       }
 509       if (fclose(fin) != 0) {
 510         perror("fclose");
 511         exit(1);
 512       }
 513       if (fclose(fout) != 0) {
 514         perror("fclose");
 515         exit(1);
 516       }
 517       fout = fopen(output_path, "rb+");  /* open output_path file and add in head info */
 518       /* seek to the offset of decompression required memory size */
 519       if (fseek(fout, 8, SEEK_SET) != 0) {
 520         fclose(fout);
 521         return -1;
 522       }
 523       msize += gmem * GAP_MEM_BLOCK;  /* there is a memory gap between IA32 and X64 environment*/
 524       fwrite(&msize, 1, sizeof(int64_t), fout); /* update final decompression required memory size */
 525       if (fclose(fout) != 0) {
 526         perror("fclose");
 527         exit(1);
 528       }
 529     }
 530   }
 531   if (verbose) {
 532     clock_t clock_end = clock();
 533     double duration = (double)(clock_end - clock_start) / CLOCKS_PER_SEC;
 534     int64_t uncompressed_size;
 535     double uncompressed_bytes_in_MB;
 536     if (duration < 1e-9) {
 537       duration = 1e-9;
 538     }
 539     uncompressed_size = FileSize(decompress ? output_path : input_path);
 540     if (uncompressed_size == -1) {
 541       fprintf(stderr, "failed to determine uncompressed file size\n");
 542       exit(1);
 543     }
 544     uncompressed_bytes_in_MB =
 545         (double)(repeat * uncompressed_size) / (1024.0 * 1024.0);
 546     if (decompress) {
 547       printf("Brotli decompression speed: ");
 548     } else {
 549       printf("Brotli compression speed: ");
 550     }
 551     printf("%g MB/s\n", uncompressed_bytes_in_MB / duration);
 552   }
 553   return 0;
 554 }