BaseTools: Add Brotli algorithm tool

[mirror_edk2.git] / BaseTools / Source / C / BrotliCompress / tools / bro.c

/* Copyright 2014 Google Inc. All Rights Reserved.\r
\r
   Distributed under MIT license.\r
   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT\r
*/\r
\r
/* Example main() function for Brotli library. */\r
\r
#include <fcntl.h>\r
#include <stdio.h>\r
#include <stdlib.h>\r
#include <string.h>\r
#include <sys/stat.h>\r
#include <sys/types.h>\r
#include <time.h>\r
\r
#include "../dec/decode.h"\r
#include "../enc/encode.h"\r
\r
#if !defined(_WIN32)\r
#include <unistd.h>\r
#else\r
#include <io.h>\r
#include <share.h>\r
\r
#define MAKE_BINARY(FILENO) (_setmode((FILENO), _O_BINARY), (FILENO))\r
\r
#if !defined(__MINGW32__)\r
#define STDIN_FILENO MAKE_BINARY(_fileno(stdin))\r
#define STDOUT_FILENO MAKE_BINARY(_fileno(stdout))\r
#define S_IRUSR S_IREAD\r
#define S_IWUSR S_IWRITE\r
#endif\r
#define fdopen _fdopen\r
#define unlink _unlink\r
\r
#define fopen ms_fopen\r
#define open ms_open\r
\r
#if defined(_MSC_VER) && (_MSC_VER >= 1400)\r
#define fseek _fseeki64\r
#define ftell _ftelli64\r
#endif\r
\r
static FILE* ms_fopen(const char *filename, const char *mode) {\r
  FILE* result = 0;\r
  fopen_s(&result, filename, mode);\r
  return result;\r
}\r
\r
static int ms_open(const char *filename, int oflag, int pmode) {\r
  int result = -1;\r
  _sopen_s(&result, filename, oflag | O_BINARY, _SH_DENYNO, pmode);\r
  return result;\r
}\r
#endif  /* WIN32 */\r
\r
static int ParseQuality(const char* s, int* quality) {\r
  if (s[0] >= '0' && s[0] <= '9') {\r
    *quality = s[0] - '0';\r
    if (s[1] >= '0' && s[1] <= '9') {\r
      *quality = *quality * 10 + s[1] - '0';\r
      return (s[2] == 0) ? 1 : 0;\r
    }\r
    return (s[1] == 0) ? 1 : 0;\r
  }\r
  return 0;\r
}\r
\r
static void ParseArgv(int argc, char **argv,\r
                      char **input_path,\r
                      char **output_path,\r
                      char **dictionary_path,\r
                      int *force,\r
                      int *quality,\r
                      int *gapmem,\r
                      int *decompress,\r
                      int *repeat,\r
                      int *verbose,\r
                      int *lgwin) {\r
  int k;\r
  *force = 0;\r
  *input_path = 0;\r
  *output_path = 0;\r
  *repeat = 1;\r
  *verbose = 0;\r
  *lgwin = 22;\r
  {\r
    size_t argv0_len = strlen(argv[0]);\r
    *decompress =\r
        argv0_len >= 5 && strcmp(&argv[0][argv0_len - 5], "unbro") == 0;\r
  }\r
  for (k = 1; k < argc; ++k) {\r
    if (!strcmp("--force", argv[k]) ||\r
        !strcmp("-f", argv[k])) {\r
      if (*force != 0) {\r
        goto error;\r
      }\r
      *force = 1;\r
      continue;\r
    } else if (!strcmp("--decompress", argv[k]) ||\r
               !strcmp("--uncompress", argv[k]) ||\r
               !strcmp("-d", argv[k])) {\r
      *decompress = 1;\r
      continue;\r
    } else if (!strcmp("--verbose", argv[k]) ||\r
               !strcmp("-v", argv[k])) {\r
      if (*verbose != 0) {\r
        goto error;\r
      }\r
      *verbose = 1;\r
      continue;\r
    }\r
    if (k < argc - 1) {\r
      if (!strcmp("--input", argv[k]) ||\r
          !strcmp("--in", argv[k]) ||\r
          !strcmp("-i", argv[k])) {\r
        if (*input_path != 0) {\r
          goto error;\r
        }\r
        *input_path = argv[k + 1];\r
        ++k;\r
        continue;\r
      } else if (!strcmp("--output", argv[k]) ||\r
                 !strcmp("--out", argv[k]) ||\r
                 !strcmp("-o", argv[k])) {\r
        if (*output_path != 0) {\r
          goto error;\r
        }\r
        *output_path = argv[k + 1];\r
        ++k;\r
        continue;\r
      } else if (!strcmp("--custom-dictionary", argv[k])) {\r
        if (*dictionary_path != 0) {\r
          goto error;\r
        }\r
        *dictionary_path = argv[k + 1];\r
        ++k;\r
        continue;\r
      } else if (!strcmp("--quality", argv[k]) ||\r
                 !strcmp("-q", argv[k])) {\r
        if (!ParseQuality(argv[k + 1], quality)) {\r
          goto error;\r
        }\r
        ++k;\r
        continue;\r
      } else if (!strcmp("--repeat", argv[k]) ||\r
                 !strcmp("-r", argv[k])) {\r
        if (!ParseQuality(argv[k + 1], repeat)) {\r
          goto error;\r
        }\r
        ++k;\r
        continue;\r
      }  else if (!strcmp("--window", argv[k]) ||\r
                  !strcmp("-w", argv[k])) {\r
        if (!ParseQuality(argv[k + 1], lgwin)) {\r
          goto error;\r
        }\r
        if (*lgwin < 10 || *lgwin >= 25) {\r
          goto error;\r
        }\r
        ++k;\r
        continue;\r
      } else if (!strcmp("--gap", argv[k]) ||\r
                  !strcmp("-g", argv[k])) {\r
        if (!ParseQuality(argv[k + 1], gapmem)) {\r
          goto error;\r
        }\r
        ++k;\r
        continue;\r
      }\r
    }\r
    goto error;\r
  }\r
  return;\r
error:\r
  fprintf(stderr,\r
          "Usage: %s [--force] [--quality n] [--gap n] [--decompress]"\r
          " [--input filename] [--output filename] [--repeat iters]"\r
          " [--verbose] [--window n] [--custom-dictionary filename]\n",\r
          argv[0]);\r
  exit(1);\r
}\r
\r
static FILE* OpenInputFile(const char* input_path) {\r
  FILE* f;\r
  if (input_path == 0) {\r
    return fdopen(STDIN_FILENO, "rb");\r
  }\r
  f = fopen(input_path, "rb");\r
  if (f == 0) {\r
    perror("fopen");\r
    exit(1);\r
  }\r
  return f;\r
}\r
\r
static FILE *OpenOutputFile(const char *output_path, const int force) {\r
  int fd;\r
  if (output_path == 0) {\r
    return fdopen(STDOUT_FILENO, "wb");\r
  }\r
  fd = open(output_path, O_CREAT | (force ? 0 : O_EXCL) | O_WRONLY | O_TRUNC,\r
            S_IRUSR | S_IWUSR);\r
  if (fd < 0) {\r
    if (!force) {\r
      struct stat statbuf;\r
      if (stat(output_path, &statbuf) == 0) {\r
        fprintf(stderr, "output file exists\n");\r
        exit(1);\r
      }\r
    }\r
    perror("open");\r
    exit(1);\r
  }\r
  return fdopen(fd, "wb");\r
}\r
\r
static int64_t FileSize(const char *path) {\r
  FILE *f = fopen(path, "rb");\r
  int64_t retval;\r
  if (f == NULL) {\r
    return -1;\r
  }\r
  if (fseek(f, 0L, SEEK_END) != 0) {\r
    fclose(f);\r
    return -1;\r
  }\r
  retval = ftell(f);\r
  if (fclose(f) != 0) {\r
    return -1;\r
  }\r
  return retval;\r
}\r
\r
/* Brotli specified memory allocate function */\r
static void *BrAlloc (void *memsize, size_t size) {\r
  *(int64_t *)memsize += size;\r
  return malloc(size);\r
}\r
\r
/* Brotli specified memory free function */\r
static void BrFree (void *memsize, void *ptr) {\r
  free(ptr);\r
}\r
\r
/* Result ownersip is passed to caller.\r
   |*dictionary_size| is set to resulting buffer size. */\r
static uint8_t* ReadDictionary(const char* path, size_t* dictionary_size, void *memsize) {\r
  static const int kMaxDictionarySize = (1 << 24) - 16;\r
  FILE *f = fopen(path, "rb");\r
  int64_t file_size_64;\r
  uint8_t* buffer;\r
  size_t bytes_read;\r
\r
  if (f == NULL) {\r
    perror("fopen");\r
    exit(1);\r
  }\r
\r
  file_size_64 = FileSize(path);\r
  if (file_size_64 == -1) {\r
    fprintf(stderr, "could not get size of dictionary file");\r
    exit(1);\r
  }\r
\r
  if (file_size_64 > kMaxDictionarySize) {\r
    fprintf(stderr, "dictionary is larger than maximum allowed: %d\n",\r
            kMaxDictionarySize);\r
    exit(1);\r
  }\r
  *dictionary_size = (size_t)file_size_64;\r
\r
  buffer = (uint8_t *)BrAlloc(memsize, *dictionary_size);\r
  if (!buffer) {\r
    fprintf(stderr, "could not read dictionary: out of memory\n");\r
    exit(1);\r
  }\r
  bytes_read = fread(buffer, sizeof(uint8_t), *dictionary_size, f);\r
  if (bytes_read != *dictionary_size) {\r
    fprintf(stderr, "could not read dictionary\n");\r
    exit(1);\r
  }\r
  fclose(f);\r
  return buffer;\r
}\r
\r
static const size_t kFileBufferSize = 65536;\r
\r
static int Decompress(FILE* fin, FILE* fout, const char* dictionary_path, void *memsize) {\r
  /* Dictionary should be kept during first rounds of decompression. */\r
  uint8_t* dictionary = NULL;\r
  uint8_t* input;\r
  uint8_t* output;\r
  size_t total_out;\r
  size_t available_in;\r
  const uint8_t* next_in;\r
  size_t available_out = kFileBufferSize;\r
  uint8_t* next_out;\r
  BrotliResult result = BROTLI_RESULT_ERROR;\r
  BrotliState* s = BrotliCreateState(BrAlloc, BrFree, memsize);\r
  if (!s) {\r
    fprintf(stderr, "out of memory\n");\r
    return 0;\r
  }\r
  if (dictionary_path != NULL) {\r
    size_t dictionary_size = 0;\r
    dictionary = ReadDictionary(dictionary_path, &dictionary_size, memsize);\r
    BrotliSetCustomDictionary(dictionary_size, dictionary, s);\r
  }\r
  input = (uint8_t*)BrAlloc(memsize, kFileBufferSize);\r
  output = (uint8_t*)BrAlloc(memsize, kFileBufferSize);\r
  if (!input || !output) {\r
    fprintf(stderr, "out of memory\n");\r
    goto end;\r
  }\r
  next_out = output;\r
  result = BROTLI_RESULT_NEEDS_MORE_INPUT;\r
  while (1) {\r
    if (result == BROTLI_RESULT_NEEDS_MORE_INPUT) {\r
      if (feof(fin)) {\r
        break;\r
      }\r
      available_in = fread(input, 1, kFileBufferSize, fin);\r
      next_in = input;\r
      if (ferror(fin)) {\r
        break;\r
      }\r
    } else if (result == BROTLI_RESULT_NEEDS_MORE_OUTPUT) {\r
      fwrite(output, 1, kFileBufferSize, fout);\r
      if (ferror(fout)) {\r
        break;\r
      }\r
      available_out = kFileBufferSize;\r
      next_out = output;\r
    } else {\r
      break; /* Error or success. */\r
    }\r
    result = BrotliDecompressStream(&available_in, &next_in,\r
        &available_out, &next_out, &total_out, s);\r
  }\r
  if (next_out != output) {\r
    fwrite(output, 1, (size_t)(next_out - output), fout);\r
  }\r
\r
  if ((result == BROTLI_RESULT_NEEDS_MORE_OUTPUT) || ferror(fout)) {\r
    fprintf(stderr, "failed to write output\n");\r
  } else if (result != BROTLI_RESULT_SUCCESS) { /* Error or needs more input. */\r
    fprintf(stderr, "corrupt input\n");\r
  }\r
\r
end:\r
  BrFree(memsize, dictionary);\r
  BrFree(memsize, input);\r
  BrFree(memsize, output);\r
  BrotliDestroyState(s);\r
  return (result == BROTLI_RESULT_SUCCESS) ? 1 : 0;\r
}\r
\r
static int Compress(int quality, int lgwin, FILE* fin, FILE* fout,\r
    const char *dictionary_path, void *memsize) {\r
  BrotliEncoderState* s = BrotliEncoderCreateInstance(BrAlloc, BrFree, memsize);\r
  uint8_t* buffer = (uint8_t*)BrAlloc(memsize, kFileBufferSize << 1);\r
  uint8_t* input = buffer;\r
  uint8_t* output = buffer + kFileBufferSize;\r
  size_t available_in = 0;\r
  const uint8_t* next_in = NULL;\r
  size_t available_out = kFileBufferSize;\r
  uint8_t* next_out = output;\r
  int is_eof = 0;\r
  int is_ok = 1;\r
\r
  if (!s || !buffer) {\r
    is_ok = 0;\r
    goto finish;\r
  }\r
\r
  BrotliEncoderSetParameter(s, BROTLI_PARAM_QUALITY, (uint32_t)quality);\r
  BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, (uint32_t)lgwin);\r
  if (dictionary_path != NULL) {\r
    size_t dictionary_size = 0;\r
    uint8_t* dictionary = ReadDictionary(dictionary_path, &dictionary_size, memsize);\r
    BrotliEncoderSetCustomDictionary(s, dictionary_size, dictionary);\r
    BrFree(memsize, dictionary);\r
  }\r
\r
  while (1) {\r
    if (available_in == 0 && !is_eof) {\r
      available_in = fread(input, 1, kFileBufferSize, fin);\r
      next_in = input;\r
      if (ferror(fin)) break;\r
      is_eof = feof(fin);\r
    }\r
\r
    if (!BrotliEncoderCompressStream(s,\r
        is_eof ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS,\r
        &available_in, &next_in, &available_out, &next_out, NULL)) {\r
      is_ok = 0;\r
      break;\r
    }\r
\r
    if (available_out != kFileBufferSize) {\r
      size_t out_size = kFileBufferSize - available_out;\r
      fwrite(output, 1, out_size, fout);\r
      if (ferror(fout)) break;\r
      available_out = kFileBufferSize;\r
      next_out = output;\r
    }\r
\r
    if (BrotliEncoderIsFinished(s)) break;\r
  }\r
\r
finish:\r
  BrFree(memsize, buffer);\r
  BrotliEncoderDestroyInstance(s);\r
\r
  if (!is_ok) {\r
    /* Should detect OOM? */\r
    fprintf(stderr, "failed to compress data\n");\r
    return 0;\r
  } else if (ferror(fout)) {\r
    fprintf(stderr, "failed to write output\n");\r
    return 0;\r
  } else if (ferror(fin)) {\r
    fprintf(stderr, "failed to read input\n");\r
    return 0;\r
  }\r
  return 1;\r
}\r
\r
#define GAP_MEM_BLOCK  4096\r
\r
int main(int argc, char** argv) {\r
  char *input_path = 0;\r
  char *output_path = 0;\r
  char *dictionary_path = 0;\r
  int force = 0;\r
  int quality = 11;\r
  int gmem = 1;\r
  int decompress = 0;\r
  int repeat = 1;\r
  int verbose = 0;\r
  int lgwin = 0;\r
  clock_t clock_start;\r
  int i;\r
  int64_t originsize = 0;\r
  int64_t msize = 0;\r
  ParseArgv(argc, argv, &input_path, &output_path, &dictionary_path, &force,\r
            &quality, &gmem, &decompress, &repeat, &verbose, &lgwin);\r
  clock_start = clock();\r
  for (i = 0; i < repeat; ++i) {\r
    FILE* fin = OpenInputFile(input_path);\r
    FILE* fout = OpenOutputFile(output_path, force || repeat);\r
    int is_ok = 0;\r
    if (decompress) {\r
      if (fseek(fin, 16, SEEK_SET) != 0) {\r
        fclose(fin);\r
        return -1;\r
      }\r
      is_ok = Decompress(fin, fout, dictionary_path, (void *)&msize);\r
    } else {\r
      originsize = FileSize(input_path);  /* get original file size */\r
      fwrite(&originsize, 1, sizeof(int64_t), fout); /* add in original binary file size */\r
      fwrite(&msize, 1, sizeof(int64_t), fout); /* add in dummy decompression required memory size */\r
      is_ok = Compress(quality, lgwin, fin, fout, dictionary_path, (void *)&msize);\r
    }\r
    if (!is_ok) {\r
      unlink(output_path);\r
      exit(1);\r
    }\r
    if (fclose(fin) != 0) {\r
      perror("fclose");\r
      exit(1);\r
    }\r
    if (fclose(fout) != 0) {\r
      perror("fclose");\r
      exit(1);\r
    }\r
    /* after compression operation then execute decompression operation\r
       to get decompression required memory size. */\r
    if (decompress == 0) {\r
      fin = OpenInputFile(output_path);\r
      fout = tmpfile ();\r
      msize = 0;\r
      if (fseek(fin, 16, SEEK_SET) != 0) {\r
        fclose(fin);\r
        return -1;\r
      }\r
      is_ok = Decompress(fin, fout, dictionary_path, (void *)&msize);\r
      if (!is_ok) {\r
        exit(1);\r
      }\r
      if (fclose(fin) != 0) {\r
        perror("fclose");\r
        exit(1);\r
      }\r
      if (fclose(fout) != 0) {\r
        perror("fclose");\r
        exit(1);\r
      }\r
      fout = fopen(output_path, "rb+");  /* open output_path file and add in head info */\r
      /* seek to the offset of decompression required memory size */\r
      if (fseek(fout, 8, SEEK_SET) != 0) {\r
        fclose(fout);\r
        return -1;\r
      }\r
      msize += gmem * GAP_MEM_BLOCK;  /* there is a memory gap between IA32 and X64 environment*/\r
      fwrite(&msize, 1, sizeof(int64_t), fout); /* update final decompression required memory size */\r
      if (fclose(fout) != 0) {\r
        perror("fclose");\r
        exit(1);\r
      }\r
    }\r
  }\r
  if (verbose) {\r
    clock_t clock_end = clock();\r
    double duration = (double)(clock_end - clock_start) / CLOCKS_PER_SEC;\r
    int64_t uncompressed_size;\r
    double uncompressed_bytes_in_MB;\r
    if (duration < 1e-9) {\r
      duration = 1e-9;\r
    }\r
    uncompressed_size = FileSize(decompress ? output_path : input_path);\r
    if (uncompressed_size == -1) {\r
      fprintf(stderr, "failed to determine uncompressed file size\n");\r
      exit(1);\r
    }\r
    uncompressed_bytes_in_MB =\r
        (double)(repeat * uncompressed_size) / (1024.0 * 1024.0);\r
    if (decompress) {\r
      printf("Brotli decompression speed: ");\r
    } else {\r
      printf("Brotli compression speed: ");\r
    }\r
    printf("%g MB/s\n", uncompressed_bytes_in_MB / duration);\r
  }\r
  return 0;\r
}\r