ceph/src/zstd/examples/dictionary_compression.c

   1 /**
   2  * Copyright 2016-present, Yann Collet, Facebook, Inc.
   3  * All rights reserved.
   4  *
   5  * This source code is licensed under the license found in the
   6  * LICENSE-examples file in the root directory of this source tree.
   7  */
   8
   9
  10 #include <stdlib.h>    // malloc, exit
  11 #include <stdio.h>     // printf
  12 #include <string.h>    // strerror
  13 #include <errno.h>     // errno
  14 #include <sys/stat.h>  // stat
  15 #include <zstd.h>      // presumes zstd library is installed
  16
  17
  18 static off_t fsize_orDie(const char *filename)
  19 {
  20     struct stat st;
  21     if (stat(filename, &st) == 0) return st.st_size;
  22     /* error */
  23     perror(filename);
  24     exit(1);
  25 }
  26
  27 static FILE* fopen_orDie(const char *filename, const char *instruction)
  28 {
  29     FILE* const inFile = fopen(filename, instruction);
  30     if (inFile) return inFile;
  31     /* error */
  32     perror(filename);
  33     exit(2);
  34 }
  35
  36 static void* malloc_orDie(size_t size)
  37 {
  38     void* const buff = malloc(size);
  39     if (buff) return buff;
  40     /* error */
  41     perror("malloc");
  42     exit(3);
  43 }
  44
  45 static void* loadFile_orDie(const char* fileName, size_t* size)
  46 {
  47     off_t const buffSize = fsize_orDie(fileName);
  48     FILE* const inFile = fopen_orDie(fileName, "rb");
  49     void* const buffer = malloc_orDie(buffSize);
  50     size_t const readSize = fread(buffer, 1, buffSize, inFile);
  51     if (readSize != (size_t)buffSize) {
  52         fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
  53         exit(4);
  54     }
  55     fclose(inFile);
  56     *size = buffSize;
  57     return buffer;
  58 }
  59
  60 static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
  61 {
  62     FILE* const oFile = fopen_orDie(fileName, "wb");
  63     size_t const wSize = fwrite(buff, 1, buffSize, oFile);
  64     if (wSize != (size_t)buffSize) {
  65         fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
  66         exit(5);
  67     }
  68     if (fclose(oFile)) {
  69         perror(fileName);
  70         exit(6);
  71     }
  72 }
  73
  74 /* createDict() :
  75    `dictFileName` is supposed to have been created using `zstd --train` */
  76 static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
  77 {
  78     size_t dictSize;
  79     printf("loading dictionary %s \n", dictFileName);
  80     void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
  81     ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
  82     if (!cdict) {
  83         fprintf(stderr, "ZSTD_createCDict error \n");
  84         exit(7);
  85     }
  86     free(dictBuffer);
  87     return cdict;
  88 }
  89
  90
  91 static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict)
  92 {
  93     size_t fSize;
  94     void* const fBuff = loadFile_orDie(fname, &fSize);
  95     size_t const cBuffSize = ZSTD_compressBound(fSize);
  96     void* const cBuff = malloc_orDie(cBuffSize);
  97
  98     ZSTD_CCtx* const cctx = ZSTD_createCCtx();
  99     if (cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); }
 100     size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
 101     if (ZSTD_isError(cSize)) {
 102         fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
 103         exit(7);
 104     }
 105
 106     saveFile_orDie(oname, cBuff, cSize);
 107
 108     /* success */
 109     printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
 110
 111     ZSTD_freeCCtx(cctx);   /* never fails */
 112     free(fBuff);
 113     free(cBuff);
 114 }
 115
 116
 117 static char* createOutFilename_orDie(const char* filename)
 118 {
 119     size_t const inL = strlen(filename);
 120     size_t const outL = inL + 5;
 121     void* outSpace = malloc_orDie(outL);
 122     memset(outSpace, 0, outL);
 123     strcat(outSpace, filename);
 124     strcat(outSpace, ".zst");
 125     return (char*)outSpace;
 126 }
 127
 128 int main(int argc, const char** argv)
 129 {
 130     const char* const exeName = argv[0];
 131     int const cLevel = 3;
 132
 133     if (argc<3) {
 134         fprintf(stderr, "wrong arguments\n");
 135         fprintf(stderr, "usage:\n");
 136         fprintf(stderr, "%s [FILES] dictionary\n", exeName);
 137         return 1;
 138     }
 139
 140     /* load dictionary only once */
 141     const char* const dictName = argv[argc-1];
 142     ZSTD_CDict* const dictPtr = createCDict_orDie(dictName, cLevel);
 143
 144     int u;
 145     for (u=1; u<argc-1; u++) {
 146         const char* inFilename = argv[u];
 147         char* const outFilename = createOutFilename_orDie(inFilename);
 148         compress(inFilename, outFilename, dictPtr);
 149         free(outFilename);
 150     }
 151
 152     ZSTD_freeCDict(dictPtr);
 153     printf("All %u files compressed. \n", argc-2);
 154     return 0;
 155 }