ceph/src/zstd/examples/multiple_streaming_compression.c

   1 /*
   2  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
   3  * All rights reserved.
   4  *
   5  * This source code is licensed under both the BSD-style license (found in the
   6  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
   7  * in the COPYING file in the root directory of this source tree).
   8  * You may select, at your option, one of the above-listed licenses.
   9  */
  10
  11
  12 /* The objective of this example is to show of to compress multiple successive files
  13 *  while preserving memory management.
  14 *  All structures and buffers will be created only once,
  15 *  and shared across all compression operations */
  16
  17 #include <stdlib.h>    // malloc, exit
  18 #include <stdio.h>     // fprintf, perror, feof
  19 #include <string.h>    // strerror
  20 #include <errno.h>     // errno
  21 #define ZSTD_STATIC_LINKING_ONLY  // streaming API defined as "experimental" for the time being
  22 #include <zstd.h>      // presumes zstd library is installed
  23
  24
  25 static void* malloc_orDie(size_t size)
  26 {
  27     void* const buff = malloc(size);
  28     if (buff) return buff;
  29     /* error */
  30     perror("malloc:");
  31     exit(1);
  32 }
  33
  34 static FILE* fopen_orDie(const char *filename, const char *instruction)
  35 {
  36     FILE* const inFile = fopen(filename, instruction);
  37     if (inFile) return inFile;
  38     /* error */
  39     perror(filename);
  40     exit(3);
  41 }
  42
  43 static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
  44 {
  45     size_t const readSize = fread(buffer, 1, sizeToRead, file);
  46     if (readSize == sizeToRead) return readSize;   /* good */
  47     if (feof(file)) return readSize;   /* good, reached end of file */
  48     /* error */
  49     perror("fread");
  50     exit(4);
  51 }
  52
  53 static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
  54 {
  55     size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
  56     if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
  57     /* error */
  58     perror("fwrite");
  59     exit(5);
  60 }
  61
  62 static size_t fclose_orDie(FILE* file)
  63 {
  64     if (!fclose(file)) return 0;
  65     /* error */
  66     perror("fclose");
  67     exit(6);
  68 }
  69
  70
  71 typedef struct {
  72     void* buffIn;
  73     void* buffOut;
  74     size_t buffInSize;
  75     size_t buffOutSize;
  76     ZSTD_CStream* cstream;
  77 } resources ;
  78
  79 static resources createResources_orDie()
  80 {
  81     resources ress;
  82     ress.buffInSize = ZSTD_CStreamInSize();   /* can always read one full block */
  83     ress.buffOutSize= ZSTD_CStreamOutSize();  /* can always flush a full block */
  84     ress.buffIn = malloc_orDie(ress.buffInSize);
  85     ress.buffOut= malloc_orDie(ress.buffOutSize);
  86     ress.cstream = ZSTD_createCStream();
  87     if (ress.cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
  88     return ress;
  89 }
  90
  91 static void freeResources(resources ress)
  92 {
  93     ZSTD_freeCStream(ress.cstream);
  94     free(ress.buffIn);
  95     free(ress.buffOut);
  96 }
  97
  98
  99 static void compressFile_orDie(resources ress, const char* fname, const char* outName, int cLevel)
 100 {
 101     FILE* const fin  = fopen_orDie(fname, "rb");
 102     FILE* const fout = fopen_orDie(outName, "wb");
 103
 104     size_t const initResult = ZSTD_initCStream(ress.cstream, cLevel);
 105     if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
 106
 107     size_t read, toRead = ress.buffInSize;
 108     while( (read = fread_orDie(ress.buffIn, toRead, fin)) ) {
 109         ZSTD_inBuffer input = { ress.buffIn, read, 0 };
 110         while (input.pos < input.size) {
 111             ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
 112             toRead = ZSTD_compressStream(ress.cstream, &output , &input);   /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
 113             if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
 114             if (toRead > ress.buffInSize) toRead = ress.buffInSize;   /* Safely handle when `buffInSize` is manually changed to a smaller value */
 115             fwrite_orDie(ress.buffOut, output.pos, fout);
 116         }
 117     }
 118
 119     ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
 120     size_t const remainingToFlush = ZSTD_endStream(ress.cstream, &output);   /* close frame */
 121     if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); }
 122     fwrite_orDie(ress.buffOut, output.pos, fout);
 123
 124     fclose_orDie(fout);
 125     fclose_orDie(fin);
 126 }
 127
 128
 129 int main(int argc, const char** argv)
 130 {
 131     const char* const exeName = argv[0];
 132
 133     if (argc<2) {
 134         printf("wrong arguments\n");
 135         printf("usage:\n");
 136         printf("%s FILE(s)\n", exeName);
 137         return 1;
 138     }
 139
 140     resources const ress = createResources_orDie();
 141     void* ofnBuffer = NULL;
 142     size_t ofnbSize = 0;
 143
 144     int argNb;
 145     for (argNb = 1; argNb < argc; argNb++) {
 146         const char* const ifn = argv[argNb];
 147         size_t const ifnSize = strlen(ifn);
 148         size_t const ofnSize = ifnSize + 5;
 149         if (ofnbSize <= ofnSize) {
 150             ofnbSize = ofnSize + 16;
 151             free(ofnBuffer);
 152             ofnBuffer = malloc_orDie(ofnbSize);
 153         }
 154         memset(ofnBuffer, 0, ofnSize);
 155         strcat(ofnBuffer, ifn);
 156         strcat(ofnBuffer, ".zst");
 157         compressFile_orDie(ress, ifn, ofnBuffer, 7);
 158     }
 159
 160     freeResources(ress);
 161     /* success */
 162     printf("compressed %i files \n", argc-1);
 163
 164     return 0;
 165 }