ceph/src/zstd/contrib/long_distance_matching/main.c

   1 #include <stdio.h>
   2 #include <stdlib.h>
   3 #include <string.h>
   4 #include <sys/time.h>
   5 #include <sys/types.h>
   6 #include <sys/mman.h>
   7 #include <sys/stat.h>
   8 #include <unistd.h>
   9 #include <zstd.h>
  10
  11 #include <fcntl.h>
  12 #include "ldm.h"
  13 #include "zstd.h"
  14
  15 // #define DECOMPRESS_AND_VERIFY
  16
  17 /* Compress file given by fname and output to oname.
  18  * Returns 0 if successful, error code otherwise.
  19  *
  20  * This adds a header from LDM_writeHeader to the beginning of the output.
  21  *
  22  * This might seg fault if the compressed size is > the decompress
  23  * size due to the mmapping and output file size allocated to be the input size
  24  * The compress function should check before writing or buffer writes.
  25  */
  26 static int compress(const char *fname, const char *oname) {
  27   int fdin, fdout;
  28   struct stat statbuf;
  29   char *src, *dst;
  30   size_t maxCompressedSize, compressedSize;
  31
  32   struct timeval tv1, tv2;
  33   double timeTaken;
  34
  35
  36   /* Open the input file. */
  37   if ((fdin = open(fname, O_RDONLY)) < 0) {
  38     perror("Error in file opening");
  39     return 1;
  40   }
  41
  42   /* Open the output file. */
  43   if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
  44     perror("Can't create output file");
  45     return 1;
  46   }
  47
  48   /* Find the size of the input file. */
  49   if (fstat (fdin, &statbuf) < 0) {
  50     perror("Fstat error");
  51     return 1;
  52   }
  53
  54   maxCompressedSize = (statbuf.st_size + LDM_HEADER_SIZE);
  55
  56   // Handle case where compressed size is > decompressed size.
  57   // TODO: The compress function should check before writing or buffer writes.
  58   maxCompressedSize += statbuf.st_size / 255;
  59
  60   ftruncate(fdout, maxCompressedSize);
  61
  62   /* mmap the input file. */
  63   if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
  64           == (caddr_t) - 1) {
  65       perror("mmap error for input");
  66       return 1;
  67   }
  68
  69   /* mmap the output file. */
  70   if ((dst = mmap(0, maxCompressedSize, PROT_READ | PROT_WRITE,
  71                   MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
  72       perror("mmap error for output");
  73       return 1;
  74   }
  75
  76   gettimeofday(&tv1, NULL);
  77
  78   compressedSize = LDM_HEADER_SIZE +
  79       LDM_compress(src, statbuf.st_size,
  80                    dst + LDM_HEADER_SIZE, maxCompressedSize);
  81
  82   gettimeofday(&tv2, NULL);
  83
  84   // Write the header.
  85   LDM_writeHeader(dst, compressedSize, statbuf.st_size);
  86
  87   // Truncate file to compressedSize.
  88   ftruncate(fdout, compressedSize);
  89
  90   printf("%25s : %10lu -> %10lu - %s \n", fname,
  91          (size_t)statbuf.st_size, (size_t)compressedSize, oname);
  92   printf("Compression ratio: %.2fx --- %.1f%%\n",
  93          (double)statbuf.st_size / (double)compressedSize,
  94          (double)compressedSize / (double)(statbuf.st_size) * 100.0);
  95
  96   timeTaken = (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
  97               (double) (tv2.tv_sec - tv1.tv_sec),
  98
  99   printf("Total compress time = %.3f seconds, Average scanning speed: %.3f MB/s\n",
 100          timeTaken,
 101          ((double)statbuf.st_size / (double) (1 << 20)) / timeTaken);
 102
 103   // Close files.
 104   close(fdin);
 105   close(fdout);
 106   return 0;
 107 }
 108
 109 #ifdef DECOMPRESS_AND_VERIFY
 110 /* Decompress file compressed using LDM_compress.
 111  * The input file should have the LDM_HEADER followed by payload.
 112  * Returns 0 if succesful, and an error code otherwise.
 113  */
 114 static int decompress(const char *fname, const char *oname) {
 115   int fdin, fdout;
 116   struct stat statbuf;
 117   char *src, *dst;
 118   U64 compressedSize, decompressedSize;
 119   size_t outSize;
 120
 121   /* Open the input file. */
 122   if ((fdin = open(fname, O_RDONLY)) < 0) {
 123     perror("Error in file opening");
 124     return 1;
 125   }
 126
 127   /* Open the output file. */
 128   if ((fdout = open(oname, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600)) < 0) {
 129     perror("Can't create output file");
 130     return 1;
 131   }
 132
 133   /* Find the size of the input file. */
 134   if (fstat (fdin, &statbuf) < 0) {
 135     perror("Fstat error");
 136     return 1;
 137   }
 138
 139   /* mmap the input file. */
 140   if ((src = mmap(0, statbuf.st_size, PROT_READ,  MAP_SHARED, fdin, 0))
 141           == (caddr_t) - 1) {
 142       perror("mmap error for input");
 143       return 1;
 144   }
 145
 146   /* Read the header. */
 147   LDM_readHeader(src, &compressedSize, &decompressedSize);
 148
 149   ftruncate(fdout, decompressedSize);
 150
 151   /* mmap the output file */
 152   if ((dst = mmap(0, decompressedSize, PROT_READ | PROT_WRITE,
 153                   MAP_SHARED, fdout, 0)) == (caddr_t) - 1) {
 154       perror("mmap error for output");
 155       return 1;
 156   }
 157
 158   outSize = LDM_decompress(
 159       src + LDM_HEADER_SIZE, statbuf.st_size - LDM_HEADER_SIZE,
 160       dst, decompressedSize);
 161   printf("Ret size out: %zu\n", outSize);
 162
 163   close(fdin);
 164   close(fdout);
 165   return 0;
 166 }
 167
 168 /* Compare two files.
 169  * Returns 0 iff they are the same.
 170  */
 171 static int compare(FILE *fp0, FILE *fp1) {
 172   int result = 0;
 173   while (result == 0) {
 174     char b0[1024];
 175     char b1[1024];
 176     const size_t r0 = fread(b0, 1, sizeof(b0), fp0);
 177     const size_t r1 = fread(b1, 1, sizeof(b1), fp1);
 178
 179     result = (int)r0 - (int)r1;
 180
 181     if (0 == r0 || 0 == r1) break;
 182
 183     if (0 == result) result = memcmp(b0, b1, r0);
 184   }
 185   return result;
 186 }
 187
 188 /* Verify the input file is the same as the decompressed file. */
 189 static int verify(const char *inpFilename, const char *decFilename) {
 190   FILE *inpFp, *decFp;
 191
 192   if ((inpFp = fopen(inpFilename, "rb")) == NULL) {
 193     perror("Could not open input file\n");
 194     return 1;
 195   }
 196
 197   if ((decFp = fopen(decFilename, "rb")) == NULL) {
 198     perror("Could not open decompressed file\n");
 199     return 1;
 200   }
 201
 202   printf("verify : %s <-> %s\n", inpFilename, decFilename);
 203   {
 204     const int cmp = compare(inpFp, decFp);
 205     if(0 == cmp) {
 206       printf("verify : OK\n");
 207     } else {
 208       printf("verify : NG\n");
 209       return 1;
 210     }
 211   }
 212
 213         fclose(decFp);
 214         fclose(inpFp);
 215   return 0;
 216 }
 217 #endif
 218
 219 int main(int argc, const char *argv[]) {
 220   const char * const exeName = argv[0];
 221   char inpFilename[256] = { 0 };
 222   char ldmFilename[256] = { 0 };
 223   char decFilename[256] = { 0 };
 224
 225   if (argc < 2) {
 226     printf("Wrong arguments\n");
 227     printf("Usage:\n");
 228     printf("%s FILE\n", exeName);
 229     return 1;
 230   }
 231
 232   snprintf(inpFilename, 256, "%s", argv[1]);
 233   snprintf(ldmFilename, 256, "%s.ldm", argv[1]);
 234   snprintf(decFilename, 256, "%s.ldm.dec", argv[1]);
 235
 236         printf("inp = [%s]\n", inpFilename);
 237         printf("ldm = [%s]\n", ldmFilename);
 238         printf("dec = [%s]\n", decFilename);
 239
 240   /* Compress */
 241   {
 242     if (compress(inpFilename, ldmFilename)) {
 243         printf("Compress error\n");
 244         return 1;
 245     }
 246   }
 247
 248 #ifdef DECOMPRESS_AND_VERIFY
 249   /* Decompress */
 250   {
 251     struct timeval tv1, tv2;
 252     gettimeofday(&tv1, NULL);
 253     if (decompress(ldmFilename, decFilename)) {
 254         printf("Decompress error\n");
 255         return 1;
 256     }
 257     gettimeofday(&tv2, NULL);
 258     printf("Total decompress time = %f seconds\n",
 259           (double) (tv2.tv_usec - tv1.tv_usec) / 1000000 +
 260           (double) (tv2.tv_sec - tv1.tv_sec));
 261   }
 262   /* verify */
 263   if (verify(inpFilename, decFilename)) {
 264     printf("Verification error\n");
 265     return 1;
 266   }
 267 #endif
 268   return 0;
 269 }