ceph/src/zstd/contrib/pzstd/Options.cpp

   1 /**
   2  * Copyright (c) 2016-present, Facebook, Inc.
   3  * All rights reserved.
   4  *
   5  * This source code is licensed under the BSD-style license found in the
   6  * LICENSE file in the root directory of this source tree. An additional grant
   7  * of patent rights can be found in the PATENTS file in the same directory.
   8  */
   9 #include "Options.h"
  10 #include "utils/ScopeGuard.h"
  11
  12 #include <algorithm>
  13 #include <cassert>
  14 #include <cstdio>
  15 #include <cstring>
  16 #include <iterator>
  17 #include <thread>
  18 #include <util.h>
  19 #include <vector>
  20
  21 #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) ||     \
  22     defined(__CYGWIN__)
  23 #include <io.h> /* _isatty */
  24 #define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
  25 #elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || \
  26       defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__)  /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
  27 #include <unistd.h> /* isatty */
  28 #define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
  29 #else
  30 #define IS_CONSOLE(stdStream) 0
  31 #endif
  32
  33 namespace pzstd {
  34
  35 namespace {
  36 unsigned defaultNumThreads() {
  37 #ifdef PZSTD_NUM_THREADS
  38   return PZSTD_NUM_THREADS;
  39 #else
  40   return std::thread::hardware_concurrency();
  41 #endif
  42 }
  43
  44 unsigned parseUnsigned(const char **arg) {
  45   unsigned result = 0;
  46   while (**arg >= '0' && **arg <= '9') {
  47     result *= 10;
  48     result += **arg - '0';
  49     ++(*arg);
  50   }
  51   return result;
  52 }
  53
  54 const char *getArgument(const char *options, const char **argv, int &i,
  55                         int argc) {
  56   if (options[1] != 0) {
  57     return options + 1;
  58   }
  59   ++i;
  60   if (i == argc) {
  61     std::fprintf(stderr, "Option -%c requires an argument, but none provided\n",
  62                  *options);
  63     return nullptr;
  64   }
  65   return argv[i];
  66 }
  67
  68 const std::string kZstdExtension = ".zst";
  69 constexpr char kStdIn[] = "-";
  70 constexpr char kStdOut[] = "-";
  71 constexpr unsigned kDefaultCompressionLevel = 3;
  72 constexpr unsigned kMaxNonUltraCompressionLevel = 19;
  73
  74 #ifdef _WIN32
  75 const char nullOutput[] = "nul";
  76 #else
  77 const char nullOutput[] = "/dev/null";
  78 #endif
  79
  80 void notSupported(const char *option) {
  81   std::fprintf(stderr, "Operation not supported: %s\n", option);
  82 }
  83
  84 void usage() {
  85   std::fprintf(stderr, "Usage:\n");
  86   std::fprintf(stderr, "  pzstd [args] [FILE(s)]\n");
  87   std::fprintf(stderr, "Parallel ZSTD options:\n");
  88   std::fprintf(stderr, "  -p, --processes   #    : number of threads to use for (de)compression (default:%d)\n", defaultNumThreads());
  89
  90   std::fprintf(stderr, "ZSTD options:\n");
  91   std::fprintf(stderr, "  -#                     : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel, kDefaultCompressionLevel);
  92   std::fprintf(stderr, "  -d, --decompress       : decompression\n");
  93   std::fprintf(stderr, "  -o                file : result stored into `file` (only if 1 input file)\n");
  94   std::fprintf(stderr, "  -f, --force            : overwrite output without prompting\n");
  95   std::fprintf(stderr, "      --rm               : remove source file(s) after successful (de)compression\n");
  96   std::fprintf(stderr, "  -k, --keep             : preserve source file(s) (default)\n");
  97   std::fprintf(stderr, "  -h, --help             : display help and exit\n");
  98   std::fprintf(stderr, "  -V, --version          : display version number and exit\n");
  99   std::fprintf(stderr, "  -v, --verbose          : verbose mode; specify multiple times to increase log level (default:2)\n");
 100   std::fprintf(stderr, "  -q, --quiet            : suppress warnings; specify twice to suppress errors too\n");
 101   std::fprintf(stderr, "  -c, --stdout           : force write to standard output, even if it is the console\n");
 102 #ifdef UTIL_HAS_CREATEFILELIST
 103   std::fprintf(stderr, "  -r                     : operate recursively on directories\n");
 104 #endif
 105   std::fprintf(stderr, "      --ultra            : enable levels beyond %i, up to %i (requires more memory)\n", kMaxNonUltraCompressionLevel, ZSTD_maxCLevel());
 106   std::fprintf(stderr, "  -C, --check            : integrity check (default)\n");
 107   std::fprintf(stderr, "      --no-check         : no integrity check\n");
 108   std::fprintf(stderr, "  -t, --test             : test compressed file integrity\n");
 109   std::fprintf(stderr, "  --                     : all arguments after \"--\" are treated as files\n");
 110 }
 111 } // anonymous namespace
 112
 113 Options::Options()
 114     : numThreads(defaultNumThreads()), maxWindowLog(23),
 115       compressionLevel(kDefaultCompressionLevel), decompress(false),
 116       overwrite(false), keepSource(true), writeMode(WriteMode::Auto),
 117       checksum(true), verbosity(2) {}
 118
 119 Options::Status Options::parse(int argc, const char **argv) {
 120   bool test = false;
 121   bool recursive = false;
 122   bool ultra = false;
 123   bool forceStdout = false;
 124   // Local copy of input files, which are pointers into argv.
 125   std::vector<const char *> localInputFiles;
 126   for (int i = 1; i < argc; ++i) {
 127     const char *arg = argv[i];
 128     // Protect against empty arguments
 129     if (arg[0] == 0) {
 130       continue;
 131     }
 132     // Everything after "--" is an input file
 133     if (!std::strcmp(arg, "--")) {
 134       ++i;
 135       std::copy(argv + i, argv + argc, std::back_inserter(localInputFiles));
 136       break;
 137     }
 138     // Long arguments that don't have a short option
 139     {
 140       bool isLongOption = true;
 141       if (!std::strcmp(arg, "--rm")) {
 142         keepSource = false;
 143       } else if (!std::strcmp(arg, "--ultra")) {
 144         ultra = true;
 145         maxWindowLog = 0;
 146       } else if (!std::strcmp(arg, "--no-check")) {
 147         checksum = false;
 148       } else if (!std::strcmp(arg, "--sparse")) {
 149         writeMode = WriteMode::Sparse;
 150         notSupported("Sparse mode");
 151         return Status::Failure;
 152       } else if (!std::strcmp(arg, "--no-sparse")) {
 153         writeMode = WriteMode::Regular;
 154         notSupported("Sparse mode");
 155         return Status::Failure;
 156       } else if (!std::strcmp(arg, "--dictID")) {
 157         notSupported(arg);
 158         return Status::Failure;
 159       } else if (!std::strcmp(arg, "--no-dictID")) {
 160         notSupported(arg);
 161         return Status::Failure;
 162       } else {
 163         isLongOption = false;
 164       }
 165       if (isLongOption) {
 166         continue;
 167       }
 168     }
 169     // Arguments with a short option simply set their short option.
 170     const char *options = nullptr;
 171     if (!std::strcmp(arg, "--processes")) {
 172       options = "p";
 173     } else if (!std::strcmp(arg, "--version")) {
 174       options = "V";
 175     } else if (!std::strcmp(arg, "--help")) {
 176       options = "h";
 177     } else if (!std::strcmp(arg, "--decompress")) {
 178       options = "d";
 179     } else if (!std::strcmp(arg, "--force")) {
 180       options = "f";
 181     } else if (!std::strcmp(arg, "--stdout")) {
 182       options = "c";
 183     } else if (!std::strcmp(arg, "--keep")) {
 184       options = "k";
 185     } else if (!std::strcmp(arg, "--verbose")) {
 186       options = "v";
 187     } else if (!std::strcmp(arg, "--quiet")) {
 188       options = "q";
 189     } else if (!std::strcmp(arg, "--check")) {
 190       options = "C";
 191     } else if (!std::strcmp(arg, "--test")) {
 192       options = "t";
 193     } else if (arg[0] == '-' && arg[1] != 0) {
 194       options = arg + 1;
 195     } else {
 196       localInputFiles.emplace_back(arg);
 197       continue;
 198     }
 199     assert(options != nullptr);
 200
 201     bool finished = false;
 202     while (!finished && *options != 0) {
 203       // Parse the compression level
 204       if (*options >= '0' && *options <= '9') {
 205         compressionLevel = parseUnsigned(&options);
 206         continue;
 207       }
 208
 209       switch (*options) {
 210       case 'h':
 211       case 'H':
 212         usage();
 213         return Status::Message;
 214       case 'V':
 215         std::fprintf(stderr, "PZSTD version: %s.\n", ZSTD_VERSION_STRING);
 216         return Status::Message;
 217       case 'p': {
 218         finished = true;
 219         const char *optionArgument = getArgument(options, argv, i, argc);
 220         if (optionArgument == nullptr) {
 221           return Status::Failure;
 222         }
 223         if (*optionArgument < '0' || *optionArgument > '9') {
 224           std::fprintf(stderr, "Option -p expects a number, but %s provided\n",
 225                        optionArgument);
 226           return Status::Failure;
 227         }
 228         numThreads = parseUnsigned(&optionArgument);
 229         if (*optionArgument != 0) {
 230           std::fprintf(stderr,
 231                        "Option -p expects a number, but %u%s provided\n",
 232                        numThreads, optionArgument);
 233           return Status::Failure;
 234         }
 235         break;
 236       }
 237       case 'o': {
 238         finished = true;
 239         const char *optionArgument = getArgument(options, argv, i, argc);
 240         if (optionArgument == nullptr) {
 241           return Status::Failure;
 242         }
 243         outputFile = optionArgument;
 244         break;
 245       }
 246       case 'C':
 247         checksum = true;
 248         break;
 249       case 'k':
 250         keepSource = true;
 251         break;
 252       case 'd':
 253         decompress = true;
 254         break;
 255       case 'f':
 256         overwrite = true;
 257         forceStdout = true;
 258         break;
 259       case 't':
 260         test = true;
 261         decompress = true;
 262         break;
 263 #ifdef UTIL_HAS_CREATEFILELIST
 264       case 'r':
 265         recursive = true;
 266         break;
 267 #endif
 268       case 'c':
 269         outputFile = kStdOut;
 270         forceStdout = true;
 271         break;
 272       case 'v':
 273         ++verbosity;
 274         break;
 275       case 'q':
 276         --verbosity;
 277         // Ignore them for now
 278         break;
 279       // Unsupported options from Zstd
 280       case 'D':
 281       case 's':
 282         notSupported("Zstd dictionaries.");
 283         return Status::Failure;
 284       case 'b':
 285       case 'e':
 286       case 'i':
 287       case 'B':
 288         notSupported("Zstd benchmarking options.");
 289         return Status::Failure;
 290       default:
 291         std::fprintf(stderr, "Invalid argument: %s\n", arg);
 292         return Status::Failure;
 293       }
 294       if (!finished) {
 295         ++options;
 296       }
 297     } // while (*options != 0);
 298   }   // for (int i = 1; i < argc; ++i);
 299
 300   // Set options for test mode
 301   if (test) {
 302     outputFile = nullOutput;
 303     keepSource = true;
 304   }
 305
 306   // Input file defaults to standard input if not provided.
 307   if (localInputFiles.empty()) {
 308     localInputFiles.emplace_back(kStdIn);
 309   }
 310
 311   // Check validity of input files
 312   if (localInputFiles.size() > 1) {
 313     const auto it = std::find(localInputFiles.begin(), localInputFiles.end(),
 314                               std::string{kStdIn});
 315     if (it != localInputFiles.end()) {
 316       std::fprintf(
 317           stderr,
 318           "Cannot specify standard input when handling multiple files\n");
 319       return Status::Failure;
 320     }
 321   }
 322   if (localInputFiles.size() > 1 || recursive) {
 323     if (!outputFile.empty() && outputFile != nullOutput) {
 324       std::fprintf(
 325           stderr,
 326           "Cannot specify an output file when handling multiple inputs\n");
 327       return Status::Failure;
 328     }
 329   }
 330
 331   // Translate input files/directories into files to (de)compress
 332   if (recursive) {
 333     char *scratchBuffer = nullptr;
 334     unsigned numFiles = 0;
 335     const char **files =
 336         UTIL_createFileList(localInputFiles.data(), localInputFiles.size(),
 337                             &scratchBuffer, &numFiles);
 338     if (files == nullptr) {
 339       std::fprintf(stderr, "Error traversing directories\n");
 340       return Status::Failure;
 341     }
 342     auto guard =
 343         makeScopeGuard([&] { UTIL_freeFileList(files, scratchBuffer); });
 344     if (numFiles == 0) {
 345       std::fprintf(stderr, "No files found\n");
 346       return Status::Failure;
 347     }
 348     inputFiles.resize(numFiles);
 349     std::copy(files, files + numFiles, inputFiles.begin());
 350   } else {
 351     inputFiles.resize(localInputFiles.size());
 352     std::copy(localInputFiles.begin(), localInputFiles.end(),
 353               inputFiles.begin());
 354   }
 355   localInputFiles.clear();
 356   assert(!inputFiles.empty());
 357
 358   // If reading from standard input, default to standard output
 359   if (inputFiles[0] == kStdIn && outputFile.empty()) {
 360     assert(inputFiles.size() == 1);
 361     outputFile = "-";
 362   }
 363
 364   if (inputFiles[0] == kStdIn && IS_CONSOLE(stdin)) {
 365     assert(inputFiles.size() == 1);
 366     std::fprintf(stderr, "Cannot read input from interactive console\n");
 367     return Status::Failure;
 368   }
 369   if (outputFile == "-" && IS_CONSOLE(stdout) && !(forceStdout && decompress)) {
 370     std::fprintf(stderr, "Will not write to console stdout unless -c or -f is "
 371                          "specified and decompressing\n");
 372     return Status::Failure;
 373   }
 374
 375   // Check compression level
 376   {
 377     unsigned maxCLevel =
 378         ultra ? ZSTD_maxCLevel() : kMaxNonUltraCompressionLevel;
 379     if (compressionLevel > maxCLevel || compressionLevel == 0) {
 380       std::fprintf(stderr, "Invalid compression level %u.\n", compressionLevel);
 381       return Status::Failure;
 382     }
 383   }
 384
 385   // Check that numThreads is set
 386   if (numThreads == 0) {
 387     std::fprintf(stderr, "Invalid arguments: # of threads not specified "
 388                          "and unable to determine hardware concurrency.\n");
 389     return Status::Failure;
 390   }
 391
 392   // Modify verbosity
 393   // If we are piping input and output, turn off interaction
 394   if (inputFiles[0] == kStdIn && outputFile == kStdOut && verbosity == 2) {
 395     verbosity = 1;
 396   }
 397   // If we are in multi-file mode, turn off interaction
 398   if (inputFiles.size() > 1 && verbosity == 2) {
 399     verbosity = 1;
 400   }
 401
 402   return Status::Success;
 403 }
 404
 405 std::string Options::getOutputFile(const std::string &inputFile) const {
 406   if (!outputFile.empty()) {
 407     return outputFile;
 408   }
 409   // Attempt to add/remove zstd extension from the input file
 410   if (decompress) {
 411     int stemSize = inputFile.size() - kZstdExtension.size();
 412     if (stemSize > 0 && inputFile.substr(stemSize) == kZstdExtension) {
 413       return inputFile.substr(0, stemSize);
 414     } else {
 415       return "";
 416     }
 417   } else {
 418     return inputFile + kZstdExtension;
 419   }
 420 }
 421 }