2 * Copyright (c) 2016-present, Facebook, Inc.
5 * This source code is licensed under both the BSD-style license (found in the
6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7 * in the COPYING file in the root directory of this source tree).
11 #include "utils/ScopeGuard.h"
25 unsigned defaultNumThreads() {
26 #ifdef PZSTD_NUM_THREADS
27 return PZSTD_NUM_THREADS
;
29 return std::thread::hardware_concurrency();
33 unsigned parseUnsigned(const char **arg
) {
35 while (**arg
>= '0' && **arg
<= '9') {
37 result
+= **arg
- '0';
43 const char *getArgument(const char *options
, const char **argv
, int &i
,
45 if (options
[1] != 0) {
50 std::fprintf(stderr
, "Option -%c requires an argument, but none provided\n",
57 const std::string kZstdExtension
= ".zst";
58 constexpr char kStdIn
[] = "-";
59 constexpr char kStdOut
[] = "-";
60 constexpr unsigned kDefaultCompressionLevel
= 3;
61 constexpr unsigned kMaxNonUltraCompressionLevel
= 19;
64 const char nullOutput
[] = "nul";
66 const char nullOutput
[] = "/dev/null";
69 void notSupported(const char *option
) {
70 std::fprintf(stderr
, "Operation not supported: %s\n", option
);
74 std::fprintf(stderr
, "Usage:\n");
75 std::fprintf(stderr
, " pzstd [args] [FILE(s)]\n");
76 std::fprintf(stderr
, "Parallel ZSTD options:\n");
77 std::fprintf(stderr
, " -p, --processes # : number of threads to use for (de)compression (default:<numcpus>)\n");
79 std::fprintf(stderr
, "ZSTD options:\n");
80 std::fprintf(stderr
, " -# : # compression level (1-%d, default:%d)\n", kMaxNonUltraCompressionLevel
, kDefaultCompressionLevel
);
81 std::fprintf(stderr
, " -d, --decompress : decompression\n");
82 std::fprintf(stderr
, " -o file : result stored into `file` (only if 1 input file)\n");
83 std::fprintf(stderr
, " -f, --force : overwrite output without prompting, (de)compress links\n");
84 std::fprintf(stderr
, " --rm : remove source file(s) after successful (de)compression\n");
85 std::fprintf(stderr
, " -k, --keep : preserve source file(s) (default)\n");
86 std::fprintf(stderr
, " -h, --help : display help and exit\n");
87 std::fprintf(stderr
, " -V, --version : display version number and exit\n");
88 std::fprintf(stderr
, " -v, --verbose : verbose mode; specify multiple times to increase log level (default:2)\n");
89 std::fprintf(stderr
, " -q, --quiet : suppress warnings; specify twice to suppress errors too\n");
90 std::fprintf(stderr
, " -c, --stdout : force write to standard output, even if it is the console\n");
91 #ifdef UTIL_HAS_CREATEFILELIST
92 std::fprintf(stderr
, " -r : operate recursively on directories\n");
94 std::fprintf(stderr
, " --ultra : enable levels beyond %i, up to %i (requires more memory)\n", kMaxNonUltraCompressionLevel
, ZSTD_maxCLevel());
95 std::fprintf(stderr
, " -C, --check : integrity check (default)\n");
96 std::fprintf(stderr
, " --no-check : no integrity check\n");
97 std::fprintf(stderr
, " -t, --test : test compressed file integrity\n");
98 std::fprintf(stderr
, " -- : all arguments after \"--\" are treated as files\n");
100 } // anonymous namespace
103 : numThreads(defaultNumThreads()), maxWindowLog(23),
104 compressionLevel(kDefaultCompressionLevel
), decompress(false),
105 overwrite(false), keepSource(true), writeMode(WriteMode::Auto
),
106 checksum(true), verbosity(2) {}
108 Options::Status
Options::parse(int argc
, const char **argv
) {
110 bool recursive
= false;
112 bool forceStdout
= false;
113 bool followLinks
= false;
114 // Local copy of input files, which are pointers into argv.
115 std::vector
<const char *> localInputFiles
;
116 for (int i
= 1; i
< argc
; ++i
) {
117 const char *arg
= argv
[i
];
118 // Protect against empty arguments
122 // Everything after "--" is an input file
123 if (!std::strcmp(arg
, "--")) {
125 std::copy(argv
+ i
, argv
+ argc
, std::back_inserter(localInputFiles
));
128 // Long arguments that don't have a short option
130 bool isLongOption
= true;
131 if (!std::strcmp(arg
, "--rm")) {
133 } else if (!std::strcmp(arg
, "--ultra")) {
136 } else if (!std::strcmp(arg
, "--no-check")) {
138 } else if (!std::strcmp(arg
, "--sparse")) {
139 writeMode
= WriteMode::Sparse
;
140 notSupported("Sparse mode");
141 return Status::Failure
;
142 } else if (!std::strcmp(arg
, "--no-sparse")) {
143 writeMode
= WriteMode::Regular
;
144 notSupported("Sparse mode");
145 return Status::Failure
;
146 } else if (!std::strcmp(arg
, "--dictID")) {
148 return Status::Failure
;
149 } else if (!std::strcmp(arg
, "--no-dictID")) {
151 return Status::Failure
;
153 isLongOption
= false;
159 // Arguments with a short option simply set their short option.
160 const char *options
= nullptr;
161 if (!std::strcmp(arg
, "--processes")) {
163 } else if (!std::strcmp(arg
, "--version")) {
165 } else if (!std::strcmp(arg
, "--help")) {
167 } else if (!std::strcmp(arg
, "--decompress")) {
169 } else if (!std::strcmp(arg
, "--force")) {
171 } else if (!std::strcmp(arg
, "--stdout")) {
173 } else if (!std::strcmp(arg
, "--keep")) {
175 } else if (!std::strcmp(arg
, "--verbose")) {
177 } else if (!std::strcmp(arg
, "--quiet")) {
179 } else if (!std::strcmp(arg
, "--check")) {
181 } else if (!std::strcmp(arg
, "--test")) {
183 } else if (arg
[0] == '-' && arg
[1] != 0) {
186 localInputFiles
.emplace_back(arg
);
189 assert(options
!= nullptr);
191 bool finished
= false;
192 while (!finished
&& *options
!= 0) {
193 // Parse the compression level
194 if (*options
>= '0' && *options
<= '9') {
195 compressionLevel
= parseUnsigned(&options
);
203 return Status::Message
;
205 std::fprintf(stderr
, "PZSTD version: %s.\n", ZSTD_VERSION_STRING
);
206 return Status::Message
;
209 const char *optionArgument
= getArgument(options
, argv
, i
, argc
);
210 if (optionArgument
== nullptr) {
211 return Status::Failure
;
213 if (*optionArgument
< '0' || *optionArgument
> '9') {
214 std::fprintf(stderr
, "Option -p expects a number, but %s provided\n",
216 return Status::Failure
;
218 numThreads
= parseUnsigned(&optionArgument
);
219 if (*optionArgument
!= 0) {
221 "Option -p expects a number, but %u%s provided\n",
222 numThreads
, optionArgument
);
223 return Status::Failure
;
229 const char *optionArgument
= getArgument(options
, argv
, i
, argc
);
230 if (optionArgument
== nullptr) {
231 return Status::Failure
;
233 outputFile
= optionArgument
;
254 #ifdef UTIL_HAS_CREATEFILELIST
260 outputFile
= kStdOut
;
268 // Ignore them for now
270 // Unsupported options from Zstd
273 notSupported("Zstd dictionaries.");
274 return Status::Failure
;
279 notSupported("Zstd benchmarking options.");
280 return Status::Failure
;
282 std::fprintf(stderr
, "Invalid argument: %s\n", arg
);
283 return Status::Failure
;
288 } // while (*options != 0);
289 } // for (int i = 1; i < argc; ++i);
291 // Set options for test mode
293 outputFile
= nullOutput
;
297 // Input file defaults to standard input if not provided.
298 if (localInputFiles
.empty()) {
299 localInputFiles
.emplace_back(kStdIn
);
302 // Check validity of input files
303 if (localInputFiles
.size() > 1) {
304 const auto it
= std::find(localInputFiles
.begin(), localInputFiles
.end(),
305 std::string
{kStdIn
});
306 if (it
!= localInputFiles
.end()) {
309 "Cannot specify standard input when handling multiple files\n");
310 return Status::Failure
;
313 if (localInputFiles
.size() > 1 || recursive
) {
314 if (!outputFile
.empty() && outputFile
!= nullOutput
) {
317 "Cannot specify an output file when handling multiple inputs\n");
318 return Status::Failure
;
322 g_utilDisplayLevel
= verbosity
;
323 // Remove local input files that are symbolic links
325 std::remove_if(localInputFiles
.begin(), localInputFiles
.end(),
326 [&](const char *path
) {
327 bool isLink
= UTIL_isLink(path
);
328 if (isLink
&& verbosity
>= 2) {
331 "Warning : %s is symbolic link, ignoring\n",
338 // Translate input files/directories into files to (de)compress
340 char *scratchBuffer
= nullptr;
341 unsigned numFiles
= 0;
343 UTIL_createFileList(localInputFiles
.data(), localInputFiles
.size(),
344 &scratchBuffer
, &numFiles
, followLinks
);
345 if (files
== nullptr) {
346 std::fprintf(stderr
, "Error traversing directories\n");
347 return Status::Failure
;
350 makeScopeGuard([&] { UTIL_freeFileList(files
, scratchBuffer
); });
352 std::fprintf(stderr
, "No files found\n");
353 return Status::Failure
;
355 inputFiles
.resize(numFiles
);
356 std::copy(files
, files
+ numFiles
, inputFiles
.begin());
358 inputFiles
.resize(localInputFiles
.size());
359 std::copy(localInputFiles
.begin(), localInputFiles
.end(),
362 localInputFiles
.clear();
363 assert(!inputFiles
.empty());
365 // If reading from standard input, default to standard output
366 if (inputFiles
[0] == kStdIn
&& outputFile
.empty()) {
367 assert(inputFiles
.size() == 1);
371 if (inputFiles
[0] == kStdIn
&& IS_CONSOLE(stdin
)) {
372 assert(inputFiles
.size() == 1);
373 std::fprintf(stderr
, "Cannot read input from interactive console\n");
374 return Status::Failure
;
376 if (outputFile
== "-" && IS_CONSOLE(stdout
) && !(forceStdout
&& decompress
)) {
377 std::fprintf(stderr
, "Will not write to console stdout unless -c or -f is "
378 "specified and decompressing\n");
379 return Status::Failure
;
382 // Check compression level
385 ultra
? ZSTD_maxCLevel() : kMaxNonUltraCompressionLevel
;
386 if (compressionLevel
> maxCLevel
|| compressionLevel
== 0) {
387 std::fprintf(stderr
, "Invalid compression level %u.\n", compressionLevel
);
388 return Status::Failure
;
392 // Check that numThreads is set
393 if (numThreads
== 0) {
394 std::fprintf(stderr
, "Invalid arguments: # of threads not specified "
395 "and unable to determine hardware concurrency.\n");
396 return Status::Failure
;
400 // If we are piping input and output, turn off interaction
401 if (inputFiles
[0] == kStdIn
&& outputFile
== kStdOut
&& verbosity
== 2) {
404 // If we are in multi-file mode, turn off interaction
405 if (inputFiles
.size() > 1 && verbosity
== 2) {
409 return Status::Success
;
412 std::string
Options::getOutputFile(const std::string
&inputFile
) const {
413 if (!outputFile
.empty()) {
416 // Attempt to add/remove zstd extension from the input file
418 int stemSize
= inputFile
.size() - kZstdExtension
.size();
419 if (stemSize
> 0 && inputFile
.substr(stemSize
) == kZstdExtension
) {
420 return inputFile
.substr(0, stemSize
);
425 return inputFile
+ kZstdExtension
;