2 * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
5 * This source code is licensed under the BSD-style license found in the
6 * LICENSE file in the root directory of this source tree. An additional grant
7 * of patent rights can be found in the PATENTS file in the same directory.
11 /*-************************************
13 **************************************/
14 #ifndef ZSTDCLI_CLEVEL_DEFAULT
15 # define ZSTDCLI_CLEVEL_DEFAULT 3
18 #ifndef ZSTDCLI_CLEVEL_MAX
19 # define ZSTDCLI_CLEVEL_MAX 19 /* when not using --ultra */
23 /*-************************************
25 **************************************/
26 #include "util.h" /* Compiler options, UTIL_HAS_CREATEFILELIST */
27 #include <string.h> /* strcmp, strlen */
28 #include <errno.h> /* errno */
31 # include "bench.h" /* BMK_benchFiles, BMK_SetNbSeconds */
36 #define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */
37 #include "zstd.h" /* ZSTD_VERSION_STRING */
40 /*-************************************
41 * OS-specific Includes
42 **************************************/
43 #if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) || defined(__CYGWIN__)
44 # include <io.h> /* _isatty */
45 # define IS_CONSOLE(stdStream) _isatty(_fileno(stdStream))
46 #elif defined(_POSIX_C_SOURCE) || defined(_XOPEN_SOURCE) || defined(_POSIX_SOURCE) || (defined(__APPLE__) && defined(__MACH__)) || \
47 defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* https://sourceforge.net/p/predef/wiki/OperatingSystems/ */
48 # include <unistd.h> /* isatty */
49 # define IS_CONSOLE(stdStream) isatty(fileno(stdStream))
51 # define IS_CONSOLE(stdStream) 0
55 /*-************************************
57 **************************************/
58 #define COMPRESSOR_NAME "zstd command line interface"
60 # define ZSTD_VERSION "v" ZSTD_VERSION_STRING
62 #define AUTHOR "Yann Collet"
63 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
65 #define ZSTD_EXTENSION ".zst"
66 #define ZSTD_CAT "zstdcat"
67 #define ZSTD_UNZSTD "unzstd"
73 #define DEFAULT_DISPLAY_LEVEL 2
75 static const char* g_defaultDictName
= "dictionary";
76 static const unsigned g_defaultMaxDictSize
= 110 KB
;
77 static const int g_defaultDictCLevel
= 3;
78 static const unsigned g_defaultSelectivityLevel
= 9;
81 /*-************************************
83 **************************************/
84 #define DISPLAY(...) fprintf(displayOut, __VA_ARGS__)
85 #define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); }
86 static FILE* displayOut
;
87 static unsigned displayLevel
= DEFAULT_DISPLAY_LEVEL
; /* 0 : no display, 1: errors, 2 : + result + interaction + warnings, 3 : + progression, 4 : + information */
90 /*-************************************
92 **************************************/
93 static int usage(const char* programName
)
95 DISPLAY( "Usage :\n");
96 DISPLAY( " %s [args] [FILE(s)] [-o file]\n", programName
);
98 DISPLAY( "FILE : a filename\n");
99 DISPLAY( " with no FILE, or when FILE is - , read standard input\n");
100 DISPLAY( "Arguments :\n");
101 #ifndef ZSTD_NOCOMPRESS
102 DISPLAY( " -# : # compression level (1-%d, default:%d) \n", ZSTDCLI_CLEVEL_MAX
, ZSTDCLI_CLEVEL_DEFAULT
);
104 #ifndef ZSTD_NODECOMPRESS
105 DISPLAY( " -d : decompression \n");
107 DISPLAY( " -D file: use `file` as Dictionary \n");
108 DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
109 DISPLAY( " -f : overwrite output without prompting \n");
110 DISPLAY( "--rm : remove source file(s) after successful de/compression \n");
111 DISPLAY( " -k : preserve source file(s) (default) \n");
112 DISPLAY( " -h/-H : display help/long help and exit\n");
116 static int usage_advanced(const char* programName
)
118 DISPLAY(WELCOME_MESSAGE
);
121 DISPLAY( "Advanced arguments :\n");
122 DISPLAY( " -V : display Version number and exit\n");
123 DISPLAY( " -v : verbose mode; specify multiple times to increase log level (default:%d)\n", DEFAULT_DISPLAY_LEVEL
);
124 DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n");
125 DISPLAY( " -c : force write to standard output, even if it is the console\n");
126 #ifdef UTIL_HAS_CREATEFILELIST
127 DISPLAY( " -r : operate recursively on directories\n");
129 #ifndef ZSTD_NOCOMPRESS
130 DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX
, ZSTD_maxCLevel());
131 DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
132 DISPLAY( "--[no-]check : integrity check (default:enabled)\n");
134 #ifndef ZSTD_NODECOMPRESS
135 DISPLAY( "--test : test compressed file integrity \n");
136 DISPLAY( "--[no-]sparse : sparse mode (default:enabled on file, disabled on stdout)\n");
138 DISPLAY( " -M# : Set a memory usage limit for decompression \n");
139 DISPLAY( "-- : All arguments after \"--\" are treated as files \n");
142 DISPLAY( "Dictionary builder :\n");
143 DISPLAY( "--train ## : create a dictionary from a training set of files \n");
144 DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName
);
145 DISPLAY( "--maxdict ## : limit dictionary to specified size (default : %u) \n", g_defaultMaxDictSize
);
146 DISPLAY( " -s# : dictionary selectivity level (default: %u)\n", g_defaultSelectivityLevel
);
147 DISPLAY( "--dictID ## : force dictionary ID to specified value (default: random)\n");
151 DISPLAY( "Benchmark arguments :\n");
152 DISPLAY( " -b# : benchmark file(s), using # compression level (default : 1) \n");
153 DISPLAY( " -e# : test all compression levels from -bX to # (default: 1)\n");
154 DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n");
155 DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n");
160 static int badusage(const char* programName
)
162 DISPLAYLEVEL(1, "Incorrect parameters\n");
163 if (displayLevel
>= 1) usage(programName
);
167 static void waitEnter(void)
170 DISPLAY("Press enter to continue...\n");
175 /*! readU32FromChar() :
176 @return : unsigned integer value read from input in `char` format
177 allows and interprets K, KB, KiB, M, MB and MiB suffix.
178 Will also modify `*stringPtr`, advancing it to position where it stopped reading.
179 Note : function result can overflow if digit string > MAX_UINT */
180 static unsigned readU32FromChar(const char** stringPtr
)
183 while ((**stringPtr
>='0') && (**stringPtr
<='9'))
184 result
*= 10, result
+= **stringPtr
- '0', (*stringPtr
)++ ;
185 if ((**stringPtr
=='K') || (**stringPtr
=='M')) {
187 if (**stringPtr
=='M') result
<<= 10;
189 if (**stringPtr
=='i') (*stringPtr
)++;
190 if (**stringPtr
=='B') (*stringPtr
)++;
195 /** longCommandWArg() :
196 * check is *stringPtr is the same as longCommand.
197 * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
198 * @return 0 and doesn't modify *stringPtr otherwise.
200 static unsigned longCommandWArg(const char** stringPtr
, const char* longCommand
)
202 size_t const comSize
= strlen(longCommand
);
203 int const result
= !strncmp(*stringPtr
, longCommand
, comSize
);
204 if (result
) *stringPtr
+= comSize
;
208 typedef enum { zom_compress
, zom_decompress
, zom_test
, zom_bench
, zom_train
} zstd_operation_mode
;
210 #define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
212 int main(int argCount
, const char* argv
[])
217 nextEntryIsDictionary
=0,
219 nextArgumentIsOutFileName
=0,
220 nextArgumentIsMaxDict
=0,
221 nextArgumentIsDictID
=0,
222 nextArgumentsAreFiles
=0,
225 zstd_operation_mode operation
= zom_compress
;
226 int cLevel
= ZSTDCLI_CLEVEL_DEFAULT
;
228 unsigned recursive
= 0;
229 unsigned memLimit
= 0;
230 const char** filenameTable
= (const char**)malloc(argCount
* sizeof(const char*)); /* argCount >= 1 */
231 unsigned filenameIdx
= 0;
232 const char* programName
= argv
[0];
233 const char* outFileName
= NULL
;
234 const char* dictFileName
= NULL
;
235 unsigned maxDictSize
= g_defaultMaxDictSize
;
237 int dictCLevel
= g_defaultDictCLevel
;
238 unsigned dictSelect
= g_defaultSelectivityLevel
;
239 #ifdef UTIL_HAS_CREATEFILELIST
240 const char** extendedFileList
= NULL
;
241 char* fileNamesBuf
= NULL
;
242 unsigned fileNamesNb
;
246 (void)recursive
; (void)cLevelLast
; /* not used when ZSTD_NOBENCH set */
247 (void)dictCLevel
; (void)dictSelect
; (void)dictID
; (void)maxDictSize
; /* not used when ZSTD_NODICT set */
248 (void)ultra
; (void)cLevel
; /* not used when ZSTD_NOCOMPRESS set */
249 (void)memLimit
; /* not used when ZSTD_NODECOMPRESS set */
250 if (filenameTable
==NULL
) { DISPLAY("zstd: %s \n", strerror(errno
)); exit(1); }
251 filenameTable
[0] = stdinmark
;
253 /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */
255 for (pos
= (int)strlen(programName
); pos
> 0; pos
--) { if (programName
[pos
] == '/') { pos
++; break; } }
259 /* preset behaviors */
260 if (!strcmp(programName
, ZSTD_UNZSTD
)) operation
=zom_decompress
;
261 if (!strcmp(programName
, ZSTD_CAT
)) { operation
=zom_decompress
; forceStdout
=1; FIO_overwriteMode(); outFileName
=stdoutmark
; displayLevel
=1; }
263 /* command switches */
264 for (argNb
=1; argNb
<argCount
; argNb
++) {
265 const char* argument
= argv
[argNb
];
266 if(!argument
) continue; /* Protection if argument empty */
268 if (nextArgumentsAreFiles
==0) {
269 /* "-" means stdin/stdout */
270 if (!strcmp(argument
, "-")){
272 filenameIdx
=1, filenameTable
[0]=stdinmark
;
273 outFileName
=stdoutmark
;
274 displayLevel
-=(displayLevel
==2);
278 /* Decode commands (note : aggregated commands are allowed) */
279 if (argument
[0]=='-') {
281 if (argument
[1]=='-') {
282 /* long commands (--long-word) */
283 if (!strcmp(argument
, "--")) { nextArgumentsAreFiles
=1; continue; } /* only file names allowed from now on */
284 if (!strcmp(argument
, "--compress")) { operation
=zom_compress
; continue; }
285 if (!strcmp(argument
, "--decompress")) { operation
=zom_decompress
; continue; }
286 if (!strcmp(argument
, "--uncompress")) { operation
=zom_decompress
; continue; }
287 if (!strcmp(argument
, "--force")) { FIO_overwriteMode(); continue; }
288 if (!strcmp(argument
, "--version")) { displayOut
=stdout
; DISPLAY(WELCOME_MESSAGE
); CLEAN_RETURN(0); }
289 if (!strcmp(argument
, "--help")) { displayOut
=stdout
; CLEAN_RETURN(usage_advanced(programName
)); }
290 if (!strcmp(argument
, "--verbose")) { displayLevel
++; continue; }
291 if (!strcmp(argument
, "--quiet")) { displayLevel
--; continue; }
292 if (!strcmp(argument
, "--stdout")) { forceStdout
=1; outFileName
=stdoutmark
; displayLevel
-=(displayLevel
==2); continue; }
293 if (!strcmp(argument
, "--ultra")) { ultra
=1; continue; }
294 if (!strcmp(argument
, "--check")) { FIO_setChecksumFlag(2); continue; }
295 if (!strcmp(argument
, "--no-check")) { FIO_setChecksumFlag(0); continue; }
296 if (!strcmp(argument
, "--sparse")) { FIO_setSparseWrite(2); continue; }
297 if (!strcmp(argument
, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
298 if (!strcmp(argument
, "--test")) { operation
=zom_test
; continue; }
299 if (!strcmp(argument
, "--train")) { operation
=zom_train
; outFileName
=g_defaultDictName
; continue; }
300 if (!strcmp(argument
, "--maxdict")) { nextArgumentIsMaxDict
=1; lastCommand
=1; continue; }
301 if (!strcmp(argument
, "--dictID")) { nextArgumentIsDictID
=1; lastCommand
=1; continue; }
302 if (!strcmp(argument
, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
303 if (!strcmp(argument
, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
304 if (!strcmp(argument
, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
306 /* long commands with arguments */
307 if (longCommandWArg(&argument
, "--memlimit=")) { memLimit
= readU32FromChar(&argument
); continue; }
308 if (longCommandWArg(&argument
, "--memory=")) { memLimit
= readU32FromChar(&argument
); continue; }
309 if (longCommandWArg(&argument
, "--memlimit-decompress=")) { memLimit
= readU32FromChar(&argument
); continue; }
310 /* fall-through, will trigger bad_usage() later on */
314 while (argument
[0]!=0) {
316 DISPLAY("error : command must be followed by argument \n");
319 #ifndef ZSTD_NOCOMPRESS
320 /* compression Level */
321 if ((*argument
>='0') && (*argument
<='9')) {
322 dictCLevel
= cLevel
= readU32FromChar(&argument
);
330 case 'V': displayOut
=stdout
; DISPLAY(WELCOME_MESSAGE
); CLEAN_RETURN(0); /* Version Only */
332 case 'h': displayOut
=stdout
; CLEAN_RETURN(usage_advanced(programName
));
335 case 'z': operation
=zom_compress
; argument
++; break;
340 if (operation
==zom_bench
) { BMK_setDecodeOnly(1); argument
++; break; } /* benchmark decode (hidden option) */
342 operation
=zom_decompress
; argument
++; break;
344 /* Force stdout, even if stdout==console */
345 case 'c': forceStdout
=1; outFileName
=stdoutmark
; argument
++; break;
347 /* Use file content as dictionary */
348 case 'D': nextEntryIsDictionary
= 1; lastCommand
= 1; argument
++; break;
351 case 'f': FIO_overwriteMode(); forceStdout
=1; argument
++; break;
354 case 'v': displayLevel
++; argument
++; break;
357 case 'q': displayLevel
--; argument
++; break;
359 /* keep source file (default); for gzip/xz compatibility */
360 case 'k': FIO_setRemoveSrcFile(0); argument
++; break;
363 case 'C': argument
++; FIO_setChecksumFlag(2); break;
365 /* test compressed file */
366 case 't': operation
=zom_test
; argument
++; break;
368 /* destination file name */
369 case 'o': nextArgumentIsOutFileName
=1; lastCommand
=1; argument
++; break;
371 /* limit decompression memory */
374 memLimit
= readU32FromChar(&argument
);
377 #ifdef UTIL_HAS_CREATEFILELIST
379 case 'r': recursive
=1; argument
++; break;
384 case 'b': operation
=zom_bench
; argument
++; break;
386 /* range bench (benchmark only) */
388 /* compression Level */
390 cLevelLast
= readU32FromChar(&argument
);
393 /* Modify Nb Iterations (benchmark only) */
396 { U32
const iters
= readU32FromChar(&argument
);
397 BMK_setNotificationLevel(displayLevel
);
398 BMK_SetNbSeconds(iters
);
402 /* cut input into blocks (benchmark only) */
405 { size_t const bSize
= readU32FromChar(&argument
);
406 BMK_setNotificationLevel(displayLevel
);
407 BMK_SetBlockSize(bSize
);
410 #endif /* ZSTD_NOBENCH */
412 /* Dictionary Selection level */
415 dictSelect
= readU32FromChar(&argument
);
418 /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
419 case 'p': argument
++;
421 if ((*argument
>='0') && (*argument
<='9')) {
422 BMK_setAdditionalParam(readU32FromChar(&argument
));
427 /* unknown command */
428 default : CLEAN_RETURN(badusage(programName
));
432 } /* if (argument[0]=='-') */
434 if (nextArgumentIsMaxDict
) {
435 nextArgumentIsMaxDict
= 0;
437 maxDictSize
= readU32FromChar(&argument
);
441 if (nextArgumentIsDictID
) {
442 nextArgumentIsDictID
= 0;
444 dictID
= readU32FromChar(&argument
);
448 } /* if (nextArgumentIsAFile==0) */
450 if (nextEntryIsDictionary
) {
451 nextEntryIsDictionary
= 0;
453 dictFileName
= argument
;
457 if (nextArgumentIsOutFileName
) {
458 nextArgumentIsOutFileName
= 0;
460 outFileName
= argument
;
461 if (!strcmp(outFileName
, "-")) outFileName
= stdoutmark
;
465 /* add filename to list */
466 filenameTable
[filenameIdx
++] = argument
;
469 if (lastCommand
) { DISPLAY("error : command must be followed by argument \n"); return 1; } /* forgotten argument */
471 /* Welcome message (if verbose) */
472 DISPLAYLEVEL(3, WELCOME_MESSAGE
);
474 #ifdef UTIL_HAS_CREATEFILELIST
475 if (recursive
) { /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
476 extendedFileList
= UTIL_createFileList(filenameTable
, filenameIdx
, &fileNamesBuf
, &fileNamesNb
);
477 if (extendedFileList
) {
479 for (u
=0; u
<fileNamesNb
; u
++) DISPLAYLEVEL(4, "%u %s\n", u
, extendedFileList
[u
]);
480 free((void*)filenameTable
);
481 filenameTable
= extendedFileList
;
482 filenameIdx
= fileNamesNb
;
487 /* Check if benchmark is selected */
488 if (operation
==zom_bench
) {
490 BMK_setNotificationLevel(displayLevel
);
491 BMK_benchFiles(filenameTable
, filenameIdx
, dictFileName
, cLevel
, cLevelLast
);
496 /* Check if dictionary builder is selected */
497 if (operation
==zom_train
) {
499 ZDICT_params_t dictParams
;
500 memset(&dictParams
, 0, sizeof(dictParams
));
501 dictParams
.compressionLevel
= dictCLevel
;
502 dictParams
.selectivityLevel
= dictSelect
;
503 dictParams
.notificationLevel
= displayLevel
;
504 dictParams
.dictID
= dictID
;
505 DiB_trainFromFiles(outFileName
, maxDictSize
, filenameTable
, filenameIdx
, dictParams
);
510 /* No input filename ==> use stdin and stdout */
511 filenameIdx
+= !filenameIdx
; /* filenameTable[0] is stdin by default */
512 if (!strcmp(filenameTable
[0], stdinmark
) && !outFileName
) outFileName
= stdoutmark
; /* when input is stdin, default output is stdout */
514 /* Check if input/output defined as console; trigger an error in this case */
515 if (!strcmp(filenameTable
[0], stdinmark
) && IS_CONSOLE(stdin
) ) CLEAN_RETURN(badusage(programName
));
516 if (outFileName
&& !strcmp(outFileName
, stdoutmark
) && IS_CONSOLE(stdout
) && strcmp(filenameTable
[0], stdinmark
) && !(forceStdout
&& (operation
==zom_decompress
)))
517 CLEAN_RETURN(badusage(programName
));
519 /* user-selected output filename, only possible with a single file */
520 if (outFileName
&& strcmp(outFileName
,stdoutmark
) && strcmp(outFileName
,nulmark
) && (filenameIdx
>1)) {
521 DISPLAY("Too many files (%u) on the command line. \n", filenameIdx
);
522 CLEAN_RETURN(filenameIdx
);
525 #ifndef ZSTD_NOCOMPRESS
526 /* check compression level limits */
527 { int const maxCLevel
= ultra
? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX
;
528 if (cLevel
> maxCLevel
) {
529 DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel
);
534 /* No warning message in pipe mode (stdin + stdout) or multi-files mode */
535 if (!strcmp(filenameTable
[0], stdinmark
) && outFileName
&& !strcmp(outFileName
,stdoutmark
) && (displayLevel
==2)) displayLevel
=1;
536 if ((filenameIdx
>1) & (displayLevel
==2)) displayLevel
=1;
539 FIO_setNotificationLevel(displayLevel
);
540 if (operation
==zom_compress
) {
541 #ifndef ZSTD_NOCOMPRESS
542 if ((filenameIdx
==1) && outFileName
)
543 operationResult
= FIO_compressFilename(outFileName
, filenameTable
[0], dictFileName
, cLevel
);
545 operationResult
= FIO_compressMultipleFilenames(filenameTable
, filenameIdx
, outFileName
? outFileName
: ZSTD_EXTENSION
, dictFileName
, cLevel
);
547 DISPLAY("Compression not supported\n");
549 } else { /* decompression or test */
550 #ifndef ZSTD_NODECOMPRESS
551 if (operation
==zom_test
) { outFileName
=nulmark
; FIO_setRemoveSrcFile(0); } /* test mode */
552 FIO_setMemLimit(memLimit
);
553 if (filenameIdx
==1 && outFileName
)
554 operationResult
= FIO_decompressFilename(outFileName
, filenameTable
[0], dictFileName
);
556 operationResult
= FIO_decompressMultipleFilenames(filenameTable
, filenameIdx
, outFileName
? outFileName
: ZSTD_EXTENSION
, dictFileName
);
558 DISPLAY("Decompression not supported\n");
563 if (main_pause
) waitEnter();
564 #ifdef UTIL_HAS_CREATEFILELIST
565 if (extendedFileList
)
566 UTIL_freeFileList(extendedFileList
, fileNamesBuf
);
569 free((void*)filenameTable
);
570 return operationResult
;