]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | From 57a3cf95b276946559f9e044c7352c11303bb9c1 Mon Sep 17 00:00:00 2001 |
2 | From: Sean Purcell <me@seanp.xyz> | |
3 | Date: Thu, 3 Aug 2017 17:47:03 -0700 | |
4 | Subject: [PATCH v6] squashfs-tools: Add zstd support | |
5 | ||
6 | This patch adds zstd support to squashfs-tools. It works with zstd | |
7 | versions >= 1.0.0. It was originally written by Sean Purcell. | |
8 | ||
9 | Signed-off-by: Sean Purcell <me@seanp.xyz> | |
10 | Signed-off-by: Nick Terrell <terrelln@fb.com> | |
11 | --- | |
12 | v4 -> v5: | |
13 | - Fix patch documentation to reflect that Sean Purcell is the author | |
9f95a23c | 14 | - Don't strip trailing whitespace of unrelated code |
11fdf7f2 TL |
15 | - Make zstd_display_options() static |
16 | ||
17 | v5 -> v6: | |
18 | - Fix build instructions in Makefile | |
19 | ||
20 | squashfs-tools/Makefile | 20 ++++ | |
21 | squashfs-tools/compressor.c | 8 ++ | |
22 | squashfs-tools/squashfs_fs.h | 1 + | |
23 | squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++++++++++ | |
24 | squashfs-tools/zstd_wrapper.h | 48 ++++++++ | |
25 | 5 files changed, 331 insertions(+) | |
26 | create mode 100644 squashfs-tools/zstd_wrapper.c | |
27 | create mode 100644 squashfs-tools/zstd_wrapper.h | |
28 | ||
29 | diff --git a/squashfs-tools/Makefile b/squashfs-tools/Makefile | |
30 | index 52d2582..22fc559 100644 | |
31 | --- a/squashfs-tools/Makefile | |
32 | +++ b/squashfs-tools/Makefile | |
33 | @@ -75,6 +75,18 @@ GZIP_SUPPORT = 1 | |
34 | #LZMA_SUPPORT = 1 | |
35 | #LZMA_DIR = ../../../../LZMA/lzma465 | |
36 | ||
37 | + | |
38 | +########### Building ZSTD support ############ | |
39 | +# | |
40 | +# The ZSTD library is supported | |
41 | +# ZSTD homepage: http://zstd.net | |
42 | +# ZSTD source repository: https://github.com/facebook/zstd | |
43 | +# | |
44 | +# To build using the ZSTD library - install the library and uncomment the | |
45 | +# ZSTD_SUPPORT line below. | |
46 | +# | |
47 | +#ZSTD_SUPPORT = 1 | |
48 | + | |
49 | ######## Specifying default compression ######## | |
50 | # | |
51 | # The next line specifies which compression algorithm is used by default | |
52 | @@ -177,6 +189,14 @@ LIBS += -llz4 | |
53 | COMPRESSORS += lz4 | |
54 | endif | |
55 | ||
56 | +ifeq ($(ZSTD_SUPPORT),1) | |
57 | +CFLAGS += -DZSTD_SUPPORT | |
58 | +MKSQUASHFS_OBJS += zstd_wrapper.o | |
59 | +UNSQUASHFS_OBJS += zstd_wrapper.o | |
60 | +LIBS += -lzstd | |
61 | +COMPRESSORS += zstd | |
62 | +endif | |
63 | + | |
64 | ifeq ($(XATTR_SUPPORT),1) | |
65 | ifeq ($(XATTR_DEFAULT),1) | |
66 | CFLAGS += -DXATTR_SUPPORT -DXATTR_DEFAULT | |
67 | diff --git a/squashfs-tools/compressor.c b/squashfs-tools/compressor.c | |
68 | index 525e316..02b5e90 100644 | |
69 | --- a/squashfs-tools/compressor.c | |
70 | +++ b/squashfs-tools/compressor.c | |
71 | @@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = { | |
72 | extern struct compressor xz_comp_ops; | |
73 | #endif | |
74 | ||
75 | +#ifndef ZSTD_SUPPORT | |
76 | +static struct compressor zstd_comp_ops = { | |
77 | + ZSTD_COMPRESSION, "zstd" | |
78 | +}; | |
79 | +#else | |
80 | +extern struct compressor zstd_comp_ops; | |
81 | +#endif | |
82 | ||
83 | static struct compressor unknown_comp_ops = { | |
84 | 0, "unknown" | |
85 | @@ -77,6 +84,7 @@ struct compressor *compressor[] = { | |
86 | &lzo_comp_ops, | |
87 | &lz4_comp_ops, | |
88 | &xz_comp_ops, | |
89 | + &zstd_comp_ops, | |
90 | &unknown_comp_ops | |
91 | }; | |
92 | ||
93 | diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h | |
94 | index 791fe12..afca918 100644 | |
95 | --- a/squashfs-tools/squashfs_fs.h | |
96 | +++ b/squashfs-tools/squashfs_fs.h | |
97 | @@ -277,6 +277,7 @@ typedef long long squashfs_inode; | |
98 | #define LZO_COMPRESSION 3 | |
99 | #define XZ_COMPRESSION 4 | |
100 | #define LZ4_COMPRESSION 5 | |
101 | +#define ZSTD_COMPRESSION 6 | |
102 | ||
103 | struct squashfs_super_block { | |
104 | unsigned int s_magic; | |
105 | diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c | |
106 | new file mode 100644 | |
107 | index 0000000..dcab75a | |
108 | --- /dev/null | |
109 | +++ b/squashfs-tools/zstd_wrapper.c | |
110 | @@ -0,0 +1,254 @@ | |
111 | +/* | |
112 | + * Copyright (c) 2017 | |
113 | + * Phillip Lougher <phillip@squashfs.org.uk> | |
114 | + * | |
115 | + * This program is free software; you can redistribute it and/or | |
116 | + * modify it under the terms of the GNU General Public License | |
117 | + * as published by the Free Software Foundation; either version 2, | |
118 | + * or (at your option) any later version. | |
119 | + * | |
120 | + * This program is distributed in the hope that it will be useful, | |
121 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
122 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
123 | + * GNU General Public License for more details. | |
124 | + * | |
125 | + * zstd_wrapper.c | |
126 | + * | |
127 | + * Support for ZSTD compression http://zstd.net | |
128 | + */ | |
129 | + | |
130 | +#include <stdio.h> | |
131 | +#include <string.h> | |
132 | +#include <stdlib.h> | |
133 | +#include <zstd.h> | |
134 | +#include <zstd_errors.h> | |
135 | + | |
136 | +#include "squashfs_fs.h" | |
137 | +#include "zstd_wrapper.h" | |
138 | +#include "compressor.h" | |
139 | + | |
140 | +static int compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL; | |
141 | + | |
142 | +/* | |
143 | + * This function is called by the options parsing code in mksquashfs.c | |
144 | + * to parse any -X compressor option. | |
145 | + * | |
146 | + * This function returns: | |
147 | + * >=0 (number of additional args parsed) on success | |
148 | + * -1 if the option was unrecognised, or | |
149 | + * -2 if the option was recognised, but otherwise bad in | |
150 | + * some way (e.g. invalid parameter) | |
151 | + * | |
152 | + * Note: this function sets internal compressor state, but does not | |
153 | + * pass back the results of the parsing other than success/failure. | |
154 | + * The zstd_dump_options() function is called later to get the options in | |
155 | + * a format suitable for writing to the filesystem. | |
156 | + */ | |
157 | +static int zstd_options(char *argv[], int argc) | |
158 | +{ | |
159 | + if (strcmp(argv[0], "-Xcompression-level") == 0) { | |
160 | + if (argc < 2) { | |
161 | + fprintf(stderr, "zstd: -Xcompression-level missing " | |
162 | + "compression level\n"); | |
163 | + fprintf(stderr, "zstd: -Xcompression-level it should " | |
164 | + "be 1 <= n <= %d\n", ZSTD_maxCLevel()); | |
165 | + goto failed; | |
166 | + } | |
167 | + | |
168 | + compression_level = atoi(argv[1]); | |
169 | + if (compression_level < 1 || | |
170 | + compression_level > ZSTD_maxCLevel()) { | |
171 | + fprintf(stderr, "zstd: -Xcompression-level invalid, it " | |
172 | + "should be 1 <= n <= %d\n", ZSTD_maxCLevel()); | |
173 | + goto failed; | |
174 | + } | |
175 | + | |
176 | + return 1; | |
177 | + } | |
178 | + | |
179 | + return -1; | |
180 | +failed: | |
181 | + return -2; | |
182 | +} | |
183 | + | |
184 | +/* | |
185 | + * This function is called by mksquashfs to dump the parsed | |
186 | + * compressor options in a format suitable for writing to the | |
187 | + * compressor options field in the filesystem (stored immediately | |
188 | + * after the superblock). | |
189 | + * | |
190 | + * This function returns a pointer to the compression options structure | |
191 | + * to be stored (and the size), or NULL if there are no compression | |
192 | + * options. | |
193 | + */ | |
194 | +static void *zstd_dump_options(int block_size, int *size) | |
195 | +{ | |
196 | + static struct zstd_comp_opts comp_opts; | |
197 | + | |
198 | + /* don't return anything if the options are all default */ | |
199 | + if (compression_level == ZSTD_DEFAULT_COMPRESSION_LEVEL) | |
200 | + return NULL; | |
201 | + | |
202 | + comp_opts.compression_level = compression_level; | |
203 | + | |
204 | + SQUASHFS_INSWAP_COMP_OPTS(&comp_opts); | |
205 | + | |
206 | + *size = sizeof(comp_opts); | |
207 | + return &comp_opts; | |
208 | +} | |
209 | + | |
210 | +/* | |
211 | + * This function is a helper specifically for the append mode of | |
212 | + * mksquashfs. Its purpose is to set the internal compressor state | |
213 | + * to the stored compressor options in the passed compressor options | |
214 | + * structure. | |
215 | + * | |
216 | + * In effect this function sets up the compressor options | |
217 | + * to the same state they were when the filesystem was originally | |
218 | + * generated, this is to ensure on appending, the compressor uses | |
219 | + * the same compression options that were used to generate the | |
220 | + * original filesystem. | |
221 | + * | |
222 | + * Note, even if there are no compressor options, this function is still | |
223 | + * called with an empty compressor structure (size == 0), to explicitly | |
224 | + * set the default options, this is to ensure any user supplied | |
225 | + * -X options on the appending mksquashfs command line are over-ridden. | |
226 | + * | |
9f95a23c | 227 | + * This function returns 0 on successful extraction of options, and -1 on error. |
11fdf7f2 TL |
228 | + */ |
229 | +static int zstd_extract_options(int block_size, void *buffer, int size) | |
230 | +{ | |
231 | + struct zstd_comp_opts *comp_opts = buffer; | |
232 | + | |
233 | + if (size == 0) { | |
234 | + /* Set default values */ | |
235 | + compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL; | |
236 | + return 0; | |
237 | + } | |
238 | + | |
239 | + /* we expect a comp_opts structure of sufficient size to be present */ | |
240 | + if (size < sizeof(*comp_opts)) | |
241 | + goto failed; | |
242 | + | |
243 | + SQUASHFS_INSWAP_COMP_OPTS(comp_opts); | |
244 | + | |
245 | + if (comp_opts->compression_level < 1 || | |
246 | + comp_opts->compression_level > ZSTD_maxCLevel()) { | |
247 | + fprintf(stderr, "zstd: bad compression level in compression " | |
248 | + "options structure\n"); | |
249 | + goto failed; | |
250 | + } | |
251 | + | |
252 | + compression_level = comp_opts->compression_level; | |
253 | + | |
254 | + return 0; | |
255 | + | |
256 | +failed: | |
257 | + fprintf(stderr, "zstd: error reading stored compressor options from " | |
258 | + "filesystem!\n"); | |
259 | + | |
260 | + return -1; | |
261 | +} | |
262 | + | |
263 | +static void zstd_display_options(void *buffer, int size) | |
264 | +{ | |
265 | + struct zstd_comp_opts *comp_opts = buffer; | |
266 | + | |
267 | + /* we expect a comp_opts structure of sufficient size to be present */ | |
268 | + if (size < sizeof(*comp_opts)) | |
269 | + goto failed; | |
270 | + | |
271 | + SQUASHFS_INSWAP_COMP_OPTS(comp_opts); | |
272 | + | |
273 | + if (comp_opts->compression_level < 1 || | |
274 | + comp_opts->compression_level > ZSTD_maxCLevel()) { | |
275 | + fprintf(stderr, "zstd: bad compression level in compression " | |
276 | + "options structure\n"); | |
277 | + goto failed; | |
278 | + } | |
279 | + | |
280 | + printf("\tcompression-level %d\n", comp_opts->compression_level); | |
281 | + | |
282 | + return; | |
283 | + | |
284 | +failed: | |
285 | + fprintf(stderr, "zstd: error reading stored compressor options from " | |
286 | + "filesystem!\n"); | |
287 | +} | |
288 | + | |
289 | +/* | |
290 | + * This function is called by mksquashfs to initialise the | |
291 | + * compressor, before compress() is called. | |
292 | + * | |
293 | + * This function returns 0 on success, and -1 on error. | |
294 | + */ | |
295 | +static int zstd_init(void **strm, int block_size, int datablock) | |
296 | +{ | |
297 | + ZSTD_CCtx *cctx = ZSTD_createCCtx(); | |
298 | + | |
299 | + if (!cctx) { | |
300 | + fprintf(stderr, "zstd: failed to allocate compression " | |
301 | + "context!\n"); | |
302 | + return -1; | |
303 | + } | |
304 | + | |
305 | + *strm = cctx; | |
306 | + return 0; | |
307 | +} | |
308 | + | |
309 | +static int zstd_compress(void *strm, void *dest, void *src, int size, | |
310 | + int block_size, int *error) | |
311 | +{ | |
312 | + const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size, | |
313 | + src, size, compression_level); | |
314 | + | |
315 | + if (ZSTD_isError(res)) { | |
316 | + /* FIXME: | |
317 | + * zstd does not expose stable error codes. The error enum may | |
318 | + * change between versions. Until upstream zstd stablizes the | |
319 | + * error codes, we have no way of knowing why the error occurs. | |
320 | + * zstd shouldn't fail to compress any input unless there isn't | |
321 | + * enough output space. We assume that is the cause and return | |
322 | + * the special error code for not enough output space. | |
323 | + */ | |
324 | + return 0; | |
325 | + } | |
326 | + | |
327 | + return (int)res; | |
328 | +} | |
329 | + | |
330 | +static int zstd_uncompress(void *dest, void *src, int size, int outsize, | |
331 | + int *error) | |
332 | +{ | |
333 | + const size_t res = ZSTD_decompress(dest, outsize, src, size); | |
334 | + | |
335 | + if (ZSTD_isError(res)) { | |
336 | + fprintf(stderr, "\t%d %d\n", outsize, size); | |
337 | + | |
338 | + *error = (int)ZSTD_getErrorCode(res); | |
339 | + return -1; | |
340 | + } | |
341 | + | |
342 | + return (int)res; | |
343 | +} | |
344 | + | |
345 | +static void zstd_usage(void) | |
346 | +{ | |
347 | + fprintf(stderr, "\t -Xcompression-level <compression-level>\n"); | |
348 | + fprintf(stderr, "\t\t<compression-level> should be 1 .. %d (default " | |
349 | + "%d)\n", ZSTD_maxCLevel(), ZSTD_DEFAULT_COMPRESSION_LEVEL); | |
350 | +} | |
351 | + | |
352 | +struct compressor zstd_comp_ops = { | |
353 | + .init = zstd_init, | |
354 | + .compress = zstd_compress, | |
355 | + .uncompress = zstd_uncompress, | |
356 | + .options = zstd_options, | |
357 | + .dump_options = zstd_dump_options, | |
358 | + .extract_options = zstd_extract_options, | |
359 | + .display_options = zstd_display_options, | |
360 | + .usage = zstd_usage, | |
361 | + .id = ZSTD_COMPRESSION, | |
362 | + .name = "zstd", | |
363 | + .supported = 1 | |
364 | +}; | |
365 | diff --git a/squashfs-tools/zstd_wrapper.h b/squashfs-tools/zstd_wrapper.h | |
366 | new file mode 100644 | |
367 | index 0000000..4fbef0a | |
368 | --- /dev/null | |
369 | +++ b/squashfs-tools/zstd_wrapper.h | |
370 | @@ -0,0 +1,48 @@ | |
371 | +#ifndef ZSTD_WRAPPER_H | |
372 | +#define ZSTD_WRAPPER_H | |
373 | +/* | |
374 | + * Squashfs | |
375 | + * | |
376 | + * Copyright (c) 2017 | |
377 | + * Phillip Lougher <phillip@squashfs.org.uk> | |
378 | + * | |
379 | + * This program is free software; you can redistribute it and/or | |
380 | + * modify it under the terms of the GNU General Public License | |
381 | + * as published by the Free Software Foundation; either version 2, | |
382 | + * or (at your option) any later version. | |
383 | + * | |
384 | + * This program is distributed in the hope that it will be useful, | |
385 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
386 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
387 | + * GNU General Public License for more details. | |
388 | + * | |
389 | + * zstd_wrapper.h | |
390 | + * | |
391 | + */ | |
392 | + | |
393 | +#ifndef linux | |
394 | +#define __BYTE_ORDER BYTE_ORDER | |
395 | +#define __BIG_ENDIAN BIG_ENDIAN | |
396 | +#define __LITTLE_ENDIAN LITTLE_ENDIAN | |
397 | +#else | |
398 | +#include <endian.h> | |
399 | +#endif | |
400 | + | |
401 | +#if __BYTE_ORDER == __BIG_ENDIAN | |
402 | +extern unsigned int inswap_le16(unsigned short); | |
403 | +extern unsigned int inswap_le32(unsigned int); | |
404 | + | |
405 | +#define SQUASHFS_INSWAP_COMP_OPTS(s) { \ | |
406 | + (s)->compression_level = inswap_le32((s)->compression_level); \ | |
407 | +} | |
408 | +#else | |
409 | +#define SQUASHFS_INSWAP_COMP_OPTS(s) | |
410 | +#endif | |
411 | + | |
412 | +/* Default compression */ | |
413 | +#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15 | |
414 | + | |
415 | +struct zstd_comp_opts { | |
416 | + int compression_level; | |
417 | +}; | |
418 | +#endif | |
419 | -- | |
420 | 2.9.5 |