]>
Commit | Line | Data |
---|---|---|
34dc7c2f BB |
1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * The contents of this file are subject to the terms of the | |
5 | * Common Development and Distribution License (the "License"). | |
6 | * You may not use this file except in compliance with the License. | |
7 | * | |
8 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | |
1d3ba0bf | 9 | * or https://opensource.org/licenses/CDDL-1.0. |
34dc7c2f BB |
10 | * See the License for the specific language governing permissions |
11 | * and limitations under the License. | |
12 | * | |
13 | * When distributing Covered Code, include this CDDL HEADER in each | |
14 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | |
15 | * If applicable, add the following below this CDDL HEADER, with the | |
16 | * fields enclosed by brackets "[]" replaced with your own identifying | |
17 | * information: Portions Copyright [yyyy] [name of copyright owner] | |
18 | * | |
19 | * CDDL HEADER END | |
20 | */ | |
21 | ||
22 | /* | |
428870ff | 23 | * Copyright 2009 Sun Microsystems, Inc. All rights reserved. |
34dc7c2f BB |
24 | * Use is subject to license terms. |
25 | */ | |
9759c60f ED |
26 | /* |
27 | * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. | |
28 | */ | |
34dc7c2f | 29 | |
2e528b49 | 30 | /* |
c3bd3fb4 | 31 | * Copyright (c) 2013, 2018 by Delphix. All rights reserved. |
10b3c7f5 MN |
32 | * Copyright (c) 2019, Klara Inc. |
33 | * Copyright (c) 2019, Allan Jude | |
2e528b49 MA |
34 | */ |
35 | ||
34dc7c2f | 36 | #include <sys/zfs_context.h> |
34dc7c2f | 37 | #include <sys/spa.h> |
99197f03 | 38 | #include <sys/zfeature.h> |
34dc7c2f BB |
39 | #include <sys/zio.h> |
40 | #include <sys/zio_compress.h> | |
10b3c7f5 | 41 | #include <sys/zstd/zstd.h> |
34dc7c2f | 42 | |
c3bd3fb4 TC |
43 | /* |
44 | * If nonzero, every 1/X decompression attempts will fail, simulating | |
45 | * an undetected memory error. | |
46 | */ | |
27218a32 | 47 | static unsigned long zio_decompress_fail_fraction = 0; |
c3bd3fb4 | 48 | |
34dc7c2f BB |
49 | /* |
50 | * Compression vectors. | |
51 | */ | |
a2d5643f | 52 | zio_compress_info_t zio_compress_table[ZIO_COMPRESS_FUNCTIONS] = { |
10b3c7f5 MN |
53 | {"inherit", 0, NULL, NULL, NULL}, |
54 | {"on", 0, NULL, NULL, NULL}, | |
55 | {"uncompressed", 0, NULL, NULL, NULL}, | |
56 | {"lzjb", 0, lzjb_compress, lzjb_decompress, NULL}, | |
57 | {"empty", 0, NULL, NULL, NULL}, | |
58 | {"gzip-1", 1, gzip_compress, gzip_decompress, NULL}, | |
59 | {"gzip-2", 2, gzip_compress, gzip_decompress, NULL}, | |
60 | {"gzip-3", 3, gzip_compress, gzip_decompress, NULL}, | |
61 | {"gzip-4", 4, gzip_compress, gzip_decompress, NULL}, | |
62 | {"gzip-5", 5, gzip_compress, gzip_decompress, NULL}, | |
63 | {"gzip-6", 6, gzip_compress, gzip_decompress, NULL}, | |
64 | {"gzip-7", 7, gzip_compress, gzip_decompress, NULL}, | |
65 | {"gzip-8", 8, gzip_compress, gzip_decompress, NULL}, | |
66 | {"gzip-9", 9, gzip_compress, gzip_decompress, NULL}, | |
67 | {"zle", 64, zle_compress, zle_decompress, NULL}, | |
68 | {"lz4", 0, lz4_compress_zfs, lz4_decompress_zfs, NULL}, | |
f375b23c | 69 | {"zstd", ZIO_ZSTD_LEVEL_DEFAULT, zfs_zstd_compress_wrap, |
184df27e | 70 | zfs_zstd_decompress, zfs_zstd_decompress_level}, |
34dc7c2f BB |
71 | }; |
72 | ||
10b3c7f5 MN |
73 | uint8_t |
74 | zio_complevel_select(spa_t *spa, enum zio_compress compress, uint8_t child, | |
75 | uint8_t parent) | |
76 | { | |
14e4e3cb | 77 | (void) spa; |
10b3c7f5 MN |
78 | uint8_t result; |
79 | ||
80 | if (!ZIO_COMPRESS_HASLEVEL(compress)) | |
81 | return (0); | |
82 | ||
83 | result = child; | |
84 | if (result == ZIO_COMPLEVEL_INHERIT) | |
85 | result = parent; | |
86 | ||
87 | return (result); | |
88 | } | |
89 | ||
428870ff | 90 | enum zio_compress |
99197f03 JG |
91 | zio_compress_select(spa_t *spa, enum zio_compress child, |
92 | enum zio_compress parent) | |
34dc7c2f | 93 | { |
99197f03 JG |
94 | enum zio_compress result; |
95 | ||
34dc7c2f BB |
96 | ASSERT(child < ZIO_COMPRESS_FUNCTIONS); |
97 | ASSERT(parent < ZIO_COMPRESS_FUNCTIONS); | |
99197f03 | 98 | ASSERT(parent != ZIO_COMPRESS_INHERIT); |
34dc7c2f | 99 | |
99197f03 JG |
100 | result = child; |
101 | if (result == ZIO_COMPRESS_INHERIT) | |
102 | result = parent; | |
34dc7c2f | 103 | |
99197f03 JG |
104 | if (result == ZIO_COMPRESS_ON) { |
105 | if (spa_feature_is_active(spa, SPA_FEATURE_LZ4_COMPRESS)) | |
106 | result = ZIO_COMPRESS_LZ4_ON_VALUE; | |
107 | else | |
108 | result = ZIO_COMPRESS_LEGACY_ON_VALUE; | |
109 | } | |
34dc7c2f | 110 | |
99197f03 | 111 | return (result); |
34dc7c2f BB |
112 | } |
113 | ||
a6255b7f DQ |
114 | static int |
115 | zio_compress_zeroed_cb(void *data, size_t len, void *private) | |
116 | { | |
14e4e3cb AZ |
117 | (void) private; |
118 | ||
a6255b7f | 119 | uint64_t *end = (uint64_t *)((char *)data + len); |
1c27024e | 120 | for (uint64_t *word = (uint64_t *)data; word < end; word++) |
a6255b7f DQ |
121 | if (*word != 0) |
122 | return (1); | |
123 | ||
124 | return (0); | |
125 | } | |
126 | ||
428870ff | 127 | size_t |
bff26b02 | 128 | zio_compress_data(enum zio_compress c, abd_t *src, void **dst, size_t s_len, |
10b3c7f5 | 129 | uint8_t level) |
34dc7c2f | 130 | { |
9b67f605 | 131 | size_t c_len, d_len; |
10b3c7f5 | 132 | uint8_t complevel; |
428870ff | 133 | zio_compress_info_t *ci = &zio_compress_table[c]; |
34dc7c2f | 134 | |
428870ff BB |
135 | ASSERT((uint_t)c < ZIO_COMPRESS_FUNCTIONS); |
136 | ASSERT((uint_t)c == ZIO_COMPRESS_EMPTY || ci->ci_compress != NULL); | |
34dc7c2f BB |
137 | |
138 | /* | |
139 | * If the data is all zeroes, we don't even need to allocate | |
428870ff | 140 | * a block for it. We indicate this by returning zero size. |
34dc7c2f | 141 | */ |
a6255b7f | 142 | if (abd_iterate_func(src, 0, s_len, zio_compress_zeroed_cb, NULL) == 0) |
34dc7c2f BB |
143 | return (0); |
144 | ||
428870ff BB |
145 | if (c == ZIO_COMPRESS_EMPTY) |
146 | return (s_len); | |
147 | ||
34dc7c2f | 148 | /* Compress at least 12.5% */ |
9b67f605 | 149 | d_len = s_len - (s_len >> 3); |
a6255b7f | 150 | |
10b3c7f5 MN |
151 | complevel = ci->ci_level; |
152 | ||
153 | if (c == ZIO_COMPRESS_ZSTD) { | |
154 | /* If we don't know the level, we can't compress it */ | |
155 | if (level == ZIO_COMPLEVEL_INHERIT) | |
156 | return (s_len); | |
157 | ||
158 | if (level == ZIO_COMPLEVEL_DEFAULT) | |
159 | complevel = ZIO_ZSTD_LEVEL_DEFAULT; | |
160 | else | |
161 | complevel = level; | |
162 | ||
163 | ASSERT3U(complevel, !=, ZIO_COMPLEVEL_INHERIT); | |
164 | } | |
165 | ||
bff26b02 RY |
166 | if (*dst == NULL) |
167 | *dst = zio_buf_alloc(s_len); | |
168 | ||
a6255b7f | 169 | /* No compression algorithms can read from ABDs directly */ |
1c27024e | 170 | void *tmp = abd_borrow_buf_copy(src, s_len); |
bff26b02 | 171 | c_len = ci->ci_compress(tmp, *dst, s_len, d_len, complevel); |
a6255b7f | 172 | abd_return_buf(src, tmp, s_len); |
34dc7c2f | 173 | |
428870ff BB |
174 | if (c_len > d_len) |
175 | return (s_len); | |
34dc7c2f | 176 | |
428870ff | 177 | ASSERT3U(c_len, <=, d_len); |
428870ff | 178 | return (c_len); |
34dc7c2f BB |
179 | } |
180 | ||
181 | int | |
a6255b7f | 182 | zio_decompress_data_buf(enum zio_compress c, void *src, void *dst, |
10b3c7f5 | 183 | size_t s_len, size_t d_len, uint8_t *level) |
34dc7c2f | 184 | { |
428870ff | 185 | zio_compress_info_t *ci = &zio_compress_table[c]; |
428870ff | 186 | if ((uint_t)c >= ZIO_COMPRESS_FUNCTIONS || ci->ci_decompress == NULL) |
2e528b49 | 187 | return (SET_ERROR(EINVAL)); |
34dc7c2f | 188 | |
10b3c7f5 MN |
189 | if (ci->ci_decompress_level != NULL && level != NULL) |
190 | return (ci->ci_decompress_level(src, dst, s_len, d_len, level)); | |
191 | ||
428870ff | 192 | return (ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level)); |
34dc7c2f | 193 | } |
a6255b7f DQ |
194 | |
195 | int | |
196 | zio_decompress_data(enum zio_compress c, abd_t *src, void *dst, | |
10b3c7f5 | 197 | size_t s_len, size_t d_len, uint8_t *level) |
a6255b7f DQ |
198 | { |
199 | void *tmp = abd_borrow_buf_copy(src, s_len); | |
10b3c7f5 | 200 | int ret = zio_decompress_data_buf(c, tmp, dst, s_len, d_len, level); |
a6255b7f DQ |
201 | abd_return_buf(src, tmp, s_len); |
202 | ||
c3bd3fb4 | 203 | /* |
e1cfd73f | 204 | * Decompression shouldn't fail, because we've already verified |
c3bd3fb4 TC |
205 | * the checksum. However, for extra protection (e.g. against bitflips |
206 | * in non-ECC RAM), we handle this error (and test it). | |
207 | */ | |
c3bd3fb4 | 208 | if (zio_decompress_fail_fraction != 0 && |
29274c9f | 209 | random_in_range(zio_decompress_fail_fraction) == 0) |
c3bd3fb4 TC |
210 | ret = SET_ERROR(EINVAL); |
211 | ||
a6255b7f DQ |
212 | return (ret); |
213 | } | |
10b3c7f5 MN |
214 | |
215 | int | |
216 | zio_compress_to_feature(enum zio_compress comp) | |
217 | { | |
218 | switch (comp) { | |
219 | case ZIO_COMPRESS_ZSTD: | |
220 | return (SPA_FEATURE_ZSTD_COMPRESS); | |
221 | default: | |
6954c22f | 222 | break; |
10b3c7f5 MN |
223 | } |
224 | return (SPA_FEATURE_NONE); | |
225 | } |