1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2008 Oracle. All rights reserved.
5 * Based on jffs2 zlib code:
6 * Copyright © 2001-2007 Red Hat, Inc.
7 * Created by David Woodhouse <dwmw2@infradead.org>
10 #include <linux/kernel.h>
11 #include <linux/slab.h>
12 #include <linux/zlib.h>
13 #include <linux/zutil.h>
15 #include <linux/init.h>
16 #include <linux/err.h>
17 #include <linux/sched.h>
18 #include <linux/pagemap.h>
19 #include <linux/bio.h>
20 #include <linux/refcount.h>
21 #include "compression.h"
23 /* workspace buffer size for s390 zlib hardware support */
24 #define ZLIB_DFLTCC_BUF_SIZE (4 * PAGE_SIZE)
29 unsigned int buf_size
;
30 struct list_head list
;
34 static struct workspace_manager wsm
;
36 struct list_head
*zlib_get_workspace(unsigned int level
)
38 struct list_head
*ws
= btrfs_get_workspace(BTRFS_COMPRESS_ZLIB
, level
);
39 struct workspace
*workspace
= list_entry(ws
, struct workspace
, list
);
41 workspace
->level
= level
;
46 void zlib_free_workspace(struct list_head
*ws
)
48 struct workspace
*workspace
= list_entry(ws
, struct workspace
, list
);
50 kvfree(workspace
->strm
.workspace
);
51 kfree(workspace
->buf
);
55 struct list_head
*zlib_alloc_workspace(unsigned int level
)
57 struct workspace
*workspace
;
60 workspace
= kzalloc(sizeof(*workspace
), GFP_KERNEL
);
62 return ERR_PTR(-ENOMEM
);
64 workspacesize
= max(zlib_deflate_workspacesize(MAX_WBITS
, MAX_MEM_LEVEL
),
65 zlib_inflate_workspacesize());
66 workspace
->strm
.workspace
= kvmalloc(workspacesize
, GFP_KERNEL
);
67 workspace
->level
= level
;
68 workspace
->buf
= NULL
;
70 * In case of s390 zlib hardware support, allocate lager workspace
71 * buffer. If allocator fails, fall back to a single page buffer.
73 if (zlib_deflate_dfltcc_enabled()) {
74 workspace
->buf
= kmalloc(ZLIB_DFLTCC_BUF_SIZE
,
75 __GFP_NOMEMALLOC
| __GFP_NORETRY
|
76 __GFP_NOWARN
| GFP_NOIO
);
77 workspace
->buf_size
= ZLIB_DFLTCC_BUF_SIZE
;
79 if (!workspace
->buf
) {
80 workspace
->buf
= kmalloc(PAGE_SIZE
, GFP_KERNEL
);
81 workspace
->buf_size
= PAGE_SIZE
;
83 if (!workspace
->strm
.workspace
|| !workspace
->buf
)
86 INIT_LIST_HEAD(&workspace
->list
);
88 return &workspace
->list
;
90 zlib_free_workspace(&workspace
->list
);
91 return ERR_PTR(-ENOMEM
);
94 int zlib_compress_pages(struct list_head
*ws
, struct address_space
*mapping
,
95 u64 start
, struct page
**pages
, unsigned long *out_pages
,
96 unsigned long *total_in
, unsigned long *total_out
)
98 struct workspace
*workspace
= list_entry(ws
, struct workspace
, list
);
103 struct page
*in_page
= NULL
;
104 struct page
*out_page
= NULL
;
105 unsigned long bytes_left
;
106 unsigned int in_buf_pages
;
107 unsigned long len
= *total_out
;
108 unsigned long nr_dest_pages
= *out_pages
;
109 const unsigned long max_out
= nr_dest_pages
* PAGE_SIZE
;
115 if (Z_OK
!= zlib_deflateInit(&workspace
->strm
, workspace
->level
)) {
116 pr_warn("BTRFS: deflateInit failed\n");
121 workspace
->strm
.total_in
= 0;
122 workspace
->strm
.total_out
= 0;
124 out_page
= alloc_page(GFP_NOFS
);
125 if (out_page
== NULL
) {
129 cpage_out
= kmap(out_page
);
133 workspace
->strm
.next_in
= workspace
->buf
;
134 workspace
->strm
.avail_in
= 0;
135 workspace
->strm
.next_out
= cpage_out
;
136 workspace
->strm
.avail_out
= PAGE_SIZE
;
138 while (workspace
->strm
.total_in
< len
) {
140 * Get next input pages and copy the contents to
141 * the workspace buffer if required.
143 if (workspace
->strm
.avail_in
== 0) {
144 bytes_left
= len
- workspace
->strm
.total_in
;
145 in_buf_pages
= min(DIV_ROUND_UP(bytes_left
, PAGE_SIZE
),
146 workspace
->buf_size
/ PAGE_SIZE
);
147 if (in_buf_pages
> 1) {
150 for (i
= 0; i
< in_buf_pages
; i
++) {
155 in_page
= find_get_page(mapping
,
156 start
>> PAGE_SHIFT
);
157 data_in
= kmap(in_page
);
158 memcpy(workspace
->buf
+ i
* PAGE_SIZE
,
162 workspace
->strm
.next_in
= workspace
->buf
;
168 in_page
= find_get_page(mapping
,
169 start
>> PAGE_SHIFT
);
170 data_in
= kmap(in_page
);
172 workspace
->strm
.next_in
= data_in
;
174 workspace
->strm
.avail_in
= min(bytes_left
,
175 (unsigned long) workspace
->buf_size
);
178 ret
= zlib_deflate(&workspace
->strm
, Z_SYNC_FLUSH
);
180 pr_debug("BTRFS: deflate in loop returned %d\n",
182 zlib_deflateEnd(&workspace
->strm
);
187 /* we're making it bigger, give up */
188 if (workspace
->strm
.total_in
> 8192 &&
189 workspace
->strm
.total_in
<
190 workspace
->strm
.total_out
) {
194 /* we need another page for writing out. Test this
195 * before the total_in so we will pull in a new page for
196 * the stream end if required
198 if (workspace
->strm
.avail_out
== 0) {
200 if (nr_pages
== nr_dest_pages
) {
205 out_page
= alloc_page(GFP_NOFS
);
206 if (out_page
== NULL
) {
210 cpage_out
= kmap(out_page
);
211 pages
[nr_pages
] = out_page
;
213 workspace
->strm
.avail_out
= PAGE_SIZE
;
214 workspace
->strm
.next_out
= cpage_out
;
217 if (workspace
->strm
.total_in
>= len
)
219 if (workspace
->strm
.total_out
> max_out
)
222 workspace
->strm
.avail_in
= 0;
224 * Call deflate with Z_FINISH flush parameter providing more output
225 * space but no more input data, until it returns with Z_STREAM_END.
227 while (ret
!= Z_STREAM_END
) {
228 ret
= zlib_deflate(&workspace
->strm
, Z_FINISH
);
229 if (ret
== Z_STREAM_END
)
231 if (ret
!= Z_OK
&& ret
!= Z_BUF_ERROR
) {
232 zlib_deflateEnd(&workspace
->strm
);
235 } else if (workspace
->strm
.avail_out
== 0) {
236 /* get another page for the stream end */
238 if (nr_pages
== nr_dest_pages
) {
243 out_page
= alloc_page(GFP_NOFS
);
244 if (out_page
== NULL
) {
248 cpage_out
= kmap(out_page
);
249 pages
[nr_pages
] = out_page
;
251 workspace
->strm
.avail_out
= PAGE_SIZE
;
252 workspace
->strm
.next_out
= cpage_out
;
255 zlib_deflateEnd(&workspace
->strm
);
257 if (workspace
->strm
.total_out
>= workspace
->strm
.total_in
) {
263 *total_out
= workspace
->strm
.total_out
;
264 *total_in
= workspace
->strm
.total_in
;
266 *out_pages
= nr_pages
;
277 int zlib_decompress_bio(struct list_head
*ws
, struct compressed_bio
*cb
)
279 struct workspace
*workspace
= list_entry(ws
, struct workspace
, list
);
281 int wbits
= MAX_WBITS
;
283 size_t total_out
= 0;
284 unsigned long page_in_index
= 0;
285 size_t srclen
= cb
->compressed_len
;
286 unsigned long total_pages_in
= DIV_ROUND_UP(srclen
, PAGE_SIZE
);
287 unsigned long buf_start
;
288 struct page
**pages_in
= cb
->compressed_pages
;
290 data_in
= kmap(pages_in
[page_in_index
]);
291 workspace
->strm
.next_in
= data_in
;
292 workspace
->strm
.avail_in
= min_t(size_t, srclen
, PAGE_SIZE
);
293 workspace
->strm
.total_in
= 0;
295 workspace
->strm
.total_out
= 0;
296 workspace
->strm
.next_out
= workspace
->buf
;
297 workspace
->strm
.avail_out
= workspace
->buf_size
;
299 /* If it's deflate, and it's got no preset dictionary, then
300 we can tell zlib to skip the adler32 check. */
301 if (srclen
> 2 && !(data_in
[1] & PRESET_DICT
) &&
302 ((data_in
[0] & 0x0f) == Z_DEFLATED
) &&
303 !(((data_in
[0]<<8) + data_in
[1]) % 31)) {
305 wbits
= -((data_in
[0] >> 4) + 8);
306 workspace
->strm
.next_in
+= 2;
307 workspace
->strm
.avail_in
-= 2;
310 if (Z_OK
!= zlib_inflateInit2(&workspace
->strm
, wbits
)) {
311 pr_warn("BTRFS: inflateInit failed\n");
312 kunmap(pages_in
[page_in_index
]);
315 while (workspace
->strm
.total_in
< srclen
) {
316 ret
= zlib_inflate(&workspace
->strm
, Z_NO_FLUSH
);
317 if (ret
!= Z_OK
&& ret
!= Z_STREAM_END
)
320 buf_start
= total_out
;
321 total_out
= workspace
->strm
.total_out
;
323 /* we didn't make progress in this inflate call, we're done */
324 if (buf_start
== total_out
)
327 ret2
= btrfs_decompress_buf2page(workspace
->buf
,
328 total_out
- buf_start
, cb
, buf_start
);
334 workspace
->strm
.next_out
= workspace
->buf
;
335 workspace
->strm
.avail_out
= workspace
->buf_size
;
337 if (workspace
->strm
.avail_in
== 0) {
339 kunmap(pages_in
[page_in_index
]);
341 if (page_in_index
>= total_pages_in
) {
345 data_in
= kmap(pages_in
[page_in_index
]);
346 workspace
->strm
.next_in
= data_in
;
347 tmp
= srclen
- workspace
->strm
.total_in
;
348 workspace
->strm
.avail_in
= min(tmp
, PAGE_SIZE
);
351 if (ret
!= Z_STREAM_END
)
356 zlib_inflateEnd(&workspace
->strm
);
358 kunmap(pages_in
[page_in_index
]);
360 zero_fill_bio(cb
->orig_bio
);
364 int zlib_decompress(struct list_head
*ws
, unsigned char *data_in
,
365 struct page
*dest_page
, unsigned long start_byte
, size_t srclen
,
368 struct workspace
*workspace
= list_entry(ws
, struct workspace
, list
);
370 int wbits
= MAX_WBITS
;
371 unsigned long bytes_left
;
372 unsigned long total_out
= 0;
373 unsigned long pg_offset
= 0;
375 destlen
= min_t(unsigned long, destlen
, PAGE_SIZE
);
376 bytes_left
= destlen
;
378 workspace
->strm
.next_in
= data_in
;
379 workspace
->strm
.avail_in
= srclen
;
380 workspace
->strm
.total_in
= 0;
382 workspace
->strm
.next_out
= workspace
->buf
;
383 workspace
->strm
.avail_out
= workspace
->buf_size
;
384 workspace
->strm
.total_out
= 0;
385 /* If it's deflate, and it's got no preset dictionary, then
386 we can tell zlib to skip the adler32 check. */
387 if (srclen
> 2 && !(data_in
[1] & PRESET_DICT
) &&
388 ((data_in
[0] & 0x0f) == Z_DEFLATED
) &&
389 !(((data_in
[0]<<8) + data_in
[1]) % 31)) {
391 wbits
= -((data_in
[0] >> 4) + 8);
392 workspace
->strm
.next_in
+= 2;
393 workspace
->strm
.avail_in
-= 2;
396 if (Z_OK
!= zlib_inflateInit2(&workspace
->strm
, wbits
)) {
397 pr_warn("BTRFS: inflateInit failed\n");
401 while (bytes_left
> 0) {
402 unsigned long buf_start
;
403 unsigned long buf_offset
;
406 ret
= zlib_inflate(&workspace
->strm
, Z_NO_FLUSH
);
407 if (ret
!= Z_OK
&& ret
!= Z_STREAM_END
)
410 buf_start
= total_out
;
411 total_out
= workspace
->strm
.total_out
;
413 if (total_out
== buf_start
) {
418 if (total_out
<= start_byte
)
421 if (total_out
> start_byte
&& buf_start
< start_byte
)
422 buf_offset
= start_byte
- buf_start
;
426 bytes
= min(PAGE_SIZE
- pg_offset
,
427 PAGE_SIZE
- (buf_offset
% PAGE_SIZE
));
428 bytes
= min(bytes
, bytes_left
);
430 memcpy_to_page(dest_page
, pg_offset
,
431 workspace
->buf
+ buf_offset
, bytes
);
436 workspace
->strm
.next_out
= workspace
->buf
;
437 workspace
->strm
.avail_out
= workspace
->buf_size
;
440 if (ret
!= Z_STREAM_END
&& bytes_left
!= 0)
445 zlib_inflateEnd(&workspace
->strm
);
448 * this should only happen if zlib returned fewer bytes than we
449 * expected. btrfs_get_block is responsible for zeroing from the
450 * end of the inline extent (destlen) to the end of the page
452 if (pg_offset
< destlen
) {
453 memzero_page(dest_page
, pg_offset
, destlen
- pg_offset
);
458 const struct btrfs_compress_op btrfs_zlib_compress
= {
459 .workspace_manager
= &wsm
,
461 .default_level
= BTRFS_ZLIB_DEFAULT_LEVEL
,