1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2008 Oracle. All rights reserved.
5 * Based on jffs2 zlib code:
6 * Copyright © 2001-2007 Red Hat, Inc.
7 * Created by David Woodhouse <dwmw2@infradead.org>
10 #include <linux/kernel.h>
11 #include <linux/slab.h>
12 #include <linux/zlib.h>
13 #include <linux/zutil.h>
15 #include <linux/init.h>
16 #include <linux/err.h>
17 #include <linux/sched.h>
18 #include <linux/pagemap.h>
19 #include <linux/bio.h>
20 #include <linux/refcount.h>
21 #include "compression.h"
26 struct list_head list
;
30 static struct workspace_manager wsm
;
32 static void zlib_init_workspace_manager(void)
34 btrfs_init_workspace_manager(&wsm
, &btrfs_zlib_compress
);
37 static void zlib_cleanup_workspace_manager(void)
39 btrfs_cleanup_workspace_manager(&wsm
);
42 static struct list_head
*zlib_get_workspace(unsigned int level
)
44 struct list_head
*ws
= btrfs_get_workspace(&wsm
, level
);
45 struct workspace
*workspace
= list_entry(ws
, struct workspace
, list
);
47 workspace
->level
= level
;
52 static void zlib_put_workspace(struct list_head
*ws
)
54 btrfs_put_workspace(&wsm
, ws
);
57 static void zlib_free_workspace(struct list_head
*ws
)
59 struct workspace
*workspace
= list_entry(ws
, struct workspace
, list
);
61 kvfree(workspace
->strm
.workspace
);
62 kfree(workspace
->buf
);
66 static struct list_head
*zlib_alloc_workspace(unsigned int level
)
68 struct workspace
*workspace
;
71 workspace
= kzalloc(sizeof(*workspace
), GFP_KERNEL
);
73 return ERR_PTR(-ENOMEM
);
75 workspacesize
= max(zlib_deflate_workspacesize(MAX_WBITS
, MAX_MEM_LEVEL
),
76 zlib_inflate_workspacesize());
77 workspace
->strm
.workspace
= kvmalloc(workspacesize
, GFP_KERNEL
);
78 workspace
->level
= level
;
79 workspace
->buf
= kmalloc(PAGE_SIZE
, GFP_KERNEL
);
80 if (!workspace
->strm
.workspace
|| !workspace
->buf
)
83 INIT_LIST_HEAD(&workspace
->list
);
85 return &workspace
->list
;
87 zlib_free_workspace(&workspace
->list
);
88 return ERR_PTR(-ENOMEM
);
91 static int zlib_compress_pages(struct list_head
*ws
,
92 struct address_space
*mapping
,
95 unsigned long *out_pages
,
96 unsigned long *total_in
,
97 unsigned long *total_out
)
99 struct workspace
*workspace
= list_entry(ws
, struct workspace
, list
);
104 struct page
*in_page
= NULL
;
105 struct page
*out_page
= NULL
;
106 unsigned long bytes_left
;
107 unsigned long len
= *total_out
;
108 unsigned long nr_dest_pages
= *out_pages
;
109 const unsigned long max_out
= nr_dest_pages
* PAGE_SIZE
;
115 if (Z_OK
!= zlib_deflateInit(&workspace
->strm
, workspace
->level
)) {
116 pr_warn("BTRFS: deflateInit failed\n");
121 workspace
->strm
.total_in
= 0;
122 workspace
->strm
.total_out
= 0;
124 in_page
= find_get_page(mapping
, start
>> PAGE_SHIFT
);
125 data_in
= kmap(in_page
);
127 out_page
= alloc_page(GFP_NOFS
| __GFP_HIGHMEM
);
128 if (out_page
== NULL
) {
132 cpage_out
= kmap(out_page
);
136 workspace
->strm
.next_in
= data_in
;
137 workspace
->strm
.next_out
= cpage_out
;
138 workspace
->strm
.avail_out
= PAGE_SIZE
;
139 workspace
->strm
.avail_in
= min(len
, PAGE_SIZE
);
141 while (workspace
->strm
.total_in
< len
) {
142 ret
= zlib_deflate(&workspace
->strm
, Z_SYNC_FLUSH
);
144 pr_debug("BTRFS: deflate in loop returned %d\n",
146 zlib_deflateEnd(&workspace
->strm
);
151 /* we're making it bigger, give up */
152 if (workspace
->strm
.total_in
> 8192 &&
153 workspace
->strm
.total_in
<
154 workspace
->strm
.total_out
) {
158 /* we need another page for writing out. Test this
159 * before the total_in so we will pull in a new page for
160 * the stream end if required
162 if (workspace
->strm
.avail_out
== 0) {
164 if (nr_pages
== nr_dest_pages
) {
169 out_page
= alloc_page(GFP_NOFS
| __GFP_HIGHMEM
);
170 if (out_page
== NULL
) {
174 cpage_out
= kmap(out_page
);
175 pages
[nr_pages
] = out_page
;
177 workspace
->strm
.avail_out
= PAGE_SIZE
;
178 workspace
->strm
.next_out
= cpage_out
;
181 if (workspace
->strm
.total_in
>= len
)
184 /* we've read in a full page, get a new one */
185 if (workspace
->strm
.avail_in
== 0) {
186 if (workspace
->strm
.total_out
> max_out
)
189 bytes_left
= len
- workspace
->strm
.total_in
;
194 in_page
= find_get_page(mapping
,
195 start
>> PAGE_SHIFT
);
196 data_in
= kmap(in_page
);
197 workspace
->strm
.avail_in
= min(bytes_left
,
199 workspace
->strm
.next_in
= data_in
;
202 workspace
->strm
.avail_in
= 0;
203 ret
= zlib_deflate(&workspace
->strm
, Z_FINISH
);
204 zlib_deflateEnd(&workspace
->strm
);
206 if (ret
!= Z_STREAM_END
) {
211 if (workspace
->strm
.total_out
>= workspace
->strm
.total_in
) {
217 *total_out
= workspace
->strm
.total_out
;
218 *total_in
= workspace
->strm
.total_in
;
220 *out_pages
= nr_pages
;
231 static int zlib_decompress_bio(struct list_head
*ws
, struct compressed_bio
*cb
)
233 struct workspace
*workspace
= list_entry(ws
, struct workspace
, list
);
235 int wbits
= MAX_WBITS
;
237 size_t total_out
= 0;
238 unsigned long page_in_index
= 0;
239 size_t srclen
= cb
->compressed_len
;
240 unsigned long total_pages_in
= DIV_ROUND_UP(srclen
, PAGE_SIZE
);
241 unsigned long buf_start
;
242 struct page
**pages_in
= cb
->compressed_pages
;
243 u64 disk_start
= cb
->start
;
244 struct bio
*orig_bio
= cb
->orig_bio
;
246 data_in
= kmap(pages_in
[page_in_index
]);
247 workspace
->strm
.next_in
= data_in
;
248 workspace
->strm
.avail_in
= min_t(size_t, srclen
, PAGE_SIZE
);
249 workspace
->strm
.total_in
= 0;
251 workspace
->strm
.total_out
= 0;
252 workspace
->strm
.next_out
= workspace
->buf
;
253 workspace
->strm
.avail_out
= PAGE_SIZE
;
255 /* If it's deflate, and it's got no preset dictionary, then
256 we can tell zlib to skip the adler32 check. */
257 if (srclen
> 2 && !(data_in
[1] & PRESET_DICT
) &&
258 ((data_in
[0] & 0x0f) == Z_DEFLATED
) &&
259 !(((data_in
[0]<<8) + data_in
[1]) % 31)) {
261 wbits
= -((data_in
[0] >> 4) + 8);
262 workspace
->strm
.next_in
+= 2;
263 workspace
->strm
.avail_in
-= 2;
266 if (Z_OK
!= zlib_inflateInit2(&workspace
->strm
, wbits
)) {
267 pr_warn("BTRFS: inflateInit failed\n");
268 kunmap(pages_in
[page_in_index
]);
271 while (workspace
->strm
.total_in
< srclen
) {
272 ret
= zlib_inflate(&workspace
->strm
, Z_NO_FLUSH
);
273 if (ret
!= Z_OK
&& ret
!= Z_STREAM_END
)
276 buf_start
= total_out
;
277 total_out
= workspace
->strm
.total_out
;
279 /* we didn't make progress in this inflate call, we're done */
280 if (buf_start
== total_out
)
283 ret2
= btrfs_decompress_buf2page(workspace
->buf
, buf_start
,
284 total_out
, disk_start
,
291 workspace
->strm
.next_out
= workspace
->buf
;
292 workspace
->strm
.avail_out
= PAGE_SIZE
;
294 if (workspace
->strm
.avail_in
== 0) {
296 kunmap(pages_in
[page_in_index
]);
298 if (page_in_index
>= total_pages_in
) {
302 data_in
= kmap(pages_in
[page_in_index
]);
303 workspace
->strm
.next_in
= data_in
;
304 tmp
= srclen
- workspace
->strm
.total_in
;
305 workspace
->strm
.avail_in
= min(tmp
,
309 if (ret
!= Z_STREAM_END
)
314 zlib_inflateEnd(&workspace
->strm
);
316 kunmap(pages_in
[page_in_index
]);
318 zero_fill_bio(orig_bio
);
322 static int zlib_decompress(struct list_head
*ws
, unsigned char *data_in
,
323 struct page
*dest_page
,
324 unsigned long start_byte
,
325 size_t srclen
, size_t destlen
)
327 struct workspace
*workspace
= list_entry(ws
, struct workspace
, list
);
329 int wbits
= MAX_WBITS
;
330 unsigned long bytes_left
;
331 unsigned long total_out
= 0;
332 unsigned long pg_offset
= 0;
335 destlen
= min_t(unsigned long, destlen
, PAGE_SIZE
);
336 bytes_left
= destlen
;
338 workspace
->strm
.next_in
= data_in
;
339 workspace
->strm
.avail_in
= srclen
;
340 workspace
->strm
.total_in
= 0;
342 workspace
->strm
.next_out
= workspace
->buf
;
343 workspace
->strm
.avail_out
= PAGE_SIZE
;
344 workspace
->strm
.total_out
= 0;
345 /* If it's deflate, and it's got no preset dictionary, then
346 we can tell zlib to skip the adler32 check. */
347 if (srclen
> 2 && !(data_in
[1] & PRESET_DICT
) &&
348 ((data_in
[0] & 0x0f) == Z_DEFLATED
) &&
349 !(((data_in
[0]<<8) + data_in
[1]) % 31)) {
351 wbits
= -((data_in
[0] >> 4) + 8);
352 workspace
->strm
.next_in
+= 2;
353 workspace
->strm
.avail_in
-= 2;
356 if (Z_OK
!= zlib_inflateInit2(&workspace
->strm
, wbits
)) {
357 pr_warn("BTRFS: inflateInit failed\n");
361 while (bytes_left
> 0) {
362 unsigned long buf_start
;
363 unsigned long buf_offset
;
366 ret
= zlib_inflate(&workspace
->strm
, Z_NO_FLUSH
);
367 if (ret
!= Z_OK
&& ret
!= Z_STREAM_END
)
370 buf_start
= total_out
;
371 total_out
= workspace
->strm
.total_out
;
373 if (total_out
== buf_start
) {
378 if (total_out
<= start_byte
)
381 if (total_out
> start_byte
&& buf_start
< start_byte
)
382 buf_offset
= start_byte
- buf_start
;
386 bytes
= min(PAGE_SIZE
- pg_offset
,
387 PAGE_SIZE
- buf_offset
);
388 bytes
= min(bytes
, bytes_left
);
390 kaddr
= kmap_atomic(dest_page
);
391 memcpy(kaddr
+ pg_offset
, workspace
->buf
+ buf_offset
, bytes
);
392 kunmap_atomic(kaddr
);
397 workspace
->strm
.next_out
= workspace
->buf
;
398 workspace
->strm
.avail_out
= PAGE_SIZE
;
401 if (ret
!= Z_STREAM_END
&& bytes_left
!= 0)
406 zlib_inflateEnd(&workspace
->strm
);
409 * this should only happen if zlib returned fewer bytes than we
410 * expected. btrfs_get_block is responsible for zeroing from the
411 * end of the inline extent (destlen) to the end of the page
413 if (pg_offset
< destlen
) {
414 kaddr
= kmap_atomic(dest_page
);
415 memset(kaddr
+ pg_offset
, 0, destlen
- pg_offset
);
416 kunmap_atomic(kaddr
);
421 const struct btrfs_compress_op btrfs_zlib_compress
= {
422 .init_workspace_manager
= zlib_init_workspace_manager
,
423 .cleanup_workspace_manager
= zlib_cleanup_workspace_manager
,
424 .get_workspace
= zlib_get_workspace
,
425 .put_workspace
= zlib_put_workspace
,
426 .alloc_workspace
= zlib_alloc_workspace
,
427 .free_workspace
= zlib_free_workspace
,
428 .compress_pages
= zlib_compress_pages
,
429 .decompress_bio
= zlib_decompress_bio
,
430 .decompress
= zlib_decompress
,
432 .default_level
= BTRFS_ZLIB_DEFAULT_LEVEL
,