]>
git.proxmox.com Git - mirror_ubuntu-eoan-kernel.git/blob - fs/btrfs/zlib.c
2 * Copyright (C) 2008 Oracle. All rights reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
18 * Based on jffs2 zlib code:
19 * Copyright © 2001-2007 Red Hat, Inc.
20 * Created by David Woodhouse <dwmw2@infradead.org>
23 #include <linux/kernel.h>
24 #include <linux/slab.h>
25 #include <linux/zlib.h>
26 #include <linux/zutil.h>
27 #include <linux/vmalloc.h>
28 #include <linux/init.h>
29 #include <linux/err.h>
30 #include <linux/sched.h>
31 #include <linux/pagemap.h>
32 #include <linux/bio.h>
33 #include "compression.h"
35 /* Plan: call deflate() with avail_in == *sourcelen,
36 avail_out = *dstlen - 12 and flush == Z_FINISH.
37 If it doesn't manage to finish, call it again with
38 avail_in == 0 and avail_out set to the remaining 12
39 bytes for it to clean up.
40 Q: Is 12 bytes sufficient?
42 #define STREAM_END_SPACE 12
48 struct list_head list
;
51 static LIST_HEAD(idle_workspace
);
52 static DEFINE_SPINLOCK(workspace_lock
);
53 static unsigned long num_workspace
;
54 static atomic_t alloc_workspace
= ATOMIC_INIT(0);
55 static DECLARE_WAIT_QUEUE_HEAD(workspace_wait
);
58 * this finds an available zlib workspace or allocates a new one
59 * NULL or an ERR_PTR is returned if things go bad.
61 static struct workspace
*find_zlib_workspace(void)
63 struct workspace
*workspace
;
65 int cpus
= num_online_cpus();
68 spin_lock(&workspace_lock
);
69 if (!list_empty(&idle_workspace
)) {
70 workspace
= list_entry(idle_workspace
.next
, struct workspace
,
72 list_del(&workspace
->list
);
74 spin_unlock(&workspace_lock
);
78 spin_unlock(&workspace_lock
);
79 if (atomic_read(&alloc_workspace
) > cpus
) {
81 prepare_to_wait(&workspace_wait
, &wait
, TASK_UNINTERRUPTIBLE
);
82 if (atomic_read(&alloc_workspace
) > cpus
)
84 finish_wait(&workspace_wait
, &wait
);
87 atomic_inc(&alloc_workspace
);
88 workspace
= kzalloc(sizeof(*workspace
), GFP_NOFS
);
94 workspace
->def_strm
.workspace
= vmalloc(zlib_deflate_workspacesize());
95 if (!workspace
->def_strm
.workspace
) {
99 workspace
->inf_strm
.workspace
= vmalloc(zlib_inflate_workspacesize());
100 if (!workspace
->inf_strm
.workspace
) {
104 workspace
->buf
= kmalloc(PAGE_CACHE_SIZE
, GFP_NOFS
);
105 if (!workspace
->buf
) {
112 vfree(workspace
->inf_strm
.workspace
);
114 vfree(workspace
->def_strm
.workspace
);
117 atomic_dec(&alloc_workspace
);
118 wake_up(&workspace_wait
);
123 * put a workspace struct back on the list or free it if we have enough
124 * idle ones sitting around
126 static int free_workspace(struct workspace
*workspace
)
128 spin_lock(&workspace_lock
);
129 if (num_workspace
< num_online_cpus()) {
130 list_add_tail(&workspace
->list
, &idle_workspace
);
132 spin_unlock(&workspace_lock
);
133 if (waitqueue_active(&workspace_wait
))
134 wake_up(&workspace_wait
);
137 spin_unlock(&workspace_lock
);
138 vfree(workspace
->def_strm
.workspace
);
139 vfree(workspace
->inf_strm
.workspace
);
140 kfree(workspace
->buf
);
143 atomic_dec(&alloc_workspace
);
144 if (waitqueue_active(&workspace_wait
))
145 wake_up(&workspace_wait
);
150 * cleanup function for module exit
152 static void free_workspaces(void)
154 struct workspace
*workspace
;
155 while (!list_empty(&idle_workspace
)) {
156 workspace
= list_entry(idle_workspace
.next
, struct workspace
,
158 list_del(&workspace
->list
);
159 vfree(workspace
->def_strm
.workspace
);
160 vfree(workspace
->inf_strm
.workspace
);
161 kfree(workspace
->buf
);
163 atomic_dec(&alloc_workspace
);
168 * given an address space and start/len, compress the bytes.
170 * pages are allocated to hold the compressed result and stored
173 * out_pages is used to return the number of pages allocated. There
174 * may be pages allocated even if we return an error
176 * total_in is used to return the number of bytes actually read. It
177 * may be smaller then len if we had to exit early because we
178 * ran out of room in the pages array or because we cross the
181 * total_out is used to return the total number of compressed bytes
183 * max_out tells us the max number of bytes that we're allowed to
186 int btrfs_zlib_compress_pages(struct address_space
*mapping
,
187 u64 start
, unsigned long len
,
189 unsigned long nr_dest_pages
,
190 unsigned long *out_pages
,
191 unsigned long *total_in
,
192 unsigned long *total_out
,
193 unsigned long max_out
)
196 struct workspace
*workspace
;
200 struct page
*in_page
= NULL
;
201 struct page
*out_page
= NULL
;
204 unsigned long bytes_left
;
210 workspace
= find_zlib_workspace();
214 if (Z_OK
!= zlib_deflateInit(&workspace
->def_strm
, 3)) {
215 printk(KERN_WARNING
"deflateInit failed\n");
220 workspace
->def_strm
.total_in
= 0;
221 workspace
->def_strm
.total_out
= 0;
223 in_page
= find_get_page(mapping
, start
>> PAGE_CACHE_SHIFT
);
224 data_in
= kmap(in_page
);
226 out_page
= alloc_page(GFP_NOFS
| __GFP_HIGHMEM
);
227 cpage_out
= kmap(out_page
);
231 workspace
->def_strm
.next_in
= data_in
;
232 workspace
->def_strm
.next_out
= cpage_out
;
233 workspace
->def_strm
.avail_out
= PAGE_CACHE_SIZE
;
234 workspace
->def_strm
.avail_in
= min(len
, PAGE_CACHE_SIZE
);
239 while (workspace
->def_strm
.total_in
< len
) {
240 ret
= zlib_deflate(&workspace
->def_strm
, Z_SYNC_FLUSH
);
242 printk(KERN_DEBUG
"btrfs deflate in loop returned %d\n",
244 zlib_deflateEnd(&workspace
->def_strm
);
249 /* we're making it bigger, give up */
250 if (workspace
->def_strm
.total_in
> 8192 &&
251 workspace
->def_strm
.total_in
<
252 workspace
->def_strm
.total_out
) {
256 /* we need another page for writing out. Test this
257 * before the total_in so we will pull in a new page for
258 * the stream end if required
260 if (workspace
->def_strm
.avail_out
== 0) {
262 if (nr_pages
== nr_dest_pages
) {
267 out_page
= alloc_page(GFP_NOFS
| __GFP_HIGHMEM
);
268 cpage_out
= kmap(out_page
);
269 pages
[nr_pages
] = out_page
;
271 workspace
->def_strm
.avail_out
= PAGE_CACHE_SIZE
;
272 workspace
->def_strm
.next_out
= cpage_out
;
275 if (workspace
->def_strm
.total_in
>= len
)
278 /* we've read in a full page, get a new one */
279 if (workspace
->def_strm
.avail_in
== 0) {
280 if (workspace
->def_strm
.total_out
> max_out
)
283 bytes_left
= len
- workspace
->def_strm
.total_in
;
285 page_cache_release(in_page
);
287 start
+= PAGE_CACHE_SIZE
;
288 in_page
= find_get_page(mapping
,
289 start
>> PAGE_CACHE_SHIFT
);
290 data_in
= kmap(in_page
);
291 workspace
->def_strm
.avail_in
= min(bytes_left
,
293 workspace
->def_strm
.next_in
= data_in
;
296 workspace
->def_strm
.avail_in
= 0;
297 ret
= zlib_deflate(&workspace
->def_strm
, Z_FINISH
);
298 zlib_deflateEnd(&workspace
->def_strm
);
300 if (ret
!= Z_STREAM_END
) {
305 if (workspace
->def_strm
.total_out
>= workspace
->def_strm
.total_in
) {
311 *total_out
= workspace
->def_strm
.total_out
;
312 *total_in
= workspace
->def_strm
.total_in
;
314 *out_pages
= nr_pages
;
320 page_cache_release(in_page
);
322 free_workspace(workspace
);
327 * pages_in is an array of pages with compressed data.
329 * disk_start is the starting logical offset of this array in the file
331 * bvec is a bio_vec of pages from the file that we want to decompress into
333 * vcnt is the count of pages in the biovec
335 * srclen is the number of bytes in pages_in
337 * The basic idea is that we have a bio that was created by readpages.
338 * The pages in the bio are for the uncompressed data, and they may not
339 * be contiguous. They all correspond to the range of bytes covered by
340 * the compressed extent.
342 int btrfs_zlib_decompress_biovec(struct page
**pages_in
,
344 struct bio_vec
*bvec
,
349 int wbits
= MAX_WBITS
;
350 struct workspace
*workspace
;
352 size_t total_out
= 0;
353 unsigned long page_bytes_left
;
354 unsigned long page_in_index
= 0;
355 unsigned long page_out_index
= 0;
356 struct page
*page_out
;
357 unsigned long total_pages_in
= (srclen
+ PAGE_CACHE_SIZE
- 1) /
359 unsigned long buf_start
;
360 unsigned long buf_offset
;
362 unsigned long working_bytes
;
363 unsigned long pg_offset
;
364 unsigned long start_byte
;
365 unsigned long current_buf_start
;
368 workspace
= find_zlib_workspace();
372 data_in
= kmap(pages_in
[page_in_index
]);
373 workspace
->inf_strm
.next_in
= data_in
;
374 workspace
->inf_strm
.avail_in
= min_t(size_t, srclen
, PAGE_CACHE_SIZE
);
375 workspace
->inf_strm
.total_in
= 0;
377 workspace
->inf_strm
.total_out
= 0;
378 workspace
->inf_strm
.next_out
= workspace
->buf
;
379 workspace
->inf_strm
.avail_out
= PAGE_CACHE_SIZE
;
380 page_out
= bvec
[page_out_index
].bv_page
;
381 page_bytes_left
= PAGE_CACHE_SIZE
;
384 /* If it's deflate, and it's got no preset dictionary, then
385 we can tell zlib to skip the adler32 check. */
386 if (srclen
> 2 && !(data_in
[1] & PRESET_DICT
) &&
387 ((data_in
[0] & 0x0f) == Z_DEFLATED
) &&
388 !(((data_in
[0]<<8) + data_in
[1]) % 31)) {
390 wbits
= -((data_in
[0] >> 4) + 8);
391 workspace
->inf_strm
.next_in
+= 2;
392 workspace
->inf_strm
.avail_in
-= 2;
395 if (Z_OK
!= zlib_inflateInit2(&workspace
->inf_strm
, wbits
)) {
396 printk(KERN_WARNING
"inflateInit failed\n");
400 while (workspace
->inf_strm
.total_in
< srclen
) {
401 ret
= zlib_inflate(&workspace
->inf_strm
, Z_NO_FLUSH
);
402 if (ret
!= Z_OK
&& ret
!= Z_STREAM_END
)
405 * buf start is the byte offset we're of the start of
406 * our workspace buffer
408 buf_start
= total_out
;
410 /* total_out is the last byte of the workspace buffer */
411 total_out
= workspace
->inf_strm
.total_out
;
413 working_bytes
= total_out
- buf_start
;
416 * start byte is the first byte of the page we're currently
417 * copying into relative to the start of the compressed data.
419 start_byte
= page_offset(page_out
) - disk_start
;
421 if (working_bytes
== 0) {
422 /* we didn't make progress in this inflate
425 if (ret
!= Z_STREAM_END
)
430 /* we haven't yet hit data corresponding to this page */
431 if (total_out
<= start_byte
)
435 * the start of the data we care about is offset into
436 * the middle of our working buffer
438 if (total_out
> start_byte
&& buf_start
< start_byte
) {
439 buf_offset
= start_byte
- buf_start
;
440 working_bytes
-= buf_offset
;
444 current_buf_start
= buf_start
;
446 /* copy bytes from the working buffer into the pages */
447 while (working_bytes
> 0) {
448 bytes
= min(PAGE_CACHE_SIZE
- pg_offset
,
449 PAGE_CACHE_SIZE
- buf_offset
);
450 bytes
= min(bytes
, working_bytes
);
451 kaddr
= kmap_atomic(page_out
, KM_USER0
);
452 memcpy(kaddr
+ pg_offset
, workspace
->buf
+ buf_offset
,
454 kunmap_atomic(kaddr
, KM_USER0
);
455 flush_dcache_page(page_out
);
458 page_bytes_left
-= bytes
;
460 working_bytes
-= bytes
;
461 current_buf_start
+= bytes
;
463 /* check if we need to pick another page */
464 if (page_bytes_left
== 0) {
466 if (page_out_index
>= vcnt
) {
471 page_out
= bvec
[page_out_index
].bv_page
;
473 page_bytes_left
= PAGE_CACHE_SIZE
;
474 start_byte
= page_offset(page_out
) - disk_start
;
477 * make sure our new page is covered by this
480 if (total_out
<= start_byte
)
483 /* the next page in the biovec might not
484 * be adjacent to the last page, but it
485 * might still be found inside this working
486 * buffer. bump our offset pointer
488 if (total_out
> start_byte
&&
489 current_buf_start
< start_byte
) {
490 buf_offset
= start_byte
- buf_start
;
491 working_bytes
= total_out
- start_byte
;
492 current_buf_start
= buf_start
+
498 workspace
->inf_strm
.next_out
= workspace
->buf
;
499 workspace
->inf_strm
.avail_out
= PAGE_CACHE_SIZE
;
501 if (workspace
->inf_strm
.avail_in
== 0) {
503 kunmap(pages_in
[page_in_index
]);
505 if (page_in_index
>= total_pages_in
) {
509 data_in
= kmap(pages_in
[page_in_index
]);
510 workspace
->inf_strm
.next_in
= data_in
;
511 tmp
= srclen
- workspace
->inf_strm
.total_in
;
512 workspace
->inf_strm
.avail_in
= min(tmp
,
516 if (ret
!= Z_STREAM_END
)
521 zlib_inflateEnd(&workspace
->inf_strm
);
523 kunmap(pages_in
[page_in_index
]);
525 free_workspace(workspace
);
530 * a less complex decompression routine. Our compressed data fits in a
531 * single page, and we want to read a single page out of it.
532 * start_byte tells us the offset into the compressed data we're interested in
534 int btrfs_zlib_decompress(unsigned char *data_in
,
535 struct page
*dest_page
,
536 unsigned long start_byte
,
537 size_t srclen
, size_t destlen
)
540 int wbits
= MAX_WBITS
;
541 struct workspace
*workspace
;
542 unsigned long bytes_left
= destlen
;
543 unsigned long total_out
= 0;
546 if (destlen
> PAGE_CACHE_SIZE
)
549 workspace
= find_zlib_workspace();
553 workspace
->inf_strm
.next_in
= data_in
;
554 workspace
->inf_strm
.avail_in
= srclen
;
555 workspace
->inf_strm
.total_in
= 0;
557 workspace
->inf_strm
.next_out
= workspace
->buf
;
558 workspace
->inf_strm
.avail_out
= PAGE_CACHE_SIZE
;
559 workspace
->inf_strm
.total_out
= 0;
560 /* If it's deflate, and it's got no preset dictionary, then
561 we can tell zlib to skip the adler32 check. */
562 if (srclen
> 2 && !(data_in
[1] & PRESET_DICT
) &&
563 ((data_in
[0] & 0x0f) == Z_DEFLATED
) &&
564 !(((data_in
[0]<<8) + data_in
[1]) % 31)) {
566 wbits
= -((data_in
[0] >> 4) + 8);
567 workspace
->inf_strm
.next_in
+= 2;
568 workspace
->inf_strm
.avail_in
-= 2;
571 if (Z_OK
!= zlib_inflateInit2(&workspace
->inf_strm
, wbits
)) {
572 printk(KERN_WARNING
"inflateInit failed\n");
577 while (bytes_left
> 0) {
578 unsigned long buf_start
;
579 unsigned long buf_offset
;
581 unsigned long pg_offset
= 0;
583 ret
= zlib_inflate(&workspace
->inf_strm
, Z_NO_FLUSH
);
584 if (ret
!= Z_OK
&& ret
!= Z_STREAM_END
)
587 buf_start
= total_out
;
588 total_out
= workspace
->inf_strm
.total_out
;
590 if (total_out
== buf_start
) {
595 if (total_out
<= start_byte
)
598 if (total_out
> start_byte
&& buf_start
< start_byte
)
599 buf_offset
= start_byte
- buf_start
;
603 bytes
= min(PAGE_CACHE_SIZE
- pg_offset
,
604 PAGE_CACHE_SIZE
- buf_offset
);
605 bytes
= min(bytes
, bytes_left
);
607 kaddr
= kmap_atomic(dest_page
, KM_USER0
);
608 memcpy(kaddr
+ pg_offset
, workspace
->buf
+ buf_offset
, bytes
);
609 kunmap_atomic(kaddr
, KM_USER0
);
614 workspace
->inf_strm
.next_out
= workspace
->buf
;
615 workspace
->inf_strm
.avail_out
= PAGE_CACHE_SIZE
;
618 if (ret
!= Z_STREAM_END
&& bytes_left
!= 0)
623 zlib_inflateEnd(&workspace
->inf_strm
);
625 free_workspace(workspace
);
629 void btrfs_zlib_exit(void)