]>
Commit | Line | Data |
---|---|---|
c1d7c514 | 1 | // SPDX-License-Identifier: GPL-2.0 |
5c1aab1d NT |
2 | /* |
3 | * Copyright (c) 2016-present, Facebook, Inc. | |
4 | * All rights reserved. | |
5 | * | |
5c1aab1d | 6 | */ |
c1d7c514 | 7 | |
5c1aab1d | 8 | #include <linux/bio.h> |
3f93aef5 | 9 | #include <linux/bitmap.h> |
5c1aab1d NT |
10 | #include <linux/err.h> |
11 | #include <linux/init.h> | |
12 | #include <linux/kernel.h> | |
13 | #include <linux/mm.h> | |
3f93aef5 | 14 | #include <linux/sched/mm.h> |
5c1aab1d NT |
15 | #include <linux/pagemap.h> |
16 | #include <linux/refcount.h> | |
17 | #include <linux/sched.h> | |
18 | #include <linux/slab.h> | |
19 | #include <linux/zstd.h> | |
20 | #include "compression.h" | |
3f93aef5 | 21 | #include "ctree.h" |
5c1aab1d NT |
22 | |
23 | #define ZSTD_BTRFS_MAX_WINDOWLOG 17 | |
24 | #define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG) | |
25 | #define ZSTD_BTRFS_DEFAULT_LEVEL 3 | |
d3c6ab75 | 26 | #define ZSTD_BTRFS_MAX_LEVEL 15 |
3f93aef5 DZ |
27 | /* 307s to avoid pathologically clashing with transaction commit */ |
28 | #define ZSTD_BTRFS_RECLAIM_JIFFIES (307 * HZ) | |
5c1aab1d | 29 | |
e0dc87af DZ |
30 | static ZSTD_parameters zstd_get_btrfs_parameters(unsigned int level, |
31 | size_t src_len) | |
5c1aab1d | 32 | { |
e0dc87af | 33 | ZSTD_parameters params = ZSTD_getParams(level, src_len, 0); |
5c1aab1d NT |
34 | |
35 | if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG) | |
36 | params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG; | |
37 | WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT); | |
38 | return params; | |
39 | } | |
40 | ||
41 | struct workspace { | |
42 | void *mem; | |
43 | size_t size; | |
44 | char *buf; | |
3f93aef5 | 45 | unsigned int level; |
e0dc87af | 46 | unsigned int req_level; |
3f93aef5 | 47 | unsigned long last_used; /* jiffies */ |
5c1aab1d | 48 | struct list_head list; |
3f93aef5 | 49 | struct list_head lru_list; |
431e9822 DS |
50 | ZSTD_inBuffer in_buf; |
51 | ZSTD_outBuffer out_buf; | |
5c1aab1d NT |
52 | }; |
53 | ||
3f93aef5 DZ |
54 | /* |
55 | * Zstd Workspace Management | |
56 | * | |
57 | * Zstd workspaces have different memory requirements depending on the level. | |
58 | * The zstd workspaces are managed by having individual lists for each level | |
59 | * and a global lru. Forward progress is maintained by protecting a max level | |
60 | * workspace. | |
61 | * | |
62 | * Getting a workspace is done by using the bitmap to identify the levels that | |
63 | * have available workspaces and scans up. This lets us recycle higher level | |
64 | * workspaces because of the monotonic memory guarantee. A workspace's | |
65 | * last_used is only updated if it is being used by the corresponding memory | |
66 | * level. Putting a workspace involves adding it back to the appropriate places | |
67 | * and adding it back to the lru if necessary. | |
68 | * | |
69 | * A timer is used to reclaim workspaces if they have not been used for | |
70 | * ZSTD_BTRFS_RECLAIM_JIFFIES. This helps keep only active workspaces around. | |
71 | * The upper bound is provided by the workqueue limit which is 2 (percpu limit). | |
72 | */ | |
73 | ||
74 | struct zstd_workspace_manager { | |
75 | const struct btrfs_compress_op *ops; | |
76 | spinlock_t lock; | |
77 | struct list_head lru_list; | |
78 | struct list_head idle_ws[ZSTD_BTRFS_MAX_LEVEL]; | |
79 | unsigned long active_map; | |
80 | wait_queue_head_t wait; | |
81 | struct timer_list timer; | |
82 | }; | |
83 | ||
84 | static struct zstd_workspace_manager wsm; | |
92ee5530 | 85 | |
d3c6ab75 DZ |
86 | static size_t zstd_ws_mem_sizes[ZSTD_BTRFS_MAX_LEVEL]; |
87 | ||
3f93aef5 DZ |
88 | static inline struct workspace *list_to_workspace(struct list_head *list) |
89 | { | |
90 | return container_of(list, struct workspace, list); | |
91 | } | |
92 | ||
93 | /* | |
94 | * zstd_reclaim_timer_fn - reclaim timer | |
95 | * @t: timer | |
96 | * | |
97 | * This scans the lru_list and attempts to reclaim any workspace that hasn't | |
98 | * been used for ZSTD_BTRFS_RECLAIM_JIFFIES. | |
99 | */ | |
100 | static void zstd_reclaim_timer_fn(struct timer_list *timer) | |
101 | { | |
102 | unsigned long reclaim_threshold = jiffies - ZSTD_BTRFS_RECLAIM_JIFFIES; | |
103 | struct list_head *pos, *next; | |
104 | ||
105 | spin_lock(&wsm.lock); | |
106 | ||
107 | if (list_empty(&wsm.lru_list)) { | |
108 | spin_unlock(&wsm.lock); | |
109 | return; | |
110 | } | |
111 | ||
112 | list_for_each_prev_safe(pos, next, &wsm.lru_list) { | |
113 | struct workspace *victim = container_of(pos, struct workspace, | |
114 | lru_list); | |
115 | unsigned int level; | |
116 | ||
117 | if (time_after(victim->last_used, reclaim_threshold)) | |
118 | break; | |
119 | ||
120 | /* workspace is in use */ | |
121 | if (victim->req_level) | |
122 | continue; | |
123 | ||
124 | level = victim->level; | |
125 | list_del(&victim->lru_list); | |
126 | list_del(&victim->list); | |
127 | wsm.ops->free_workspace(&victim->list); | |
128 | ||
129 | if (list_empty(&wsm.idle_ws[level - 1])) | |
130 | clear_bit(level - 1, &wsm.active_map); | |
131 | ||
132 | } | |
133 | ||
134 | if (!list_empty(&wsm.lru_list)) | |
135 | mod_timer(&wsm.timer, jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES); | |
136 | ||
137 | spin_unlock(&wsm.lock); | |
138 | } | |
139 | ||
d3c6ab75 DZ |
140 | /* |
141 | * zstd_calc_ws_mem_sizes - calculate monotonic memory bounds | |
142 | * | |
143 | * It is possible based on the level configurations that a higher level | |
144 | * workspace uses less memory than a lower level workspace. In order to reuse | |
145 | * workspaces, this must be made a monotonic relationship. This precomputes | |
146 | * the required memory for each level and enforces the monotonicity between | |
147 | * level and memory required. | |
148 | */ | |
149 | static void zstd_calc_ws_mem_sizes(void) | |
150 | { | |
151 | size_t max_size = 0; | |
152 | unsigned int level; | |
153 | ||
154 | for (level = 1; level <= ZSTD_BTRFS_MAX_LEVEL; level++) { | |
155 | ZSTD_parameters params = | |
156 | zstd_get_btrfs_parameters(level, ZSTD_BTRFS_MAX_INPUT); | |
157 | size_t level_size = | |
158 | max_t(size_t, | |
159 | ZSTD_CStreamWorkspaceBound(params.cParams), | |
160 | ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT)); | |
161 | ||
162 | max_size = max_t(size_t, max_size, level_size); | |
163 | zstd_ws_mem_sizes[level - 1] = max_size; | |
164 | } | |
165 | } | |
166 | ||
92ee5530 DZ |
167 | static void zstd_init_workspace_manager(void) |
168 | { | |
3f93aef5 DZ |
169 | struct list_head *ws; |
170 | int i; | |
171 | ||
d3c6ab75 DZ |
172 | zstd_calc_ws_mem_sizes(); |
173 | ||
3f93aef5 DZ |
174 | wsm.ops = &btrfs_zstd_compress; |
175 | spin_lock_init(&wsm.lock); | |
176 | init_waitqueue_head(&wsm.wait); | |
177 | timer_setup(&wsm.timer, zstd_reclaim_timer_fn, 0); | |
178 | ||
179 | INIT_LIST_HEAD(&wsm.lru_list); | |
180 | for (i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++) | |
181 | INIT_LIST_HEAD(&wsm.idle_ws[i]); | |
182 | ||
183 | ws = wsm.ops->alloc_workspace(ZSTD_BTRFS_MAX_LEVEL); | |
184 | if (IS_ERR(ws)) { | |
185 | pr_warn( | |
186 | "BTRFS: cannot preallocate zstd compression workspace\n"); | |
187 | } else { | |
188 | set_bit(ZSTD_BTRFS_MAX_LEVEL - 1, &wsm.active_map); | |
189 | list_add(ws, &wsm.idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1]); | |
190 | } | |
92ee5530 DZ |
191 | } |
192 | ||
193 | static void zstd_cleanup_workspace_manager(void) | |
194 | { | |
3f93aef5 DZ |
195 | struct workspace *workspace; |
196 | int i; | |
197 | ||
d3865159 | 198 | spin_lock(&wsm.lock); |
3f93aef5 DZ |
199 | for (i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++) { |
200 | while (!list_empty(&wsm.idle_ws[i])) { | |
201 | workspace = container_of(wsm.idle_ws[i].next, | |
202 | struct workspace, list); | |
203 | list_del(&workspace->list); | |
204 | list_del(&workspace->lru_list); | |
205 | wsm.ops->free_workspace(&workspace->list); | |
206 | } | |
207 | } | |
d3865159 DZ |
208 | spin_unlock(&wsm.lock); |
209 | ||
210 | del_timer_sync(&wsm.timer); | |
3f93aef5 DZ |
211 | } |
212 | ||
213 | /* | |
214 | * zstd_find_workspace - find workspace | |
215 | * @level: compression level | |
216 | * | |
217 | * This iterates over the set bits in the active_map beginning at the requested | |
218 | * compression level. This lets us utilize already allocated workspaces before | |
219 | * allocating a new one. If the workspace is of a larger size, it is used, but | |
220 | * the place in the lru_list and last_used times are not updated. This is to | |
221 | * offer the opportunity to reclaim the workspace in favor of allocating an | |
222 | * appropriately sized one in the future. | |
223 | */ | |
224 | static struct list_head *zstd_find_workspace(unsigned int level) | |
225 | { | |
226 | struct list_head *ws; | |
227 | struct workspace *workspace; | |
228 | int i = level - 1; | |
229 | ||
230 | spin_lock(&wsm.lock); | |
231 | for_each_set_bit_from(i, &wsm.active_map, ZSTD_BTRFS_MAX_LEVEL) { | |
232 | if (!list_empty(&wsm.idle_ws[i])) { | |
233 | ws = wsm.idle_ws[i].next; | |
234 | workspace = list_to_workspace(ws); | |
235 | list_del_init(ws); | |
236 | /* keep its place if it's a lower level using this */ | |
237 | workspace->req_level = level; | |
238 | if (level == workspace->level) | |
239 | list_del(&workspace->lru_list); | |
240 | if (list_empty(&wsm.idle_ws[i])) | |
241 | clear_bit(i, &wsm.active_map); | |
242 | spin_unlock(&wsm.lock); | |
243 | return ws; | |
244 | } | |
245 | } | |
246 | spin_unlock(&wsm.lock); | |
247 | ||
248 | return NULL; | |
92ee5530 DZ |
249 | } |
250 | ||
3f93aef5 DZ |
251 | /* |
252 | * zstd_get_workspace - zstd's get_workspace | |
253 | * @level: compression level | |
254 | * | |
255 | * If @level is 0, then any compression level can be used. Therefore, we begin | |
256 | * scanning from 1. We first scan through possible workspaces and then after | |
257 | * attempt to allocate a new workspace. If we fail to allocate one due to | |
258 | * memory pressure, go to sleep waiting for the max level workspace to free up. | |
259 | */ | |
7bf49943 | 260 | static struct list_head *zstd_get_workspace(unsigned int level) |
92ee5530 | 261 | { |
3f93aef5 DZ |
262 | struct list_head *ws; |
263 | unsigned int nofs_flag; | |
e0dc87af | 264 | |
3f93aef5 DZ |
265 | /* level == 0 means we can use any workspace */ |
266 | if (!level) | |
267 | level = 1; | |
268 | ||
269 | again: | |
270 | ws = zstd_find_workspace(level); | |
271 | if (ws) | |
272 | return ws; | |
273 | ||
274 | nofs_flag = memalloc_nofs_save(); | |
275 | ws = wsm.ops->alloc_workspace(level); | |
276 | memalloc_nofs_restore(nofs_flag); | |
277 | ||
278 | if (IS_ERR(ws)) { | |
279 | DEFINE_WAIT(wait); | |
280 | ||
281 | prepare_to_wait(&wsm.wait, &wait, TASK_UNINTERRUPTIBLE); | |
282 | schedule(); | |
283 | finish_wait(&wsm.wait, &wait); | |
284 | ||
285 | goto again; | |
286 | } | |
e0dc87af DZ |
287 | |
288 | return ws; | |
92ee5530 DZ |
289 | } |
290 | ||
3f93aef5 DZ |
291 | /* |
292 | * zstd_put_workspace - zstd put_workspace | |
293 | * @ws: list_head for the workspace | |
294 | * | |
295 | * When putting back a workspace, we only need to update the LRU if we are of | |
296 | * the requested compression level. Here is where we continue to protect the | |
297 | * max level workspace or update last_used accordingly. If the reclaim timer | |
298 | * isn't set, it is also set here. Only the max level workspace tries and wakes | |
299 | * up waiting workspaces. | |
300 | */ | |
92ee5530 DZ |
301 | static void zstd_put_workspace(struct list_head *ws) |
302 | { | |
3f93aef5 DZ |
303 | struct workspace *workspace = list_to_workspace(ws); |
304 | ||
305 | spin_lock(&wsm.lock); | |
306 | ||
307 | /* A node is only taken off the lru if we are the corresponding level */ | |
308 | if (workspace->req_level == workspace->level) { | |
309 | /* Hide a max level workspace from reclaim */ | |
310 | if (list_empty(&wsm.idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1])) { | |
311 | INIT_LIST_HEAD(&workspace->lru_list); | |
312 | } else { | |
313 | workspace->last_used = jiffies; | |
314 | list_add(&workspace->lru_list, &wsm.lru_list); | |
315 | if (!timer_pending(&wsm.timer)) | |
316 | mod_timer(&wsm.timer, | |
317 | jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES); | |
318 | } | |
319 | } | |
320 | ||
321 | set_bit(workspace->level - 1, &wsm.active_map); | |
322 | list_add(&workspace->list, &wsm.idle_ws[workspace->level - 1]); | |
323 | workspace->req_level = 0; | |
324 | ||
325 | spin_unlock(&wsm.lock); | |
326 | ||
327 | if (workspace->level == ZSTD_BTRFS_MAX_LEVEL) | |
328 | cond_wake_up(&wsm.wait); | |
92ee5530 DZ |
329 | } |
330 | ||
5c1aab1d NT |
331 | static void zstd_free_workspace(struct list_head *ws) |
332 | { | |
333 | struct workspace *workspace = list_entry(ws, struct workspace, list); | |
334 | ||
335 | kvfree(workspace->mem); | |
336 | kfree(workspace->buf); | |
337 | kfree(workspace); | |
338 | } | |
339 | ||
7bf49943 | 340 | static struct list_head *zstd_alloc_workspace(unsigned int level) |
5c1aab1d | 341 | { |
5c1aab1d NT |
342 | struct workspace *workspace; |
343 | ||
344 | workspace = kzalloc(sizeof(*workspace), GFP_KERNEL); | |
345 | if (!workspace) | |
346 | return ERR_PTR(-ENOMEM); | |
347 | ||
d3c6ab75 | 348 | workspace->size = zstd_ws_mem_sizes[level - 1]; |
3f93aef5 DZ |
349 | workspace->level = level; |
350 | workspace->req_level = level; | |
351 | workspace->last_used = jiffies; | |
5c1aab1d NT |
352 | workspace->mem = kvmalloc(workspace->size, GFP_KERNEL); |
353 | workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL); | |
354 | if (!workspace->mem || !workspace->buf) | |
355 | goto fail; | |
356 | ||
357 | INIT_LIST_HEAD(&workspace->list); | |
3f93aef5 | 358 | INIT_LIST_HEAD(&workspace->lru_list); |
5c1aab1d NT |
359 | |
360 | return &workspace->list; | |
361 | fail: | |
362 | zstd_free_workspace(&workspace->list); | |
363 | return ERR_PTR(-ENOMEM); | |
364 | } | |
365 | ||
366 | static int zstd_compress_pages(struct list_head *ws, | |
367 | struct address_space *mapping, | |
368 | u64 start, | |
369 | struct page **pages, | |
370 | unsigned long *out_pages, | |
371 | unsigned long *total_in, | |
372 | unsigned long *total_out) | |
373 | { | |
374 | struct workspace *workspace = list_entry(ws, struct workspace, list); | |
375 | ZSTD_CStream *stream; | |
376 | int ret = 0; | |
377 | int nr_pages = 0; | |
378 | struct page *in_page = NULL; /* The current page to read */ | |
379 | struct page *out_page = NULL; /* The current page to write to */ | |
5c1aab1d NT |
380 | unsigned long tot_in = 0; |
381 | unsigned long tot_out = 0; | |
382 | unsigned long len = *total_out; | |
383 | const unsigned long nr_dest_pages = *out_pages; | |
384 | unsigned long max_out = nr_dest_pages * PAGE_SIZE; | |
e0dc87af DZ |
385 | ZSTD_parameters params = zstd_get_btrfs_parameters(workspace->req_level, |
386 | len); | |
5c1aab1d NT |
387 | |
388 | *out_pages = 0; | |
389 | *total_out = 0; | |
390 | *total_in = 0; | |
391 | ||
392 | /* Initialize the stream */ | |
393 | stream = ZSTD_initCStream(params, len, workspace->mem, | |
394 | workspace->size); | |
395 | if (!stream) { | |
396 | pr_warn("BTRFS: ZSTD_initCStream failed\n"); | |
397 | ret = -EIO; | |
398 | goto out; | |
399 | } | |
400 | ||
401 | /* map in the first page of input data */ | |
402 | in_page = find_get_page(mapping, start >> PAGE_SHIFT); | |
431e9822 DS |
403 | workspace->in_buf.src = kmap(in_page); |
404 | workspace->in_buf.pos = 0; | |
405 | workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE); | |
5c1aab1d NT |
406 | |
407 | ||
408 | /* Allocate and map in the output buffer */ | |
409 | out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); | |
410 | if (out_page == NULL) { | |
411 | ret = -ENOMEM; | |
412 | goto out; | |
413 | } | |
414 | pages[nr_pages++] = out_page; | |
431e9822 DS |
415 | workspace->out_buf.dst = kmap(out_page); |
416 | workspace->out_buf.pos = 0; | |
417 | workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE); | |
5c1aab1d NT |
418 | |
419 | while (1) { | |
420 | size_t ret2; | |
421 | ||
431e9822 DS |
422 | ret2 = ZSTD_compressStream(stream, &workspace->out_buf, |
423 | &workspace->in_buf); | |
5c1aab1d NT |
424 | if (ZSTD_isError(ret2)) { |
425 | pr_debug("BTRFS: ZSTD_compressStream returned %d\n", | |
426 | ZSTD_getErrorCode(ret2)); | |
427 | ret = -EIO; | |
428 | goto out; | |
429 | } | |
430 | ||
431 | /* Check to see if we are making it bigger */ | |
431e9822 DS |
432 | if (tot_in + workspace->in_buf.pos > 8192 && |
433 | tot_in + workspace->in_buf.pos < | |
434 | tot_out + workspace->out_buf.pos) { | |
5c1aab1d NT |
435 | ret = -E2BIG; |
436 | goto out; | |
437 | } | |
438 | ||
439 | /* We've reached the end of our output range */ | |
431e9822 DS |
440 | if (workspace->out_buf.pos >= max_out) { |
441 | tot_out += workspace->out_buf.pos; | |
5c1aab1d NT |
442 | ret = -E2BIG; |
443 | goto out; | |
444 | } | |
445 | ||
446 | /* Check if we need more output space */ | |
431e9822 | 447 | if (workspace->out_buf.pos == workspace->out_buf.size) { |
5c1aab1d NT |
448 | tot_out += PAGE_SIZE; |
449 | max_out -= PAGE_SIZE; | |
450 | kunmap(out_page); | |
451 | if (nr_pages == nr_dest_pages) { | |
452 | out_page = NULL; | |
453 | ret = -E2BIG; | |
454 | goto out; | |
455 | } | |
456 | out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); | |
457 | if (out_page == NULL) { | |
458 | ret = -ENOMEM; | |
459 | goto out; | |
460 | } | |
461 | pages[nr_pages++] = out_page; | |
431e9822 DS |
462 | workspace->out_buf.dst = kmap(out_page); |
463 | workspace->out_buf.pos = 0; | |
464 | workspace->out_buf.size = min_t(size_t, max_out, | |
465 | PAGE_SIZE); | |
5c1aab1d NT |
466 | } |
467 | ||
468 | /* We've reached the end of the input */ | |
431e9822 DS |
469 | if (workspace->in_buf.pos >= len) { |
470 | tot_in += workspace->in_buf.pos; | |
5c1aab1d NT |
471 | break; |
472 | } | |
473 | ||
474 | /* Check if we need more input */ | |
431e9822 | 475 | if (workspace->in_buf.pos == workspace->in_buf.size) { |
5c1aab1d NT |
476 | tot_in += PAGE_SIZE; |
477 | kunmap(in_page); | |
478 | put_page(in_page); | |
479 | ||
480 | start += PAGE_SIZE; | |
481 | len -= PAGE_SIZE; | |
482 | in_page = find_get_page(mapping, start >> PAGE_SHIFT); | |
431e9822 DS |
483 | workspace->in_buf.src = kmap(in_page); |
484 | workspace->in_buf.pos = 0; | |
485 | workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE); | |
5c1aab1d NT |
486 | } |
487 | } | |
488 | while (1) { | |
489 | size_t ret2; | |
490 | ||
431e9822 | 491 | ret2 = ZSTD_endStream(stream, &workspace->out_buf); |
5c1aab1d NT |
492 | if (ZSTD_isError(ret2)) { |
493 | pr_debug("BTRFS: ZSTD_endStream returned %d\n", | |
494 | ZSTD_getErrorCode(ret2)); | |
495 | ret = -EIO; | |
496 | goto out; | |
497 | } | |
498 | if (ret2 == 0) { | |
431e9822 | 499 | tot_out += workspace->out_buf.pos; |
5c1aab1d NT |
500 | break; |
501 | } | |
431e9822 DS |
502 | if (workspace->out_buf.pos >= max_out) { |
503 | tot_out += workspace->out_buf.pos; | |
5c1aab1d NT |
504 | ret = -E2BIG; |
505 | goto out; | |
506 | } | |
507 | ||
508 | tot_out += PAGE_SIZE; | |
509 | max_out -= PAGE_SIZE; | |
510 | kunmap(out_page); | |
511 | if (nr_pages == nr_dest_pages) { | |
512 | out_page = NULL; | |
513 | ret = -E2BIG; | |
514 | goto out; | |
515 | } | |
516 | out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); | |
517 | if (out_page == NULL) { | |
518 | ret = -ENOMEM; | |
519 | goto out; | |
520 | } | |
521 | pages[nr_pages++] = out_page; | |
431e9822 DS |
522 | workspace->out_buf.dst = kmap(out_page); |
523 | workspace->out_buf.pos = 0; | |
524 | workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE); | |
5c1aab1d NT |
525 | } |
526 | ||
527 | if (tot_out >= tot_in) { | |
528 | ret = -E2BIG; | |
529 | goto out; | |
530 | } | |
531 | ||
532 | ret = 0; | |
533 | *total_in = tot_in; | |
534 | *total_out = tot_out; | |
535 | out: | |
536 | *out_pages = nr_pages; | |
537 | /* Cleanup */ | |
538 | if (in_page) { | |
539 | kunmap(in_page); | |
540 | put_page(in_page); | |
541 | } | |
542 | if (out_page) | |
543 | kunmap(out_page); | |
544 | return ret; | |
545 | } | |
546 | ||
547 | static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) | |
548 | { | |
549 | struct workspace *workspace = list_entry(ws, struct workspace, list); | |
550 | struct page **pages_in = cb->compressed_pages; | |
551 | u64 disk_start = cb->start; | |
552 | struct bio *orig_bio = cb->orig_bio; | |
553 | size_t srclen = cb->compressed_len; | |
554 | ZSTD_DStream *stream; | |
555 | int ret = 0; | |
556 | unsigned long page_in_index = 0; | |
557 | unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE); | |
558 | unsigned long buf_start; | |
559 | unsigned long total_out = 0; | |
5c1aab1d NT |
560 | |
561 | stream = ZSTD_initDStream( | |
562 | ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size); | |
563 | if (!stream) { | |
564 | pr_debug("BTRFS: ZSTD_initDStream failed\n"); | |
565 | ret = -EIO; | |
566 | goto done; | |
567 | } | |
568 | ||
431e9822 DS |
569 | workspace->in_buf.src = kmap(pages_in[page_in_index]); |
570 | workspace->in_buf.pos = 0; | |
571 | workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE); | |
5c1aab1d | 572 | |
431e9822 DS |
573 | workspace->out_buf.dst = workspace->buf; |
574 | workspace->out_buf.pos = 0; | |
575 | workspace->out_buf.size = PAGE_SIZE; | |
5c1aab1d NT |
576 | |
577 | while (1) { | |
578 | size_t ret2; | |
579 | ||
431e9822 DS |
580 | ret2 = ZSTD_decompressStream(stream, &workspace->out_buf, |
581 | &workspace->in_buf); | |
5c1aab1d NT |
582 | if (ZSTD_isError(ret2)) { |
583 | pr_debug("BTRFS: ZSTD_decompressStream returned %d\n", | |
584 | ZSTD_getErrorCode(ret2)); | |
585 | ret = -EIO; | |
586 | goto done; | |
587 | } | |
588 | buf_start = total_out; | |
431e9822 DS |
589 | total_out += workspace->out_buf.pos; |
590 | workspace->out_buf.pos = 0; | |
5c1aab1d | 591 | |
431e9822 DS |
592 | ret = btrfs_decompress_buf2page(workspace->out_buf.dst, |
593 | buf_start, total_out, disk_start, orig_bio); | |
5c1aab1d NT |
594 | if (ret == 0) |
595 | break; | |
596 | ||
431e9822 | 597 | if (workspace->in_buf.pos >= srclen) |
5c1aab1d NT |
598 | break; |
599 | ||
600 | /* Check if we've hit the end of a frame */ | |
601 | if (ret2 == 0) | |
602 | break; | |
603 | ||
431e9822 | 604 | if (workspace->in_buf.pos == workspace->in_buf.size) { |
5c1aab1d NT |
605 | kunmap(pages_in[page_in_index++]); |
606 | if (page_in_index >= total_pages_in) { | |
431e9822 | 607 | workspace->in_buf.src = NULL; |
5c1aab1d NT |
608 | ret = -EIO; |
609 | goto done; | |
610 | } | |
611 | srclen -= PAGE_SIZE; | |
431e9822 DS |
612 | workspace->in_buf.src = kmap(pages_in[page_in_index]); |
613 | workspace->in_buf.pos = 0; | |
614 | workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE); | |
5c1aab1d NT |
615 | } |
616 | } | |
617 | ret = 0; | |
618 | zero_fill_bio(orig_bio); | |
619 | done: | |
431e9822 | 620 | if (workspace->in_buf.src) |
5c1aab1d NT |
621 | kunmap(pages_in[page_in_index]); |
622 | return ret; | |
623 | } | |
624 | ||
625 | static int zstd_decompress(struct list_head *ws, unsigned char *data_in, | |
626 | struct page *dest_page, | |
627 | unsigned long start_byte, | |
628 | size_t srclen, size_t destlen) | |
629 | { | |
630 | struct workspace *workspace = list_entry(ws, struct workspace, list); | |
631 | ZSTD_DStream *stream; | |
632 | int ret = 0; | |
633 | size_t ret2; | |
5c1aab1d NT |
634 | unsigned long total_out = 0; |
635 | unsigned long pg_offset = 0; | |
636 | char *kaddr; | |
637 | ||
638 | stream = ZSTD_initDStream( | |
639 | ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size); | |
640 | if (!stream) { | |
641 | pr_warn("BTRFS: ZSTD_initDStream failed\n"); | |
642 | ret = -EIO; | |
643 | goto finish; | |
644 | } | |
645 | ||
646 | destlen = min_t(size_t, destlen, PAGE_SIZE); | |
647 | ||
431e9822 DS |
648 | workspace->in_buf.src = data_in; |
649 | workspace->in_buf.pos = 0; | |
650 | workspace->in_buf.size = srclen; | |
5c1aab1d | 651 | |
431e9822 DS |
652 | workspace->out_buf.dst = workspace->buf; |
653 | workspace->out_buf.pos = 0; | |
654 | workspace->out_buf.size = PAGE_SIZE; | |
5c1aab1d NT |
655 | |
656 | ret2 = 1; | |
431e9822 DS |
657 | while (pg_offset < destlen |
658 | && workspace->in_buf.pos < workspace->in_buf.size) { | |
5c1aab1d NT |
659 | unsigned long buf_start; |
660 | unsigned long buf_offset; | |
661 | unsigned long bytes; | |
662 | ||
663 | /* Check if the frame is over and we still need more input */ | |
664 | if (ret2 == 0) { | |
665 | pr_debug("BTRFS: ZSTD_decompressStream ended early\n"); | |
666 | ret = -EIO; | |
667 | goto finish; | |
668 | } | |
431e9822 DS |
669 | ret2 = ZSTD_decompressStream(stream, &workspace->out_buf, |
670 | &workspace->in_buf); | |
5c1aab1d NT |
671 | if (ZSTD_isError(ret2)) { |
672 | pr_debug("BTRFS: ZSTD_decompressStream returned %d\n", | |
673 | ZSTD_getErrorCode(ret2)); | |
674 | ret = -EIO; | |
675 | goto finish; | |
676 | } | |
677 | ||
678 | buf_start = total_out; | |
431e9822 DS |
679 | total_out += workspace->out_buf.pos; |
680 | workspace->out_buf.pos = 0; | |
5c1aab1d NT |
681 | |
682 | if (total_out <= start_byte) | |
683 | continue; | |
684 | ||
685 | if (total_out > start_byte && buf_start < start_byte) | |
686 | buf_offset = start_byte - buf_start; | |
687 | else | |
688 | buf_offset = 0; | |
689 | ||
690 | bytes = min_t(unsigned long, destlen - pg_offset, | |
431e9822 | 691 | workspace->out_buf.size - buf_offset); |
5c1aab1d NT |
692 | |
693 | kaddr = kmap_atomic(dest_page); | |
431e9822 DS |
694 | memcpy(kaddr + pg_offset, workspace->out_buf.dst + buf_offset, |
695 | bytes); | |
5c1aab1d NT |
696 | kunmap_atomic(kaddr); |
697 | ||
698 | pg_offset += bytes; | |
699 | } | |
700 | ret = 0; | |
701 | finish: | |
702 | if (pg_offset < destlen) { | |
703 | kaddr = kmap_atomic(dest_page); | |
704 | memset(kaddr + pg_offset, 0, destlen - pg_offset); | |
705 | kunmap_atomic(kaddr); | |
706 | } | |
707 | return ret; | |
708 | } | |
709 | ||
d0ab62ce | 710 | static unsigned int zstd_set_level(unsigned int level) |
f51d2b59 | 711 | { |
3f93aef5 DZ |
712 | if (!level) |
713 | return ZSTD_BTRFS_DEFAULT_LEVEL; | |
714 | ||
715 | return min_t(unsigned int, level, ZSTD_BTRFS_MAX_LEVEL); | |
f51d2b59 DS |
716 | } |
717 | ||
5c1aab1d | 718 | const struct btrfs_compress_op btrfs_zstd_compress = { |
92ee5530 DZ |
719 | .init_workspace_manager = zstd_init_workspace_manager, |
720 | .cleanup_workspace_manager = zstd_cleanup_workspace_manager, | |
721 | .get_workspace = zstd_get_workspace, | |
722 | .put_workspace = zstd_put_workspace, | |
5c1aab1d NT |
723 | .alloc_workspace = zstd_alloc_workspace, |
724 | .free_workspace = zstd_free_workspace, | |
725 | .compress_pages = zstd_compress_pages, | |
726 | .decompress_bio = zstd_decompress_bio, | |
727 | .decompress = zstd_decompress, | |
f51d2b59 | 728 | .set_level = zstd_set_level, |
5c1aab1d | 729 | }; |