]>
Commit | Line | Data |
---|---|---|
6cbd5570 CM |
1 | /* |
2 | * Copyright (C) 2007 Oracle. All rights reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public | |
6 | * License v2 as published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, | |
9 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
11 | * General Public License for more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public | |
14 | * License along with this program; if not, write to the | |
15 | * Free Software Foundation, Inc., 59 Temple Place - Suite 330, | |
16 | * Boston, MA 021110-1307, USA. | |
17 | */ | |
18 | ||
065631f6 | 19 | #include <linux/bio.h> |
5a0e3ad6 | 20 | #include <linux/slab.h> |
065631f6 CM |
21 | #include <linux/pagemap.h> |
22 | #include <linux/highmem.h> | |
1e1d2701 | 23 | #include "ctree.h" |
dee26a9f | 24 | #include "disk-io.h" |
9f5fae2f | 25 | #include "transaction.h" |
facc8a22 | 26 | #include "volumes.h" |
1de037a4 | 27 | #include "print-tree.h" |
1e1d2701 | 28 | |
995e01b7 | 29 | #define __MAX_CSUM_ITEMS(r, size) ((unsigned long)(((BTRFS_LEAF_DATA_SIZE(r) - \ |
607d432d JB |
30 | sizeof(struct btrfs_item) * 2) / \ |
31 | size) - 1)) | |
07d400a6 | 32 | |
221b8318 ZB |
33 | #define MAX_CSUM_ITEMS(r, size) (min_t(u32, __MAX_CSUM_ITEMS(r, size), \ |
34 | PAGE_CACHE_SIZE)) | |
7ca4be45 | 35 | |
07d400a6 YZ |
36 | #define MAX_ORDERED_SUM_BYTES(r) ((PAGE_SIZE - \ |
37 | sizeof(struct btrfs_ordered_sum)) / \ | |
f51a4a18 | 38 | sizeof(u32) * (r)->sectorsize) |
07d400a6 | 39 | |
b18c6685 | 40 | int btrfs_insert_file_extent(struct btrfs_trans_handle *trans, |
f2eb0a24 SW |
41 | struct btrfs_root *root, |
42 | u64 objectid, u64 pos, | |
43 | u64 disk_offset, u64 disk_num_bytes, | |
c8b97818 CM |
44 | u64 num_bytes, u64 offset, u64 ram_bytes, |
45 | u8 compression, u8 encryption, u16 other_encoding) | |
9f5fae2f | 46 | { |
dee26a9f CM |
47 | int ret = 0; |
48 | struct btrfs_file_extent_item *item; | |
49 | struct btrfs_key file_key; | |
5caf2a00 | 50 | struct btrfs_path *path; |
5f39d397 | 51 | struct extent_buffer *leaf; |
dee26a9f | 52 | |
5caf2a00 | 53 | path = btrfs_alloc_path(); |
db5b493a TI |
54 | if (!path) |
55 | return -ENOMEM; | |
dee26a9f | 56 | file_key.objectid = objectid; |
b18c6685 | 57 | file_key.offset = pos; |
dee26a9f CM |
58 | btrfs_set_key_type(&file_key, BTRFS_EXTENT_DATA_KEY); |
59 | ||
b9473439 | 60 | path->leave_spinning = 1; |
5caf2a00 | 61 | ret = btrfs_insert_empty_item(trans, root, path, &file_key, |
dee26a9f | 62 | sizeof(*item)); |
54aa1f4d CM |
63 | if (ret < 0) |
64 | goto out; | |
79787eaa | 65 | BUG_ON(ret); /* Can't happen */ |
5f39d397 CM |
66 | leaf = path->nodes[0]; |
67 | item = btrfs_item_ptr(leaf, path->slots[0], | |
dee26a9f | 68 | struct btrfs_file_extent_item); |
f2eb0a24 | 69 | btrfs_set_file_extent_disk_bytenr(leaf, item, disk_offset); |
db94535d | 70 | btrfs_set_file_extent_disk_num_bytes(leaf, item, disk_num_bytes); |
f2eb0a24 | 71 | btrfs_set_file_extent_offset(leaf, item, offset); |
db94535d | 72 | btrfs_set_file_extent_num_bytes(leaf, item, num_bytes); |
c8b97818 | 73 | btrfs_set_file_extent_ram_bytes(leaf, item, ram_bytes); |
5f39d397 CM |
74 | btrfs_set_file_extent_generation(leaf, item, trans->transid); |
75 | btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG); | |
c8b97818 CM |
76 | btrfs_set_file_extent_compression(leaf, item, compression); |
77 | btrfs_set_file_extent_encryption(leaf, item, encryption); | |
78 | btrfs_set_file_extent_other_encoding(leaf, item, other_encoding); | |
79 | ||
5f39d397 | 80 | btrfs_mark_buffer_dirty(leaf); |
54aa1f4d | 81 | out: |
5caf2a00 | 82 | btrfs_free_path(path); |
54aa1f4d | 83 | return ret; |
9f5fae2f | 84 | } |
dee26a9f | 85 | |
48a3b636 ES |
86 | static struct btrfs_csum_item * |
87 | btrfs_lookup_csum(struct btrfs_trans_handle *trans, | |
88 | struct btrfs_root *root, | |
89 | struct btrfs_path *path, | |
90 | u64 bytenr, int cow) | |
6567e837 CM |
91 | { |
92 | int ret; | |
93 | struct btrfs_key file_key; | |
94 | struct btrfs_key found_key; | |
95 | struct btrfs_csum_item *item; | |
5f39d397 | 96 | struct extent_buffer *leaf; |
6567e837 | 97 | u64 csum_offset = 0; |
6c41761f | 98 | u16 csum_size = btrfs_super_csum_size(root->fs_info->super_copy); |
a429e513 | 99 | int csums_in_item; |
6567e837 | 100 | |
d20f7043 CM |
101 | file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; |
102 | file_key.offset = bytenr; | |
103 | btrfs_set_key_type(&file_key, BTRFS_EXTENT_CSUM_KEY); | |
b18c6685 | 104 | ret = btrfs_search_slot(trans, root, &file_key, path, 0, cow); |
6567e837 CM |
105 | if (ret < 0) |
106 | goto fail; | |
5f39d397 | 107 | leaf = path->nodes[0]; |
6567e837 CM |
108 | if (ret > 0) { |
109 | ret = 1; | |
70b2befd | 110 | if (path->slots[0] == 0) |
6567e837 CM |
111 | goto fail; |
112 | path->slots[0]--; | |
5f39d397 | 113 | btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); |
d20f7043 | 114 | if (btrfs_key_type(&found_key) != BTRFS_EXTENT_CSUM_KEY) |
6567e837 | 115 | goto fail; |
d20f7043 CM |
116 | |
117 | csum_offset = (bytenr - found_key.offset) >> | |
6567e837 | 118 | root->fs_info->sb->s_blocksize_bits; |
5f39d397 | 119 | csums_in_item = btrfs_item_size_nr(leaf, path->slots[0]); |
607d432d | 120 | csums_in_item /= csum_size; |
a429e513 | 121 | |
82d130ff | 122 | if (csum_offset == csums_in_item) { |
a429e513 | 123 | ret = -EFBIG; |
6567e837 | 124 | goto fail; |
82d130ff MX |
125 | } else if (csum_offset > csums_in_item) { |
126 | goto fail; | |
6567e837 CM |
127 | } |
128 | } | |
129 | item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item); | |
509659cd | 130 | item = (struct btrfs_csum_item *)((unsigned char *)item + |
607d432d | 131 | csum_offset * csum_size); |
6567e837 CM |
132 | return item; |
133 | fail: | |
134 | if (ret > 0) | |
b18c6685 | 135 | ret = -ENOENT; |
6567e837 CM |
136 | return ERR_PTR(ret); |
137 | } | |
138 | ||
dee26a9f CM |
139 | int btrfs_lookup_file_extent(struct btrfs_trans_handle *trans, |
140 | struct btrfs_root *root, | |
141 | struct btrfs_path *path, u64 objectid, | |
9773a788 | 142 | u64 offset, int mod) |
dee26a9f CM |
143 | { |
144 | int ret; | |
145 | struct btrfs_key file_key; | |
146 | int ins_len = mod < 0 ? -1 : 0; | |
147 | int cow = mod != 0; | |
148 | ||
149 | file_key.objectid = objectid; | |
70b2befd | 150 | file_key.offset = offset; |
dee26a9f CM |
151 | btrfs_set_key_type(&file_key, BTRFS_EXTENT_DATA_KEY); |
152 | ret = btrfs_search_slot(trans, root, &file_key, path, ins_len, cow); | |
153 | return ret; | |
154 | } | |
f254e52c | 155 | |
facc8a22 MX |
156 | static void btrfs_io_bio_endio_readpage(struct btrfs_io_bio *bio, int err) |
157 | { | |
158 | kfree(bio->csum_allocated); | |
159 | } | |
160 | ||
4b46fce2 JB |
161 | static int __btrfs_lookup_bio_sums(struct btrfs_root *root, |
162 | struct inode *inode, struct bio *bio, | |
163 | u64 logical_offset, u32 *dst, int dio) | |
61b49440 | 164 | { |
61b49440 | 165 | struct bio_vec *bvec = bio->bi_io_vec; |
facc8a22 MX |
166 | struct btrfs_io_bio *btrfs_bio = btrfs_io_bio(bio); |
167 | struct btrfs_csum_item *item = NULL; | |
168 | struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; | |
169 | struct btrfs_path *path; | |
170 | u8 *csum; | |
4b46fce2 | 171 | u64 offset = 0; |
61b49440 CM |
172 | u64 item_start_offset = 0; |
173 | u64 item_last_offset = 0; | |
d20f7043 | 174 | u64 disk_bytenr; |
61b49440 | 175 | u32 diff; |
facc8a22 MX |
176 | int nblocks; |
177 | int bio_index = 0; | |
e4100d98 | 178 | int count; |
facc8a22 | 179 | u16 csum_size = btrfs_super_csum_size(root->fs_info->super_copy); |
61b49440 CM |
180 | |
181 | path = btrfs_alloc_path(); | |
c2db1073 TI |
182 | if (!path) |
183 | return -ENOMEM; | |
facc8a22 | 184 | |
4f024f37 | 185 | nblocks = bio->bi_iter.bi_size >> inode->i_sb->s_blocksize_bits; |
facc8a22 MX |
186 | if (!dst) { |
187 | if (nblocks * csum_size > BTRFS_BIO_INLINE_CSUM_SIZE) { | |
188 | btrfs_bio->csum_allocated = kmalloc(nblocks * csum_size, | |
189 | GFP_NOFS); | |
190 | if (!btrfs_bio->csum_allocated) { | |
191 | btrfs_free_path(path); | |
192 | return -ENOMEM; | |
193 | } | |
194 | btrfs_bio->csum = btrfs_bio->csum_allocated; | |
195 | btrfs_bio->end_io = btrfs_io_bio_endio_readpage; | |
196 | } else { | |
197 | btrfs_bio->csum = btrfs_bio->csum_inline; | |
198 | } | |
199 | csum = btrfs_bio->csum; | |
200 | } else { | |
201 | csum = (u8 *)dst; | |
202 | } | |
203 | ||
4f024f37 | 204 | if (bio->bi_iter.bi_size > PAGE_CACHE_SIZE * 8) |
4d1b5fb4 | 205 | path->reada = 2; |
61b49440 CM |
206 | |
207 | WARN_ON(bio->bi_vcnt <= 0); | |
208 | ||
2cf8572d CM |
209 | /* |
210 | * the free space stuff is only read when it hasn't been | |
211 | * updated in the current transaction. So, we can safely | |
212 | * read from the commit root and sidestep a nasty deadlock | |
213 | * between reading the free space cache and updating the csum tree. | |
214 | */ | |
83eea1f1 | 215 | if (btrfs_is_free_space_inode(inode)) { |
2cf8572d | 216 | path->search_commit_root = 1; |
ddf23b3f JB |
217 | path->skip_locking = 1; |
218 | } | |
2cf8572d | 219 | |
4f024f37 | 220 | disk_bytenr = (u64)bio->bi_iter.bi_sector << 9; |
4b46fce2 JB |
221 | if (dio) |
222 | offset = logical_offset; | |
d397712b | 223 | while (bio_index < bio->bi_vcnt) { |
4b46fce2 JB |
224 | if (!dio) |
225 | offset = page_offset(bvec->bv_page) + bvec->bv_offset; | |
facc8a22 MX |
226 | count = btrfs_find_ordered_sum(inode, offset, disk_bytenr, |
227 | (u32 *)csum, nblocks); | |
e4100d98 | 228 | if (count) |
61b49440 CM |
229 | goto found; |
230 | ||
d20f7043 CM |
231 | if (!item || disk_bytenr < item_start_offset || |
232 | disk_bytenr >= item_last_offset) { | |
61b49440 CM |
233 | struct btrfs_key found_key; |
234 | u32 item_size; | |
235 | ||
236 | if (item) | |
b3b4aa74 | 237 | btrfs_release_path(path); |
d20f7043 CM |
238 | item = btrfs_lookup_csum(NULL, root->fs_info->csum_root, |
239 | path, disk_bytenr, 0); | |
61b49440 | 240 | if (IS_ERR(item)) { |
e4100d98 | 241 | count = 1; |
facc8a22 | 242 | memset(csum, 0, csum_size); |
17d217fe YZ |
243 | if (BTRFS_I(inode)->root->root_key.objectid == |
244 | BTRFS_DATA_RELOC_TREE_OBJECTID) { | |
245 | set_extent_bits(io_tree, offset, | |
246 | offset + bvec->bv_len - 1, | |
247 | EXTENT_NODATASUM, GFP_NOFS); | |
248 | } else { | |
efe120a0 FH |
249 | btrfs_info(BTRFS_I(inode)->root->fs_info, |
250 | "no csum found for inode %llu start %llu", | |
c1c9ff7c | 251 | btrfs_ino(inode), offset); |
17d217fe | 252 | } |
6dab8157 | 253 | item = NULL; |
b3b4aa74 | 254 | btrfs_release_path(path); |
61b49440 CM |
255 | goto found; |
256 | } | |
257 | btrfs_item_key_to_cpu(path->nodes[0], &found_key, | |
258 | path->slots[0]); | |
259 | ||
260 | item_start_offset = found_key.offset; | |
261 | item_size = btrfs_item_size_nr(path->nodes[0], | |
262 | path->slots[0]); | |
263 | item_last_offset = item_start_offset + | |
607d432d | 264 | (item_size / csum_size) * |
61b49440 CM |
265 | root->sectorsize; |
266 | item = btrfs_item_ptr(path->nodes[0], path->slots[0], | |
267 | struct btrfs_csum_item); | |
268 | } | |
269 | /* | |
270 | * this byte range must be able to fit inside | |
271 | * a single leaf so it will also fit inside a u32 | |
272 | */ | |
d20f7043 | 273 | diff = disk_bytenr - item_start_offset; |
61b49440 | 274 | diff = diff / root->sectorsize; |
607d432d | 275 | diff = diff * csum_size; |
facc8a22 MX |
276 | count = min_t(int, nblocks, (item_last_offset - disk_bytenr) >> |
277 | inode->i_sb->s_blocksize_bits); | |
278 | read_extent_buffer(path->nodes[0], csum, | |
3de9d6b6 | 279 | ((unsigned long)item) + diff, |
e4100d98 | 280 | csum_size * count); |
61b49440 | 281 | found: |
facc8a22 MX |
282 | csum += count * csum_size; |
283 | nblocks -= count; | |
e4100d98 MX |
284 | while (count--) { |
285 | disk_bytenr += bvec->bv_len; | |
286 | offset += bvec->bv_len; | |
287 | bio_index++; | |
288 | bvec++; | |
289 | } | |
61b49440 CM |
290 | } |
291 | btrfs_free_path(path); | |
292 | return 0; | |
293 | } | |
294 | ||
4b46fce2 JB |
295 | int btrfs_lookup_bio_sums(struct btrfs_root *root, struct inode *inode, |
296 | struct bio *bio, u32 *dst) | |
297 | { | |
298 | return __btrfs_lookup_bio_sums(root, inode, bio, 0, dst, 0); | |
299 | } | |
300 | ||
301 | int btrfs_lookup_bio_sums_dio(struct btrfs_root *root, struct inode *inode, | |
facc8a22 MX |
302 | struct btrfs_dio_private *dip, struct bio *bio, |
303 | u64 offset) | |
4b46fce2 | 304 | { |
4f024f37 | 305 | int len = (bio->bi_iter.bi_sector << 9) - dip->disk_bytenr; |
facc8a22 MX |
306 | u16 csum_size = btrfs_super_csum_size(root->fs_info->super_copy); |
307 | int ret; | |
308 | ||
309 | len >>= inode->i_sb->s_blocksize_bits; | |
310 | len *= csum_size; | |
311 | ||
312 | ret = __btrfs_lookup_bio_sums(root, inode, bio, offset, | |
313 | (u32 *)(dip->csum + len), 1); | |
314 | return ret; | |
4b46fce2 JB |
315 | } |
316 | ||
17d217fe | 317 | int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, |
a2de733c | 318 | struct list_head *list, int search_commit) |
17d217fe YZ |
319 | { |
320 | struct btrfs_key key; | |
321 | struct btrfs_path *path; | |
322 | struct extent_buffer *leaf; | |
323 | struct btrfs_ordered_sum *sums; | |
17d217fe | 324 | struct btrfs_csum_item *item; |
0678b618 | 325 | LIST_HEAD(tmplist); |
17d217fe YZ |
326 | unsigned long offset; |
327 | int ret; | |
328 | size_t size; | |
329 | u64 csum_end; | |
6c41761f | 330 | u16 csum_size = btrfs_super_csum_size(root->fs_info->super_copy); |
17d217fe | 331 | |
4277a9c3 JB |
332 | ASSERT(start == ALIGN(start, root->sectorsize) && |
333 | (end + 1) == ALIGN(end + 1, root->sectorsize)); | |
334 | ||
17d217fe | 335 | path = btrfs_alloc_path(); |
d8926bb3 MF |
336 | if (!path) |
337 | return -ENOMEM; | |
17d217fe | 338 | |
a2de733c AJ |
339 | if (search_commit) { |
340 | path->skip_locking = 1; | |
341 | path->reada = 2; | |
342 | path->search_commit_root = 1; | |
343 | } | |
344 | ||
17d217fe YZ |
345 | key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; |
346 | key.offset = start; | |
347 | key.type = BTRFS_EXTENT_CSUM_KEY; | |
348 | ||
07d400a6 | 349 | ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); |
17d217fe YZ |
350 | if (ret < 0) |
351 | goto fail; | |
352 | if (ret > 0 && path->slots[0] > 0) { | |
353 | leaf = path->nodes[0]; | |
354 | btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1); | |
355 | if (key.objectid == BTRFS_EXTENT_CSUM_OBJECTID && | |
356 | key.type == BTRFS_EXTENT_CSUM_KEY) { | |
357 | offset = (start - key.offset) >> | |
358 | root->fs_info->sb->s_blocksize_bits; | |
359 | if (offset * csum_size < | |
360 | btrfs_item_size_nr(leaf, path->slots[0] - 1)) | |
361 | path->slots[0]--; | |
362 | } | |
363 | } | |
364 | ||
365 | while (start <= end) { | |
366 | leaf = path->nodes[0]; | |
367 | if (path->slots[0] >= btrfs_header_nritems(leaf)) { | |
07d400a6 | 368 | ret = btrfs_next_leaf(root, path); |
17d217fe YZ |
369 | if (ret < 0) |
370 | goto fail; | |
371 | if (ret > 0) | |
372 | break; | |
373 | leaf = path->nodes[0]; | |
374 | } | |
375 | ||
376 | btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); | |
377 | if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || | |
628c8282 ZYW |
378 | key.type != BTRFS_EXTENT_CSUM_KEY || |
379 | key.offset > end) | |
17d217fe YZ |
380 | break; |
381 | ||
382 | if (key.offset > start) | |
383 | start = key.offset; | |
384 | ||
385 | size = btrfs_item_size_nr(leaf, path->slots[0]); | |
386 | csum_end = key.offset + (size / csum_size) * root->sectorsize; | |
87b29b20 YZ |
387 | if (csum_end <= start) { |
388 | path->slots[0]++; | |
389 | continue; | |
390 | } | |
17d217fe | 391 | |
07d400a6 | 392 | csum_end = min(csum_end, end + 1); |
17d217fe YZ |
393 | item = btrfs_item_ptr(path->nodes[0], path->slots[0], |
394 | struct btrfs_csum_item); | |
07d400a6 YZ |
395 | while (start < csum_end) { |
396 | size = min_t(size_t, csum_end - start, | |
f51a4a18 | 397 | MAX_ORDERED_SUM_BYTES(root)); |
07d400a6 | 398 | sums = kzalloc(btrfs_ordered_sum_size(root, size), |
f51a4a18 | 399 | GFP_NOFS); |
0678b618 MF |
400 | if (!sums) { |
401 | ret = -ENOMEM; | |
402 | goto fail; | |
403 | } | |
17d217fe | 404 | |
07d400a6 | 405 | sums->bytenr = start; |
f51a4a18 | 406 | sums->len = (int)size; |
07d400a6 YZ |
407 | |
408 | offset = (start - key.offset) >> | |
409 | root->fs_info->sb->s_blocksize_bits; | |
410 | offset *= csum_size; | |
f51a4a18 | 411 | size >>= root->fs_info->sb->s_blocksize_bits; |
07d400a6 | 412 | |
f51a4a18 MX |
413 | read_extent_buffer(path->nodes[0], |
414 | sums->sums, | |
415 | ((unsigned long)item) + offset, | |
416 | csum_size * size); | |
417 | ||
418 | start += root->sectorsize * size; | |
0678b618 | 419 | list_add_tail(&sums->list, &tmplist); |
07d400a6 | 420 | } |
17d217fe YZ |
421 | path->slots[0]++; |
422 | } | |
423 | ret = 0; | |
424 | fail: | |
0678b618 MF |
425 | while (ret < 0 && !list_empty(&tmplist)) { |
426 | sums = list_entry(&tmplist, struct btrfs_ordered_sum, list); | |
427 | list_del(&sums->list); | |
428 | kfree(sums); | |
429 | } | |
430 | list_splice_tail(&tmplist, list); | |
431 | ||
17d217fe YZ |
432 | btrfs_free_path(path); |
433 | return ret; | |
434 | } | |
435 | ||
3edf7d33 | 436 | int btrfs_csum_one_bio(struct btrfs_root *root, struct inode *inode, |
d20f7043 | 437 | struct bio *bio, u64 file_start, int contig) |
e015640f | 438 | { |
e6dcd2dc | 439 | struct btrfs_ordered_sum *sums; |
3edf7d33 | 440 | struct btrfs_ordered_extent *ordered; |
e015640f CM |
441 | char *data; |
442 | struct bio_vec *bvec = bio->bi_io_vec; | |
443 | int bio_index = 0; | |
f51a4a18 | 444 | int index; |
3edf7d33 CM |
445 | unsigned long total_bytes = 0; |
446 | unsigned long this_sum_bytes = 0; | |
447 | u64 offset; | |
e015640f | 448 | |
e6dcd2dc | 449 | WARN_ON(bio->bi_vcnt <= 0); |
4f024f37 KO |
450 | sums = kzalloc(btrfs_ordered_sum_size(root, bio->bi_iter.bi_size), |
451 | GFP_NOFS); | |
e015640f CM |
452 | if (!sums) |
453 | return -ENOMEM; | |
3edf7d33 | 454 | |
4f024f37 | 455 | sums->len = bio->bi_iter.bi_size; |
e6dcd2dc | 456 | INIT_LIST_HEAD(&sums->list); |
d20f7043 CM |
457 | |
458 | if (contig) | |
459 | offset = file_start; | |
460 | else | |
461 | offset = page_offset(bvec->bv_page) + bvec->bv_offset; | |
462 | ||
463 | ordered = btrfs_lookup_ordered_extent(inode, offset); | |
79787eaa | 464 | BUG_ON(!ordered); /* Logic error */ |
4f024f37 | 465 | sums->bytenr = (u64)bio->bi_iter.bi_sector << 9; |
f51a4a18 | 466 | index = 0; |
e015640f | 467 | |
d397712b | 468 | while (bio_index < bio->bi_vcnt) { |
d20f7043 CM |
469 | if (!contig) |
470 | offset = page_offset(bvec->bv_page) + bvec->bv_offset; | |
471 | ||
e58dd74b JB |
472 | if (offset >= ordered->file_offset + ordered->len || |
473 | offset < ordered->file_offset) { | |
3edf7d33 CM |
474 | unsigned long bytes_left; |
475 | sums->len = this_sum_bytes; | |
476 | this_sum_bytes = 0; | |
477 | btrfs_add_ordered_sum(inode, ordered, sums); | |
478 | btrfs_put_ordered_extent(ordered); | |
479 | ||
4f024f37 | 480 | bytes_left = bio->bi_iter.bi_size - total_bytes; |
3edf7d33 CM |
481 | |
482 | sums = kzalloc(btrfs_ordered_sum_size(root, bytes_left), | |
483 | GFP_NOFS); | |
79787eaa | 484 | BUG_ON(!sums); /* -ENOMEM */ |
3edf7d33 | 485 | sums->len = bytes_left; |
d20f7043 | 486 | ordered = btrfs_lookup_ordered_extent(inode, offset); |
79787eaa | 487 | BUG_ON(!ordered); /* Logic error */ |
4f024f37 | 488 | sums->bytenr = ((u64)bio->bi_iter.bi_sector << 9) + |
f51a4a18 MX |
489 | total_bytes; |
490 | index = 0; | |
3edf7d33 CM |
491 | } |
492 | ||
7ac687d9 | 493 | data = kmap_atomic(bvec->bv_page); |
f51a4a18 MX |
494 | sums->sums[index] = ~(u32)0; |
495 | sums->sums[index] = btrfs_csum_data(data + bvec->bv_offset, | |
496 | sums->sums[index], | |
497 | bvec->bv_len); | |
7ac687d9 | 498 | kunmap_atomic(data); |
f51a4a18 MX |
499 | btrfs_csum_final(sums->sums[index], |
500 | (char *)(sums->sums + index)); | |
ed98b56a | 501 | |
e015640f | 502 | bio_index++; |
f51a4a18 | 503 | index++; |
3edf7d33 CM |
504 | total_bytes += bvec->bv_len; |
505 | this_sum_bytes += bvec->bv_len; | |
d20f7043 | 506 | offset += bvec->bv_len; |
e015640f CM |
507 | bvec++; |
508 | } | |
ed98b56a | 509 | this_sum_bytes = 0; |
3edf7d33 CM |
510 | btrfs_add_ordered_sum(inode, ordered, sums); |
511 | btrfs_put_ordered_extent(ordered); | |
e015640f CM |
512 | return 0; |
513 | } | |
514 | ||
459931ec CM |
515 | /* |
516 | * helper function for csum removal, this expects the | |
517 | * key to describe the csum pointed to by the path, and it expects | |
518 | * the csum to overlap the range [bytenr, len] | |
519 | * | |
520 | * The csum should not be entirely contained in the range and the | |
521 | * range should not be entirely contained in the csum. | |
522 | * | |
523 | * This calls btrfs_truncate_item with the correct args based on the | |
524 | * overlap, and fixes up the key as required. | |
525 | */ | |
afe5fea7 | 526 | static noinline void truncate_one_csum(struct btrfs_root *root, |
143bede5 JM |
527 | struct btrfs_path *path, |
528 | struct btrfs_key *key, | |
529 | u64 bytenr, u64 len) | |
459931ec CM |
530 | { |
531 | struct extent_buffer *leaf; | |
6c41761f | 532 | u16 csum_size = btrfs_super_csum_size(root->fs_info->super_copy); |
459931ec CM |
533 | u64 csum_end; |
534 | u64 end_byte = bytenr + len; | |
535 | u32 blocksize_bits = root->fs_info->sb->s_blocksize_bits; | |
459931ec CM |
536 | |
537 | leaf = path->nodes[0]; | |
538 | csum_end = btrfs_item_size_nr(leaf, path->slots[0]) / csum_size; | |
539 | csum_end <<= root->fs_info->sb->s_blocksize_bits; | |
540 | csum_end += key->offset; | |
541 | ||
542 | if (key->offset < bytenr && csum_end <= end_byte) { | |
543 | /* | |
544 | * [ bytenr - len ] | |
545 | * [ ] | |
546 | * [csum ] | |
547 | * A simple truncate off the end of the item | |
548 | */ | |
549 | u32 new_size = (bytenr - key->offset) >> blocksize_bits; | |
550 | new_size *= csum_size; | |
afe5fea7 | 551 | btrfs_truncate_item(root, path, new_size, 1); |
459931ec CM |
552 | } else if (key->offset >= bytenr && csum_end > end_byte && |
553 | end_byte > key->offset) { | |
554 | /* | |
555 | * [ bytenr - len ] | |
556 | * [ ] | |
557 | * [csum ] | |
558 | * we need to truncate from the beginning of the csum | |
559 | */ | |
560 | u32 new_size = (csum_end - end_byte) >> blocksize_bits; | |
561 | new_size *= csum_size; | |
562 | ||
afe5fea7 | 563 | btrfs_truncate_item(root, path, new_size, 0); |
459931ec CM |
564 | |
565 | key->offset = end_byte; | |
afe5fea7 | 566 | btrfs_set_item_key_safe(root, path, key); |
459931ec CM |
567 | } else { |
568 | BUG(); | |
569 | } | |
459931ec CM |
570 | } |
571 | ||
572 | /* | |
573 | * deletes the csum items from the csum tree for a given | |
574 | * range of bytes. | |
575 | */ | |
576 | int btrfs_del_csums(struct btrfs_trans_handle *trans, | |
577 | struct btrfs_root *root, u64 bytenr, u64 len) | |
578 | { | |
579 | struct btrfs_path *path; | |
580 | struct btrfs_key key; | |
581 | u64 end_byte = bytenr + len; | |
582 | u64 csum_end; | |
583 | struct extent_buffer *leaf; | |
584 | int ret; | |
6c41761f | 585 | u16 csum_size = btrfs_super_csum_size(root->fs_info->super_copy); |
459931ec CM |
586 | int blocksize_bits = root->fs_info->sb->s_blocksize_bits; |
587 | ||
588 | root = root->fs_info->csum_root; | |
589 | ||
590 | path = btrfs_alloc_path(); | |
2a29edc6 | 591 | if (!path) |
592 | return -ENOMEM; | |
459931ec | 593 | |
d397712b | 594 | while (1) { |
459931ec CM |
595 | key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; |
596 | key.offset = end_byte - 1; | |
597 | key.type = BTRFS_EXTENT_CSUM_KEY; | |
598 | ||
b9473439 | 599 | path->leave_spinning = 1; |
459931ec CM |
600 | ret = btrfs_search_slot(trans, root, &key, path, -1, 1); |
601 | if (ret > 0) { | |
602 | if (path->slots[0] == 0) | |
65a246c5 | 603 | break; |
459931ec | 604 | path->slots[0]--; |
ad0397a7 | 605 | } else if (ret < 0) { |
65a246c5 | 606 | break; |
459931ec | 607 | } |
ad0397a7 | 608 | |
459931ec CM |
609 | leaf = path->nodes[0]; |
610 | btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); | |
611 | ||
612 | if (key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || | |
613 | key.type != BTRFS_EXTENT_CSUM_KEY) { | |
614 | break; | |
615 | } | |
616 | ||
617 | if (key.offset >= end_byte) | |
618 | break; | |
619 | ||
620 | csum_end = btrfs_item_size_nr(leaf, path->slots[0]) / csum_size; | |
621 | csum_end <<= blocksize_bits; | |
622 | csum_end += key.offset; | |
623 | ||
624 | /* this csum ends before we start, we're done */ | |
625 | if (csum_end <= bytenr) | |
626 | break; | |
627 | ||
628 | /* delete the entire item, it is inside our range */ | |
629 | if (key.offset >= bytenr && csum_end <= end_byte) { | |
630 | ret = btrfs_del_item(trans, root, path); | |
65a246c5 TI |
631 | if (ret) |
632 | goto out; | |
dcbdd4dc CM |
633 | if (key.offset == bytenr) |
634 | break; | |
459931ec CM |
635 | } else if (key.offset < bytenr && csum_end > end_byte) { |
636 | unsigned long offset; | |
637 | unsigned long shift_len; | |
638 | unsigned long item_offset; | |
639 | /* | |
640 | * [ bytenr - len ] | |
641 | * [csum ] | |
642 | * | |
643 | * Our bytes are in the middle of the csum, | |
644 | * we need to split this item and insert a new one. | |
645 | * | |
646 | * But we can't drop the path because the | |
647 | * csum could change, get removed, extended etc. | |
648 | * | |
649 | * The trick here is the max size of a csum item leaves | |
650 | * enough room in the tree block for a single | |
651 | * item header. So, we split the item in place, | |
652 | * adding a new header pointing to the existing | |
653 | * bytes. Then we loop around again and we have | |
654 | * a nicely formed csum item that we can neatly | |
655 | * truncate. | |
656 | */ | |
657 | offset = (bytenr - key.offset) >> blocksize_bits; | |
658 | offset *= csum_size; | |
659 | ||
660 | shift_len = (len >> blocksize_bits) * csum_size; | |
661 | ||
662 | item_offset = btrfs_item_ptr_offset(leaf, | |
663 | path->slots[0]); | |
664 | ||
665 | memset_extent_buffer(leaf, 0, item_offset + offset, | |
666 | shift_len); | |
667 | key.offset = bytenr; | |
668 | ||
669 | /* | |
670 | * btrfs_split_item returns -EAGAIN when the | |
671 | * item changed size or key | |
672 | */ | |
673 | ret = btrfs_split_item(trans, root, path, &key, offset); | |
79787eaa JM |
674 | if (ret && ret != -EAGAIN) { |
675 | btrfs_abort_transaction(trans, root, ret); | |
676 | goto out; | |
677 | } | |
459931ec CM |
678 | |
679 | key.offset = end_byte - 1; | |
680 | } else { | |
afe5fea7 | 681 | truncate_one_csum(root, path, &key, bytenr, len); |
dcbdd4dc CM |
682 | if (key.offset < bytenr) |
683 | break; | |
459931ec | 684 | } |
b3b4aa74 | 685 | btrfs_release_path(path); |
459931ec | 686 | } |
65a246c5 | 687 | ret = 0; |
459931ec CM |
688 | out: |
689 | btrfs_free_path(path); | |
65a246c5 | 690 | return ret; |
459931ec CM |
691 | } |
692 | ||
065631f6 | 693 | int btrfs_csum_file_blocks(struct btrfs_trans_handle *trans, |
d20f7043 | 694 | struct btrfs_root *root, |
e6dcd2dc | 695 | struct btrfs_ordered_sum *sums) |
f254e52c | 696 | { |
f254e52c | 697 | struct btrfs_key file_key; |
6567e837 | 698 | struct btrfs_key found_key; |
5caf2a00 | 699 | struct btrfs_path *path; |
f254e52c | 700 | struct btrfs_csum_item *item; |
065631f6 | 701 | struct btrfs_csum_item *item_end; |
ff79f819 | 702 | struct extent_buffer *leaf = NULL; |
f51a4a18 MX |
703 | u64 next_offset; |
704 | u64 total_bytes = 0; | |
6567e837 | 705 | u64 csum_offset; |
f51a4a18 | 706 | u64 bytenr; |
f578d4bd CM |
707 | u32 nritems; |
708 | u32 ins_size; | |
f51a4a18 MX |
709 | int index = 0; |
710 | int found_next; | |
711 | int ret; | |
6c41761f | 712 | u16 csum_size = btrfs_super_csum_size(root->fs_info->super_copy); |
6e92f5e6 | 713 | |
5caf2a00 | 714 | path = btrfs_alloc_path(); |
d8926bb3 MF |
715 | if (!path) |
716 | return -ENOMEM; | |
065631f6 CM |
717 | again: |
718 | next_offset = (u64)-1; | |
719 | found_next = 0; | |
f51a4a18 | 720 | bytenr = sums->bytenr + total_bytes; |
d20f7043 | 721 | file_key.objectid = BTRFS_EXTENT_CSUM_OBJECTID; |
f51a4a18 | 722 | file_key.offset = bytenr; |
d20f7043 | 723 | btrfs_set_key_type(&file_key, BTRFS_EXTENT_CSUM_KEY); |
a429e513 | 724 | |
f51a4a18 | 725 | item = btrfs_lookup_csum(trans, root, path, bytenr, 1); |
ff79f819 | 726 | if (!IS_ERR(item)) { |
639cb586 | 727 | ret = 0; |
f51a4a18 MX |
728 | leaf = path->nodes[0]; |
729 | item_end = btrfs_item_ptr(leaf, path->slots[0], | |
730 | struct btrfs_csum_item); | |
731 | item_end = (struct btrfs_csum_item *)((char *)item_end + | |
732 | btrfs_item_size_nr(leaf, path->slots[0])); | |
a429e513 | 733 | goto found; |
ff79f819 | 734 | } |
a429e513 | 735 | ret = PTR_ERR(item); |
4a500fd1 YZ |
736 | if (ret != -EFBIG && ret != -ENOENT) |
737 | goto fail_unlock; | |
738 | ||
a429e513 CM |
739 | if (ret == -EFBIG) { |
740 | u32 item_size; | |
741 | /* we found one, but it isn't big enough yet */ | |
5f39d397 CM |
742 | leaf = path->nodes[0]; |
743 | item_size = btrfs_item_size_nr(leaf, path->slots[0]); | |
607d432d JB |
744 | if ((item_size / csum_size) >= |
745 | MAX_CSUM_ITEMS(root, csum_size)) { | |
a429e513 CM |
746 | /* already at max size, make a new one */ |
747 | goto insert; | |
748 | } | |
749 | } else { | |
f578d4bd | 750 | int slot = path->slots[0] + 1; |
a429e513 | 751 | /* we didn't find a csum item, insert one */ |
f578d4bd CM |
752 | nritems = btrfs_header_nritems(path->nodes[0]); |
753 | if (path->slots[0] >= nritems - 1) { | |
754 | ret = btrfs_next_leaf(root, path); | |
b56baf5b | 755 | if (ret == 1) |
f578d4bd | 756 | found_next = 1; |
b56baf5b | 757 | if (ret != 0) |
f578d4bd | 758 | goto insert; |
b56baf5b | 759 | slot = 0; |
f578d4bd CM |
760 | } |
761 | btrfs_item_key_to_cpu(path->nodes[0], &found_key, slot); | |
d20f7043 CM |
762 | if (found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || |
763 | found_key.type != BTRFS_EXTENT_CSUM_KEY) { | |
f578d4bd CM |
764 | found_next = 1; |
765 | goto insert; | |
766 | } | |
767 | next_offset = found_key.offset; | |
768 | found_next = 1; | |
a429e513 CM |
769 | goto insert; |
770 | } | |
771 | ||
772 | /* | |
773 | * at this point, we know the tree has an item, but it isn't big | |
774 | * enough yet to put our csum in. Grow it | |
775 | */ | |
b3b4aa74 | 776 | btrfs_release_path(path); |
6567e837 | 777 | ret = btrfs_search_slot(trans, root, &file_key, path, |
607d432d | 778 | csum_size, 1); |
6567e837 | 779 | if (ret < 0) |
53863232 | 780 | goto fail_unlock; |
459931ec CM |
781 | |
782 | if (ret > 0) { | |
783 | if (path->slots[0] == 0) | |
784 | goto insert; | |
785 | path->slots[0]--; | |
6567e837 | 786 | } |
459931ec | 787 | |
5f39d397 CM |
788 | leaf = path->nodes[0]; |
789 | btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); | |
d20f7043 | 790 | csum_offset = (bytenr - found_key.offset) >> |
6567e837 | 791 | root->fs_info->sb->s_blocksize_bits; |
459931ec | 792 | |
d20f7043 CM |
793 | if (btrfs_key_type(&found_key) != BTRFS_EXTENT_CSUM_KEY || |
794 | found_key.objectid != BTRFS_EXTENT_CSUM_OBJECTID || | |
607d432d | 795 | csum_offset >= MAX_CSUM_ITEMS(root, csum_size)) { |
6567e837 CM |
796 | goto insert; |
797 | } | |
459931ec | 798 | |
2f697dc6 | 799 | if (csum_offset == btrfs_item_size_nr(leaf, path->slots[0]) / |
607d432d | 800 | csum_size) { |
2f697dc6 LB |
801 | int extend_nr; |
802 | u64 tmp; | |
803 | u32 diff; | |
804 | u32 free_space; | |
459931ec | 805 | |
2f697dc6 LB |
806 | if (btrfs_leaf_free_space(root, leaf) < |
807 | sizeof(struct btrfs_item) + csum_size * 2) | |
808 | goto insert; | |
809 | ||
810 | free_space = btrfs_leaf_free_space(root, leaf) - | |
811 | sizeof(struct btrfs_item) - csum_size; | |
f51a4a18 | 812 | tmp = sums->len - total_bytes; |
2f697dc6 LB |
813 | tmp >>= root->fs_info->sb->s_blocksize_bits; |
814 | WARN_ON(tmp < 1); | |
815 | ||
816 | extend_nr = max_t(int, 1, (int)tmp); | |
817 | diff = (csum_offset + extend_nr) * csum_size; | |
818 | diff = min(diff, MAX_CSUM_ITEMS(root, csum_size) * csum_size); | |
459931ec | 819 | |
5f39d397 | 820 | diff = diff - btrfs_item_size_nr(leaf, path->slots[0]); |
2f697dc6 LB |
821 | diff = min(free_space, diff); |
822 | diff /= csum_size; | |
823 | diff *= csum_size; | |
459931ec | 824 | |
4b90c680 | 825 | btrfs_extend_item(root, path, diff); |
f51a4a18 | 826 | ret = 0; |
6567e837 CM |
827 | goto csum; |
828 | } | |
829 | ||
830 | insert: | |
b3b4aa74 | 831 | btrfs_release_path(path); |
6567e837 | 832 | csum_offset = 0; |
f578d4bd | 833 | if (found_next) { |
2f697dc6 | 834 | u64 tmp; |
d20f7043 | 835 | |
f51a4a18 | 836 | tmp = sums->len - total_bytes; |
f578d4bd | 837 | tmp >>= root->fs_info->sb->s_blocksize_bits; |
2f697dc6 LB |
838 | tmp = min(tmp, (next_offset - file_key.offset) >> |
839 | root->fs_info->sb->s_blocksize_bits); | |
840 | ||
f578d4bd | 841 | tmp = max((u64)1, tmp); |
607d432d JB |
842 | tmp = min(tmp, (u64)MAX_CSUM_ITEMS(root, csum_size)); |
843 | ins_size = csum_size * tmp; | |
f578d4bd | 844 | } else { |
607d432d | 845 | ins_size = csum_size; |
f578d4bd | 846 | } |
b9473439 | 847 | path->leave_spinning = 1; |
5caf2a00 | 848 | ret = btrfs_insert_empty_item(trans, root, path, &file_key, |
f578d4bd | 849 | ins_size); |
b9473439 | 850 | path->leave_spinning = 0; |
54aa1f4d | 851 | if (ret < 0) |
53863232 | 852 | goto fail_unlock; |
fae7f21c | 853 | if (WARN_ON(ret != 0)) |
53863232 | 854 | goto fail_unlock; |
5f39d397 | 855 | leaf = path->nodes[0]; |
f51a4a18 | 856 | csum: |
5f39d397 | 857 | item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_csum_item); |
f51a4a18 MX |
858 | item_end = (struct btrfs_csum_item *)((unsigned char *)item + |
859 | btrfs_item_size_nr(leaf, path->slots[0])); | |
509659cd | 860 | item = (struct btrfs_csum_item *)((unsigned char *)item + |
607d432d | 861 | csum_offset * csum_size); |
b18c6685 | 862 | found: |
f51a4a18 MX |
863 | ins_size = (u32)(sums->len - total_bytes) >> |
864 | root->fs_info->sb->s_blocksize_bits; | |
865 | ins_size *= csum_size; | |
866 | ins_size = min_t(u32, (unsigned long)item_end - (unsigned long)item, | |
867 | ins_size); | |
868 | write_extent_buffer(leaf, sums->sums + index, (unsigned long)item, | |
869 | ins_size); | |
870 | ||
871 | ins_size /= csum_size; | |
872 | total_bytes += ins_size * root->sectorsize; | |
873 | index += ins_size; | |
a6591715 | 874 | |
5caf2a00 | 875 | btrfs_mark_buffer_dirty(path->nodes[0]); |
e6dcd2dc | 876 | if (total_bytes < sums->len) { |
b3b4aa74 | 877 | btrfs_release_path(path); |
b9473439 | 878 | cond_resched(); |
065631f6 CM |
879 | goto again; |
880 | } | |
53863232 | 881 | out: |
5caf2a00 | 882 | btrfs_free_path(path); |
f254e52c | 883 | return ret; |
53863232 CM |
884 | |
885 | fail_unlock: | |
53863232 | 886 | goto out; |
f254e52c | 887 | } |