]> git.proxmox.com Git - mirror_zfs.git/blob - module/zfs/dmu.c
projects / mirror_zfs.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Implement a new type of zfs receive: corrective receive (-c)
[mirror_zfs.git] / module / zfs / dmu.c
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
24 * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
25 * Copyright (c) 2013, Joyent, Inc. All rights reserved.
26 * Copyright (c) 2016, Nexenta Systems, Inc. All rights reserved.
27 * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
28 * Copyright (c) 2019 Datto Inc.
29 * Copyright (c) 2019, Klara Inc.
30 * Copyright (c) 2019, Allan Jude
31 */
32
33 #include <sys/dmu.h>
34 #include <sys/dmu_impl.h>
35 #include <sys/dmu_tx.h>
36 #include <sys/dbuf.h>
37 #include <sys/dnode.h>
38 #include <sys/zfs_context.h>
39 #include <sys/dmu_objset.h>
40 #include <sys/dmu_traverse.h>
41 #include <sys/dsl_dataset.h>
42 #include <sys/dsl_dir.h>
43 #include <sys/dsl_pool.h>
44 #include <sys/dsl_synctask.h>
45 #include <sys/dsl_prop.h>
46 #include <sys/dmu_zfetch.h>
47 #include <sys/zfs_ioctl.h>
48 #include <sys/zap.h>
49 #include <sys/zio_checksum.h>
50 #include <sys/zio_compress.h>
51 #include <sys/sa.h>
52 #include <sys/zfeature.h>
53 #include <sys/abd.h>
54 #include <sys/trace_zfs.h>
55 #include <sys/zfs_racct.h>
56 #include <sys/zfs_rlock.h>
57 #ifdef _KERNEL
58 #include <sys/vmsystm.h>
59 #include <sys/zfs_znode.h>
60 #endif
61
62 /*
63 * Enable/disable nopwrite feature.
64 */
65 static int zfs_nopwrite_enabled = 1;
66
67 /*
68 * Tunable to control percentage of dirtied L1 blocks from frees allowed into
69 * one TXG. After this threshold is crossed, additional dirty blocks from frees
70 * will wait until the next TXG.
71 * A value of zero will disable this throttle.
72 */
73 static unsigned long zfs_per_txg_dirty_frees_percent = 5;
74
75 /*
76 * Enable/disable forcing txg sync when dirty checking for holes with lseek().
77 * By default this is enabled to ensure accurate hole reporting, it can result
78 * in a significant performance penalty for lseek(SEEK_HOLE) heavy workloads.
79 * Disabling this option will result in holes never being reported in dirty
80 * files which is always safe.
81 */
82 static int zfs_dmu_offset_next_sync = 1;
83
84 /*
85 * Limit the amount we can prefetch with one call to this amount. This
86 * helps to limit the amount of memory that can be used by prefetching.
87 * Larger objects should be prefetched a bit at a time.
88 */
89 int dmu_prefetch_max = 8 * SPA_MAXBLOCKSIZE;
90
91 const dmu_object_type_info_t dmu_ot[DMU_OT_NUMTYPES] = {
92 {DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "unallocated" },
93 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "object directory" },
94 {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "object array" },
95 {DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "packed nvlist" },
96 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "packed nvlist size" },
97 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj" },
98 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj header" },
99 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA space map header" },
100 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA space map" },
101 {DMU_BSWAP_UINT64, TRUE, FALSE, TRUE, "ZIL intent log" },
102 {DMU_BSWAP_DNODE, TRUE, FALSE, TRUE, "DMU dnode" },
103 {DMU_BSWAP_OBJSET, TRUE, TRUE, FALSE, "DMU objset" },
104 {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL directory" },
105 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL directory child map"},
106 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dataset snap map" },
107 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL props" },
108 {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL dataset" },
109 {DMU_BSWAP_ZNODE, TRUE, FALSE, FALSE, "ZFS znode" },
110 {DMU_BSWAP_OLDACL, TRUE, FALSE, TRUE, "ZFS V0 ACL" },
111 {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "ZFS plain file" },
112 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS directory" },
113 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "ZFS master node" },
114 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS delete queue" },
115 {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "zvol object" },
116 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "zvol prop" },
117 {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "other uint8[]" },
118 {DMU_BSWAP_UINT64, FALSE, FALSE, TRUE, "other uint64[]" },
119 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "other ZAP" },
120 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "persistent error log" },
121 {DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "SPA history" },
122 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA history offsets" },
123 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "Pool properties" },
124 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL permissions" },
125 {DMU_BSWAP_ACL, TRUE, FALSE, TRUE, "ZFS ACL" },
126 {DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "ZFS SYSACL" },
127 {DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "FUID table" },
128 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "FUID table size" },
129 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dataset next clones"},
130 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "scan work queue" },
131 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS user/group/project used" },
132 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS user/group/project quota"},
133 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "snapshot refcount tags"},
134 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "DDT ZAP algorithm" },
135 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "DDT statistics" },
136 {DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "System attributes" },
137 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA master node" },
138 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA attr registration" },
139 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA attr layouts" },
140 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "scan translations" },
141 {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "deduplicated block" },
142 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL deadlist map" },
143 {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL deadlist map hdr" },
144 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dir clones" },
145 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj subobj" }
146 };
147
148 const dmu_object_byteswap_info_t dmu_ot_byteswap[DMU_BSWAP_NUMFUNCS] = {
149 { byteswap_uint8_array, "uint8" },
150 { byteswap_uint16_array, "uint16" },
151 { byteswap_uint32_array, "uint32" },
152 { byteswap_uint64_array, "uint64" },
153 { zap_byteswap, "zap" },
154 { dnode_buf_byteswap, "dnode" },
155 { dmu_objset_byteswap, "objset" },
156 { zfs_znode_byteswap, "znode" },
157 { zfs_oldacl_byteswap, "oldacl" },
158 { zfs_acl_byteswap, "acl" }
159 };
160
161 static int
162 dmu_buf_hold_noread_by_dnode(dnode_t *dn, uint64_t offset,
163 const void *tag, dmu_buf_t **dbp)
164 {
165 uint64_t blkid;
166 dmu_buf_impl_t *db;
167
168 rw_enter(&dn->dn_struct_rwlock, RW_READER);
169 blkid = dbuf_whichblock(dn, 0, offset);
170 db = dbuf_hold(dn, blkid, tag);
171 rw_exit(&dn->dn_struct_rwlock);
172
173 if (db == NULL) {
174 *dbp = NULL;
175 return (SET_ERROR(EIO));
176 }
177
178 *dbp = &db->db;
179 return (0);
180 }
181 int
182 dmu_buf_hold_noread(objset_t *os, uint64_t object, uint64_t offset,
183 const void *tag, dmu_buf_t **dbp)
184 {
185 dnode_t *dn;
186 uint64_t blkid;
187 dmu_buf_impl_t *db;
188 int err;
189
190 err = dnode_hold(os, object, FTAG, &dn);
191 if (err)
192 return (err);
193 rw_enter(&dn->dn_struct_rwlock, RW_READER);
194 blkid = dbuf_whichblock(dn, 0, offset);
195 db = dbuf_hold(dn, blkid, tag);
196 rw_exit(&dn->dn_struct_rwlock);
197 dnode_rele(dn, FTAG);
198
199 if (db == NULL) {
200 *dbp = NULL;
201 return (SET_ERROR(EIO));
202 }
203
204 *dbp = &db->db;
205 return (err);
206 }
207
208 int
209 dmu_buf_hold_by_dnode(dnode_t *dn, uint64_t offset,
210 const void *tag, dmu_buf_t **dbp, int flags)
211 {
212 int err;
213 int db_flags = DB_RF_CANFAIL;
214
215 if (flags & DMU_READ_NO_PREFETCH)
216 db_flags |= DB_RF_NOPREFETCH;
217 if (flags & DMU_READ_NO_DECRYPT)
218 db_flags |= DB_RF_NO_DECRYPT;
219
220 err = dmu_buf_hold_noread_by_dnode(dn, offset, tag, dbp);
221 if (err == 0) {
222 dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp);
223 err = dbuf_read(db, NULL, db_flags);
224 if (err != 0) {
225 dbuf_rele(db, tag);
226 *dbp = NULL;
227 }
228 }
229
230 return (err);
231 }
232
233 int
234 dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset,
235 const void *tag, dmu_buf_t **dbp, int flags)
236 {
237 int err;
238 int db_flags = DB_RF_CANFAIL;
239
240 if (flags & DMU_READ_NO_PREFETCH)
241 db_flags |= DB_RF_NOPREFETCH;
242 if (flags & DMU_READ_NO_DECRYPT)
243 db_flags |= DB_RF_NO_DECRYPT;
244
245 err = dmu_buf_hold_noread(os, object, offset, tag, dbp);
246 if (err == 0) {
247 dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp);
248 err = dbuf_read(db, NULL, db_flags);
249 if (err != 0) {
250 dbuf_rele(db, tag);
251 *dbp = NULL;
252 }
253 }
254
255 return (err);
256 }
257
258 int
259 dmu_bonus_max(void)
260 {
261 return (DN_OLD_MAX_BONUSLEN);
262 }
263
264 int
265 dmu_set_bonus(dmu_buf_t *db_fake, int newsize, dmu_tx_t *tx)
266 {
267 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
268 dnode_t *dn;
269 int error;
270
271 DB_DNODE_ENTER(db);
272 dn = DB_DNODE(db);
273
274 if (dn->dn_bonus != db) {
275 error = SET_ERROR(EINVAL);
276 } else if (newsize < 0 || newsize > db_fake->db_size) {
277 error = SET_ERROR(EINVAL);
278 } else {
279 dnode_setbonuslen(dn, newsize, tx);
280 error = 0;
281 }
282
283 DB_DNODE_EXIT(db);
284 return (error);
285 }
286
287 int
288 dmu_set_bonustype(dmu_buf_t *db_fake, dmu_object_type_t type, dmu_tx_t *tx)
289 {
290 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
291 dnode_t *dn;
292 int error;
293
294 DB_DNODE_ENTER(db);
295 dn = DB_DNODE(db);
296
297 if (!DMU_OT_IS_VALID(type)) {
298 error = SET_ERROR(EINVAL);
299 } else if (dn->dn_bonus != db) {
300 error = SET_ERROR(EINVAL);
301 } else {
302 dnode_setbonus_type(dn, type, tx);
303 error = 0;
304 }
305
306 DB_DNODE_EXIT(db);
307 return (error);
308 }
309
310 dmu_object_type_t
311 dmu_get_bonustype(dmu_buf_t *db_fake)
312 {
313 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
314 dnode_t *dn;
315 dmu_object_type_t type;
316
317 DB_DNODE_ENTER(db);
318 dn = DB_DNODE(db);
319 type = dn->dn_bonustype;
320 DB_DNODE_EXIT(db);
321
322 return (type);
323 }
324
325 int
326 dmu_rm_spill(objset_t *os, uint64_t object, dmu_tx_t *tx)
327 {
328 dnode_t *dn;
329 int error;
330
331 error = dnode_hold(os, object, FTAG, &dn);
332 dbuf_rm_spill(dn, tx);
333 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
334 dnode_rm_spill(dn, tx);
335 rw_exit(&dn->dn_struct_rwlock);
336 dnode_rele(dn, FTAG);
337 return (error);
338 }
339
340 /*
341 * Lookup and hold the bonus buffer for the provided dnode. If the dnode
342 * has not yet been allocated a new bonus dbuf a will be allocated.
343 * Returns ENOENT, EIO, or 0.
344 */
345 int dmu_bonus_hold_by_dnode(dnode_t *dn, const void *tag, dmu_buf_t **dbp,
346 uint32_t flags)
347 {
348 dmu_buf_impl_t *db;
349 int error;
350 uint32_t db_flags = DB_RF_MUST_SUCCEED;
351
352 if (flags & DMU_READ_NO_PREFETCH)
353 db_flags |= DB_RF_NOPREFETCH;
354 if (flags & DMU_READ_NO_DECRYPT)
355 db_flags |= DB_RF_NO_DECRYPT;
356
357 rw_enter(&dn->dn_struct_rwlock, RW_READER);
358 if (dn->dn_bonus == NULL) {
359 rw_exit(&dn->dn_struct_rwlock);
360 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
361 if (dn->dn_bonus == NULL)
362 dbuf_create_bonus(dn);
363 }
364 db = dn->dn_bonus;
365
366 /* as long as the bonus buf is held, the dnode will be held */
367 if (zfs_refcount_add(&db->db_holds, tag) == 1) {
368 VERIFY(dnode_add_ref(dn, db));
369 atomic_inc_32(&dn->dn_dbufs_count);
370 }
371
372 /*
373 * Wait to drop dn_struct_rwlock until after adding the bonus dbuf's
374 * hold and incrementing the dbuf count to ensure that dnode_move() sees
375 * a dnode hold for every dbuf.
376 */
377 rw_exit(&dn->dn_struct_rwlock);
378
379 error = dbuf_read(db, NULL, db_flags);
380 if (error) {
381 dnode_evict_bonus(dn);
382 dbuf_rele(db, tag);
383 *dbp = NULL;
384 return (error);
385 }
386
387 *dbp = &db->db;
388 return (0);
389 }
390
391 int
392 dmu_bonus_hold(objset_t *os, uint64_t object, const void *tag, dmu_buf_t **dbp)
393 {
394 dnode_t *dn;
395 int error;
396
397 error = dnode_hold(os, object, FTAG, &dn);
398 if (error)
399 return (error);
400
401 error = dmu_bonus_hold_by_dnode(dn, tag, dbp, DMU_READ_NO_PREFETCH);
402 dnode_rele(dn, FTAG);
403
404 return (error);
405 }
406
407 /*
408 * returns ENOENT, EIO, or 0.
409 *
410 * This interface will allocate a blank spill dbuf when a spill blk
411 * doesn't already exist on the dnode.
412 *
413 * if you only want to find an already existing spill db, then
414 * dmu_spill_hold_existing() should be used.
415 */
416 int
417 dmu_spill_hold_by_dnode(dnode_t *dn, uint32_t flags, const void *tag,
418 dmu_buf_t **dbp)
419 {
420 dmu_buf_impl_t *db = NULL;
421 int err;
422
423 if ((flags & DB_RF_HAVESTRUCT) == 0)
424 rw_enter(&dn->dn_struct_rwlock, RW_READER);
425
426 db = dbuf_hold(dn, DMU_SPILL_BLKID, tag);
427
428 if ((flags & DB_RF_HAVESTRUCT) == 0)
429 rw_exit(&dn->dn_struct_rwlock);
430
431 if (db == NULL) {
432 *dbp = NULL;
433 return (SET_ERROR(EIO));
434 }
435 err = dbuf_read(db, NULL, flags);
436 if (err == 0)
437 *dbp = &db->db;
438 else {
439 dbuf_rele(db, tag);
440 *dbp = NULL;
441 }
442 return (err);
443 }
444
445 int
446 dmu_spill_hold_existing(dmu_buf_t *bonus, const void *tag, dmu_buf_t **dbp)
447 {
448 dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus;
449 dnode_t *dn;
450 int err;
451
452 DB_DNODE_ENTER(db);
453 dn = DB_DNODE(db);
454
455 if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_SA) {
456 err = SET_ERROR(EINVAL);
457 } else {
458 rw_enter(&dn->dn_struct_rwlock, RW_READER);
459
460 if (!dn->dn_have_spill) {
461 err = SET_ERROR(ENOENT);
462 } else {
463 err = dmu_spill_hold_by_dnode(dn,
464 DB_RF_HAVESTRUCT | DB_RF_CANFAIL, tag, dbp);
465 }
466
467 rw_exit(&dn->dn_struct_rwlock);
468 }
469
470 DB_DNODE_EXIT(db);
471 return (err);
472 }
473
474 int
475 dmu_spill_hold_by_bonus(dmu_buf_t *bonus, uint32_t flags, const void *tag,
476 dmu_buf_t **dbp)
477 {
478 dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus;
479 dnode_t *dn;
480 int err;
481 uint32_t db_flags = DB_RF_CANFAIL;
482
483 if (flags & DMU_READ_NO_DECRYPT)
484 db_flags |= DB_RF_NO_DECRYPT;
485
486 DB_DNODE_ENTER(db);
487 dn = DB_DNODE(db);
488 err = dmu_spill_hold_by_dnode(dn, db_flags, tag, dbp);
489 DB_DNODE_EXIT(db);
490
491 return (err);
492 }
493
494 /*
495 * Note: longer-term, we should modify all of the dmu_buf_*() interfaces
496 * to take a held dnode rather than <os, object> -- the lookup is wasteful,
497 * and can induce severe lock contention when writing to several files
498 * whose dnodes are in the same block.
499 */
500 int
501 dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
502 boolean_t read, const void *tag, int *numbufsp, dmu_buf_t ***dbpp,
503 uint32_t flags)
504 {
505 dmu_buf_t **dbp;
506 zstream_t *zs = NULL;
507 uint64_t blkid, nblks, i;
508 uint32_t dbuf_flags;
509 int err;
510 zio_t *zio = NULL;
511 boolean_t missed = B_FALSE;
512
513 ASSERT(length <= DMU_MAX_ACCESS);
514
515 /*
516 * Note: We directly notify the prefetch code of this read, so that
517 * we can tell it about the multi-block read. dbuf_read() only knows
518 * about the one block it is accessing.
519 */
520 dbuf_flags = DB_RF_CANFAIL | DB_RF_NEVERWAIT | DB_RF_HAVESTRUCT |
521 DB_RF_NOPREFETCH;
522
523 if ((flags & DMU_READ_NO_DECRYPT) != 0)
524 dbuf_flags |= DB_RF_NO_DECRYPT;
525
526 rw_enter(&dn->dn_struct_rwlock, RW_READER);
527 if (dn->dn_datablkshift) {
528 int blkshift = dn->dn_datablkshift;
529 nblks = (P2ROUNDUP(offset + length, 1ULL << blkshift) -
530 P2ALIGN(offset, 1ULL << blkshift)) >> blkshift;
531 } else {
532 if (offset + length > dn->dn_datablksz) {
533 zfs_panic_recover("zfs: accessing past end of object "
534 "%llx/%llx (size=%u access=%llu+%llu)",
535 (longlong_t)dn->dn_objset->
536 os_dsl_dataset->ds_object,
537 (longlong_t)dn->dn_object, dn->dn_datablksz,
538 (longlong_t)offset, (longlong_t)length);
539 rw_exit(&dn->dn_struct_rwlock);
540 return (SET_ERROR(EIO));
541 }
542 nblks = 1;
543 }
544 dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_SLEEP);
545
546 if (read)
547 zio = zio_root(dn->dn_objset->os_spa, NULL, NULL,
548 ZIO_FLAG_CANFAIL);
549 blkid = dbuf_whichblock(dn, 0, offset);
550 if ((flags & DMU_READ_NO_PREFETCH) == 0 &&
551 DNODE_META_IS_CACHEABLE(dn) && length <= zfetch_array_rd_sz) {
552 /*
553 * Prepare the zfetch before initiating the demand reads, so
554 * that if multiple threads block on same indirect block, we
555 * base predictions on the original less racy request order.
556 */
557 zs = dmu_zfetch_prepare(&dn->dn_zfetch, blkid, nblks,
558 read && DNODE_IS_CACHEABLE(dn), B_TRUE);
559 }
560 for (i = 0; i < nblks; i++) {
561 dmu_buf_impl_t *db = dbuf_hold(dn, blkid + i, tag);
562 if (db == NULL) {
563 if (zs)
564 dmu_zfetch_run(zs, missed, B_TRUE);
565 rw_exit(&dn->dn_struct_rwlock);
566 dmu_buf_rele_array(dbp, nblks, tag);
567 if (read)
568 zio_nowait(zio);
569 return (SET_ERROR(EIO));
570 }
571
572 /*
573 * Initiate async demand data read.
574 * We check the db_state after calling dbuf_read() because
575 * (1) dbuf_read() may change the state to CACHED due to a
576 * hit in the ARC, and (2) on a cache miss, a child will
577 * have been added to "zio" but not yet completed, so the
578 * state will not yet be CACHED.
579 */
580 if (read) {
581 (void) dbuf_read(db, zio, dbuf_flags);
582 if (db->db_state != DB_CACHED)
583 missed = B_TRUE;
584 }
585 dbp[i] = &db->db;
586 }
587
588 if (!read)
589 zfs_racct_write(length, nblks);
590
591 if (zs)
592 dmu_zfetch_run(zs, missed, B_TRUE);
593 rw_exit(&dn->dn_struct_rwlock);
594
595 if (read) {
596 /* wait for async read i/o */
597 err = zio_wait(zio);
598 if (err) {
599 dmu_buf_rele_array(dbp, nblks, tag);
600 return (err);
601 }
602
603 /* wait for other io to complete */
604 for (i = 0; i < nblks; i++) {
605 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp[i];
606 mutex_enter(&db->db_mtx);
607 while (db->db_state == DB_READ ||
608 db->db_state == DB_FILL)
609 cv_wait(&db->db_changed, &db->db_mtx);
610 if (db->db_state == DB_UNCACHED)
611 err = SET_ERROR(EIO);
612 mutex_exit(&db->db_mtx);
613 if (err) {
614 dmu_buf_rele_array(dbp, nblks, tag);
615 return (err);
616 }
617 }
618 }
619
620 *numbufsp = nblks;
621 *dbpp = dbp;
622 return (0);
623 }
624
625 int
626 dmu_buf_hold_array(objset_t *os, uint64_t object, uint64_t offset,
627 uint64_t length, int read, const void *tag, int *numbufsp,
628 dmu_buf_t ***dbpp)
629 {
630 dnode_t *dn;
631 int err;
632
633 err = dnode_hold(os, object, FTAG, &dn);
634 if (err)
635 return (err);
636
637 err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
638 numbufsp, dbpp, DMU_READ_PREFETCH);
639
640 dnode_rele(dn, FTAG);
641
642 return (err);
643 }
644
645 int
646 dmu_buf_hold_array_by_bonus(dmu_buf_t *db_fake, uint64_t offset,
647 uint64_t length, boolean_t read, const void *tag, int *numbufsp,
648 dmu_buf_t ***dbpp)
649 {
650 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
651 dnode_t *dn;
652 int err;
653
654 DB_DNODE_ENTER(db);
655 dn = DB_DNODE(db);
656 err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
657 numbufsp, dbpp, DMU_READ_PREFETCH);
658 DB_DNODE_EXIT(db);
659
660 return (err);
661 }
662
663 void
664 dmu_buf_rele_array(dmu_buf_t **dbp_fake, int numbufs, const void *tag)
665 {
666 int i;
667 dmu_buf_impl_t **dbp = (dmu_buf_impl_t **)dbp_fake;
668
669 if (numbufs == 0)
670 return;
671
672 for (i = 0; i < numbufs; i++) {
673 if (dbp[i])
674 dbuf_rele(dbp[i], tag);
675 }
676
677 kmem_free(dbp, sizeof (dmu_buf_t *) * numbufs);
678 }
679
680 /*
681 * Issue prefetch i/os for the given blocks. If level is greater than 0, the
682 * indirect blocks prefetched will be those that point to the blocks containing
683 * the data starting at offset, and continuing to offset + len.
684 *
685 * Note that if the indirect blocks above the blocks being prefetched are not
686 * in cache, they will be asynchronously read in.
687 */
688 void
689 dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
690 uint64_t len, zio_priority_t pri)
691 {
692 dnode_t *dn;
693 uint64_t blkid;
694 int nblks, err;
695
696 if (len == 0) { /* they're interested in the bonus buffer */
697 dn = DMU_META_DNODE(os);
698
699 if (object == 0 || object >= DN_MAX_OBJECT)
700 return;
701
702 rw_enter(&dn->dn_struct_rwlock, RW_READER);
703 blkid = dbuf_whichblock(dn, level,
704 object * sizeof (dnode_phys_t));
705 dbuf_prefetch(dn, level, blkid, pri, 0);
706 rw_exit(&dn->dn_struct_rwlock);
707 return;
708 }
709
710 /*
711 * See comment before the definition of dmu_prefetch_max.
712 */
713 len = MIN(len, dmu_prefetch_max);
714
715 /*
716 * XXX - Note, if the dnode for the requested object is not
717 * already cached, we will do a *synchronous* read in the
718 * dnode_hold() call. The same is true for any indirects.
719 */
720 err = dnode_hold(os, object, FTAG, &dn);
721 if (err != 0)
722 return;
723
724 /*
725 * offset + len - 1 is the last byte we want to prefetch for, and offset
726 * is the first. Then dbuf_whichblk(dn, level, off + len - 1) is the
727 * last block we want to prefetch, and dbuf_whichblock(dn, level,
728 * offset) is the first. Then the number we need to prefetch is the
729 * last - first + 1.
730 */
731 rw_enter(&dn->dn_struct_rwlock, RW_READER);
732 if (level > 0 || dn->dn_datablkshift != 0) {
733 nblks = dbuf_whichblock(dn, level, offset + len - 1) -
734 dbuf_whichblock(dn, level, offset) + 1;
735 } else {
736 nblks = (offset < dn->dn_datablksz);
737 }
738
739 if (nblks != 0) {
740 blkid = dbuf_whichblock(dn, level, offset);
741 for (int i = 0; i < nblks; i++)
742 dbuf_prefetch(dn, level, blkid + i, pri, 0);
743 }
744 rw_exit(&dn->dn_struct_rwlock);
745
746 dnode_rele(dn, FTAG);
747 }
748
749 /*
750 * Get the next "chunk" of file data to free. We traverse the file from
751 * the end so that the file gets shorter over time (if we crashes in the
752 * middle, this will leave us in a better state). We find allocated file
753 * data by simply searching the allocated level 1 indirects.
754 *
755 * On input, *start should be the first offset that does not need to be
756 * freed (e.g. "offset + length"). On return, *start will be the first
757 * offset that should be freed and l1blks is set to the number of level 1
758 * indirect blocks found within the chunk.
759 */
760 static int
761 get_next_chunk(dnode_t *dn, uint64_t *start, uint64_t minimum, uint64_t *l1blks)
762 {
763 uint64_t blks;
764 uint64_t maxblks = DMU_MAX_ACCESS >> (dn->dn_indblkshift + 1);
765 /* bytes of data covered by a level-1 indirect block */
766 uint64_t iblkrange = (uint64_t)dn->dn_datablksz *
767 EPB(dn->dn_indblkshift, SPA_BLKPTRSHIFT);
768
769 ASSERT3U(minimum, <=, *start);
770
771 /*
772 * Check if we can free the entire range assuming that all of the
773 * L1 blocks in this range have data. If we can, we use this
774 * worst case value as an estimate so we can avoid having to look
775 * at the object's actual data.
776 */
777 uint64_t total_l1blks =
778 (roundup(*start, iblkrange) - (minimum / iblkrange * iblkrange)) /
779 iblkrange;
780 if (total_l1blks <= maxblks) {
781 *l1blks = total_l1blks;
782 *start = minimum;
783 return (0);
784 }
785 ASSERT(ISP2(iblkrange));
786
787 for (blks = 0; *start > minimum && blks < maxblks; blks++) {
788 int err;
789
790 /*
791 * dnode_next_offset(BACKWARDS) will find an allocated L1
792 * indirect block at or before the input offset. We must
793 * decrement *start so that it is at the end of the region
794 * to search.
795 */
796 (*start)--;
797
798 err = dnode_next_offset(dn,
799 DNODE_FIND_BACKWARDS, start, 2, 1, 0);
800
801 /* if there are no indirect blocks before start, we are done */
802 if (err == ESRCH) {
803 *start = minimum;
804 break;
805 } else if (err != 0) {
806 *l1blks = blks;
807 return (err);
808 }
809
810 /* set start to the beginning of this L1 indirect */
811 *start = P2ALIGN(*start, iblkrange);
812 }
813 if (*start < minimum)
814 *start = minimum;
815 *l1blks = blks;
816
817 return (0);
818 }
819
820 /*
821 * If this objset is of type OST_ZFS return true if vfs's unmounted flag is set,
822 * otherwise return false.
823 * Used below in dmu_free_long_range_impl() to enable abort when unmounting
824 */
825 static boolean_t
826 dmu_objset_zfs_unmounting(objset_t *os)
827 {
828 #ifdef _KERNEL
829 if (dmu_objset_type(os) == DMU_OST_ZFS)
830 return (zfs_get_vfs_flag_unmounted(os));
831 #else
832 (void) os;
833 #endif
834 return (B_FALSE);
835 }
836
837 static int
838 dmu_free_long_range_impl(objset_t *os, dnode_t *dn, uint64_t offset,
839 uint64_t length)
840 {
841 uint64_t object_size;
842 int err;
843 uint64_t dirty_frees_threshold;
844 dsl_pool_t *dp = dmu_objset_pool(os);
845
846 if (dn == NULL)
847 return (SET_ERROR(EINVAL));
848
849 object_size = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
850 if (offset >= object_size)
851 return (0);
852
853 if (zfs_per_txg_dirty_frees_percent <= 100)
854 dirty_frees_threshold =
855 zfs_per_txg_dirty_frees_percent * zfs_dirty_data_max / 100;
856 else
857 dirty_frees_threshold = zfs_dirty_data_max / 20;
858
859 if (length == DMU_OBJECT_END || offset + length > object_size)
860 length = object_size - offset;
861
862 while (length != 0) {
863 uint64_t chunk_end, chunk_begin, chunk_len;
864 uint64_t l1blks;
865 dmu_tx_t *tx;
866
867 if (dmu_objset_zfs_unmounting(dn->dn_objset))
868 return (SET_ERROR(EINTR));
869
870 chunk_end = chunk_begin = offset + length;
871
872 /* move chunk_begin backwards to the beginning of this chunk */
873 err = get_next_chunk(dn, &chunk_begin, offset, &l1blks);
874 if (err)
875 return (err);
876 ASSERT3U(chunk_begin, >=, offset);
877 ASSERT3U(chunk_begin, <=, chunk_end);
878
879 chunk_len = chunk_end - chunk_begin;
880
881 tx = dmu_tx_create(os);
882 dmu_tx_hold_free(tx, dn->dn_object, chunk_begin, chunk_len);
883
884 /*
885 * Mark this transaction as typically resulting in a net
886 * reduction in space used.
887 */
888 dmu_tx_mark_netfree(tx);
889 err = dmu_tx_assign(tx, TXG_WAIT);
890 if (err) {
891 dmu_tx_abort(tx);
892 return (err);
893 }
894
895 uint64_t txg = dmu_tx_get_txg(tx);
896
897 mutex_enter(&dp->dp_lock);
898 uint64_t long_free_dirty =
899 dp->dp_long_free_dirty_pertxg[txg & TXG_MASK];
900 mutex_exit(&dp->dp_lock);
901
902 /*
903 * To avoid filling up a TXG with just frees, wait for
904 * the next TXG to open before freeing more chunks if
905 * we have reached the threshold of frees.
906 */
907 if (dirty_frees_threshold != 0 &&
908 long_free_dirty >= dirty_frees_threshold) {
909 DMU_TX_STAT_BUMP(dmu_tx_dirty_frees_delay);
910 dmu_tx_commit(tx);
911 txg_wait_open(dp, 0, B_TRUE);
912 continue;
913 }
914
915 /*
916 * In order to prevent unnecessary write throttling, for each
917 * TXG, we track the cumulative size of L1 blocks being dirtied
918 * in dnode_free_range() below. We compare this number to a
919 * tunable threshold, past which we prevent new L1 dirty freeing
920 * blocks from being added into the open TXG. See
921 * dmu_free_long_range_impl() for details. The threshold
922 * prevents write throttle activation due to dirty freeing L1
923 * blocks taking up a large percentage of zfs_dirty_data_max.
924 */
925 mutex_enter(&dp->dp_lock);
926 dp->dp_long_free_dirty_pertxg[txg & TXG_MASK] +=
927 l1blks << dn->dn_indblkshift;
928 mutex_exit(&dp->dp_lock);
929 DTRACE_PROBE3(free__long__range,
930 uint64_t, long_free_dirty, uint64_t, chunk_len,
931 uint64_t, txg);
932 dnode_free_range(dn, chunk_begin, chunk_len, tx);
933
934 dmu_tx_commit(tx);
935
936 length -= chunk_len;
937 }
938 return (0);
939 }
940
941 int
942 dmu_free_long_range(objset_t *os, uint64_t object,
943 uint64_t offset, uint64_t length)
944 {
945 dnode_t *dn;
946 int err;
947
948 err = dnode_hold(os, object, FTAG, &dn);
949 if (err != 0)
950 return (err);
951 err = dmu_free_long_range_impl(os, dn, offset, length);
952
953 /*
954 * It is important to zero out the maxblkid when freeing the entire
955 * file, so that (a) subsequent calls to dmu_free_long_range_impl()
956 * will take the fast path, and (b) dnode_reallocate() can verify
957 * that the entire file has been freed.
958 */
959 if (err == 0 && offset == 0 && length == DMU_OBJECT_END)
960 dn->dn_maxblkid = 0;
961
962 dnode_rele(dn, FTAG);
963 return (err);
964 }
965
966 int
967 dmu_free_long_object(objset_t *os, uint64_t object)
968 {
969 dmu_tx_t *tx;
970 int err;
971
972 err = dmu_free_long_range(os, object, 0, DMU_OBJECT_END);
973 if (err != 0)
974 return (err);
975
976 tx = dmu_tx_create(os);
977 dmu_tx_hold_bonus(tx, object);
978 dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END);
979 dmu_tx_mark_netfree(tx);
980 err = dmu_tx_assign(tx, TXG_WAIT);
981 if (err == 0) {
982 err = dmu_object_free(os, object, tx);
983 dmu_tx_commit(tx);
984 } else {
985 dmu_tx_abort(tx);
986 }
987
988 return (err);
989 }
990
991 int
992 dmu_free_range(objset_t *os, uint64_t object, uint64_t offset,
993 uint64_t size, dmu_tx_t *tx)
994 {
995 dnode_t *dn;
996 int err = dnode_hold(os, object, FTAG, &dn);
997 if (err)
998 return (err);
999 ASSERT(offset < UINT64_MAX);
1000 ASSERT(size == DMU_OBJECT_END || size <= UINT64_MAX - offset);
1001 dnode_free_range(dn, offset, size, tx);
1002 dnode_rele(dn, FTAG);
1003 return (0);
1004 }
1005
1006 static int
1007 dmu_read_impl(dnode_t *dn, uint64_t offset, uint64_t size,
1008 void *buf, uint32_t flags)
1009 {
1010 dmu_buf_t **dbp;
1011 int numbufs, err = 0;
1012
1013 /*
1014 * Deal with odd block sizes, where there can't be data past the first
1015 * block. If we ever do the tail block optimization, we will need to
1016 * handle that here as well.
1017 */
1018 if (dn->dn_maxblkid == 0) {
1019 uint64_t newsz = offset > dn->dn_datablksz ? 0 :
1020 MIN(size, dn->dn_datablksz - offset);
1021 memset((char *)buf + newsz, 0, size - newsz);
1022 size = newsz;
1023 }
1024
1025 while (size > 0) {
1026 uint64_t mylen = MIN(size, DMU_MAX_ACCESS / 2);
1027 int i;
1028
1029 /*
1030 * NB: we could do this block-at-a-time, but it's nice
1031 * to be reading in parallel.
1032 */
1033 err = dmu_buf_hold_array_by_dnode(dn, offset, mylen,
1034 TRUE, FTAG, &numbufs, &dbp, flags);
1035 if (err)
1036 break;
1037
1038 for (i = 0; i < numbufs; i++) {
1039 uint64_t tocpy;
1040 int64_t bufoff;
1041 dmu_buf_t *db = dbp[i];
1042
1043 ASSERT(size > 0);
1044
1045 bufoff = offset - db->db_offset;
1046 tocpy = MIN(db->db_size - bufoff, size);
1047
1048 (void) memcpy(buf, (char *)db->db_data + bufoff, tocpy);
1049
1050 offset += tocpy;
1051 size -= tocpy;
1052 buf = (char *)buf + tocpy;
1053 }
1054 dmu_buf_rele_array(dbp, numbufs, FTAG);
1055 }
1056 return (err);
1057 }
1058
1059 int
1060 dmu_read(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
1061 void *buf, uint32_t flags)
1062 {
1063 dnode_t *dn;
1064 int err;
1065
1066 err = dnode_hold(os, object, FTAG, &dn);
1067 if (err != 0)
1068 return (err);
1069
1070 err = dmu_read_impl(dn, offset, size, buf, flags);
1071 dnode_rele(dn, FTAG);
1072 return (err);
1073 }
1074
1075 int
1076 dmu_read_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size, void *buf,
1077 uint32_t flags)
1078 {
1079 return (dmu_read_impl(dn, offset, size, buf, flags));
1080 }
1081
1082 static void
1083 dmu_write_impl(dmu_buf_t **dbp, int numbufs, uint64_t offset, uint64_t size,
1084 const void *buf, dmu_tx_t *tx)
1085 {
1086 int i;
1087
1088 for (i = 0; i < numbufs; i++) {
1089 uint64_t tocpy;
1090 int64_t bufoff;
1091 dmu_buf_t *db = dbp[i];
1092
1093 ASSERT(size > 0);
1094
1095 bufoff = offset - db->db_offset;
1096 tocpy = MIN(db->db_size - bufoff, size);
1097
1098 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
1099
1100 if (tocpy == db->db_size)
1101 dmu_buf_will_fill(db, tx);
1102 else
1103 dmu_buf_will_dirty(db, tx);
1104
1105 (void) memcpy((char *)db->db_data + bufoff, buf, tocpy);
1106
1107 if (tocpy == db->db_size)
1108 dmu_buf_fill_done(db, tx);
1109
1110 offset += tocpy;
1111 size -= tocpy;
1112 buf = (char *)buf + tocpy;
1113 }
1114 }
1115
1116 void
1117 dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
1118 const void *buf, dmu_tx_t *tx)
1119 {
1120 dmu_buf_t **dbp;
1121 int numbufs;
1122
1123 if (size == 0)
1124 return;
1125
1126 VERIFY0(dmu_buf_hold_array(os, object, offset, size,
1127 FALSE, FTAG, &numbufs, &dbp));
1128 dmu_write_impl(dbp, numbufs, offset, size, buf, tx);
1129 dmu_buf_rele_array(dbp, numbufs, FTAG);
1130 }
1131
1132 /*
1133 * Note: Lustre is an external consumer of this interface.
1134 */
1135 void
1136 dmu_write_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size,
1137 const void *buf, dmu_tx_t *tx)
1138 {
1139 dmu_buf_t **dbp;
1140 int numbufs;
1141
1142 if (size == 0)
1143 return;
1144
1145 VERIFY0(dmu_buf_hold_array_by_dnode(dn, offset, size,
1146 FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH));
1147 dmu_write_impl(dbp, numbufs, offset, size, buf, tx);
1148 dmu_buf_rele_array(dbp, numbufs, FTAG);
1149 }
1150
1151 void
1152 dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
1153 dmu_tx_t *tx)
1154 {
1155 dmu_buf_t **dbp;
1156 int numbufs, i;
1157
1158 if (size == 0)
1159 return;
1160
1161 VERIFY(0 == dmu_buf_hold_array(os, object, offset, size,
1162 FALSE, FTAG, &numbufs, &dbp));
1163
1164 for (i = 0; i < numbufs; i++) {
1165 dmu_buf_t *db = dbp[i];
1166
1167 dmu_buf_will_not_fill(db, tx);
1168 }
1169 dmu_buf_rele_array(dbp, numbufs, FTAG);
1170 }
1171
1172 void
1173 dmu_write_embedded(objset_t *os, uint64_t object, uint64_t offset,
1174 void *data, uint8_t etype, uint8_t comp, int uncompressed_size,
1175 int compressed_size, int byteorder, dmu_tx_t *tx)
1176 {
1177 dmu_buf_t *db;
1178
1179 ASSERT3U(etype, <, NUM_BP_EMBEDDED_TYPES);
1180 ASSERT3U(comp, <, ZIO_COMPRESS_FUNCTIONS);
1181 VERIFY0(dmu_buf_hold_noread(os, object, offset,
1182 FTAG, &db));
1183
1184 dmu_buf_write_embedded(db,
1185 data, (bp_embedded_type_t)etype, (enum zio_compress)comp,
1186 uncompressed_size, compressed_size, byteorder, tx);
1187
1188 dmu_buf_rele(db, FTAG);
1189 }
1190
1191 void
1192 dmu_redact(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
1193 dmu_tx_t *tx)
1194 {
1195 int numbufs, i;
1196 dmu_buf_t **dbp;
1197
1198 VERIFY0(dmu_buf_hold_array(os, object, offset, size, FALSE, FTAG,
1199 &numbufs, &dbp));
1200 for (i = 0; i < numbufs; i++)
1201 dmu_buf_redact(dbp[i], tx);
1202 dmu_buf_rele_array(dbp, numbufs, FTAG);
1203 }
1204
1205 #ifdef _KERNEL
1206 int
1207 dmu_read_uio_dnode(dnode_t *dn, zfs_uio_t *uio, uint64_t size)
1208 {
1209 dmu_buf_t **dbp;
1210 int numbufs, i, err;
1211
1212 /*
1213 * NB: we could do this block-at-a-time, but it's nice
1214 * to be reading in parallel.
1215 */
1216 err = dmu_buf_hold_array_by_dnode(dn, zfs_uio_offset(uio), size,
1217 TRUE, FTAG, &numbufs, &dbp, 0);
1218 if (err)
1219 return (err);
1220
1221 for (i = 0; i < numbufs; i++) {
1222 uint64_t tocpy;
1223 int64_t bufoff;
1224 dmu_buf_t *db = dbp[i];
1225
1226 ASSERT(size > 0);
1227
1228 bufoff = zfs_uio_offset(uio) - db->db_offset;
1229 tocpy = MIN(db->db_size - bufoff, size);
1230
1231 err = zfs_uio_fault_move((char *)db->db_data + bufoff, tocpy,
1232 UIO_READ, uio);
1233
1234 if (err)
1235 break;
1236
1237 size -= tocpy;
1238 }
1239 dmu_buf_rele_array(dbp, numbufs, FTAG);
1240
1241 return (err);
1242 }
1243
1244 /*
1245 * Read 'size' bytes into the uio buffer.
1246 * From object zdb->db_object.
1247 * Starting at zfs_uio_offset(uio).
1248 *
1249 * If the caller already has a dbuf in the target object
1250 * (e.g. its bonus buffer), this routine is faster than dmu_read_uio(),
1251 * because we don't have to find the dnode_t for the object.
1252 */
1253 int
1254 dmu_read_uio_dbuf(dmu_buf_t *zdb, zfs_uio_t *uio, uint64_t size)
1255 {
1256 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb;
1257 dnode_t *dn;
1258 int err;
1259
1260 if (size == 0)
1261 return (0);
1262
1263 DB_DNODE_ENTER(db);
1264 dn = DB_DNODE(db);
1265 err = dmu_read_uio_dnode(dn, uio, size);
1266 DB_DNODE_EXIT(db);
1267
1268 return (err);
1269 }
1270
1271 /*
1272 * Read 'size' bytes into the uio buffer.
1273 * From the specified object
1274 * Starting at offset zfs_uio_offset(uio).
1275 */
1276 int
1277 dmu_read_uio(objset_t *os, uint64_t object, zfs_uio_t *uio, uint64_t size)
1278 {
1279 dnode_t *dn;
1280 int err;
1281
1282 if (size == 0)
1283 return (0);
1284
1285 err = dnode_hold(os, object, FTAG, &dn);
1286 if (err)
1287 return (err);
1288
1289 err = dmu_read_uio_dnode(dn, uio, size);
1290
1291 dnode_rele(dn, FTAG);
1292
1293 return (err);
1294 }
1295
1296 int
1297 dmu_write_uio_dnode(dnode_t *dn, zfs_uio_t *uio, uint64_t size, dmu_tx_t *tx)
1298 {
1299 dmu_buf_t **dbp;
1300 int numbufs;
1301 int err = 0;
1302 int i;
1303
1304 err = dmu_buf_hold_array_by_dnode(dn, zfs_uio_offset(uio), size,
1305 FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH);
1306 if (err)
1307 return (err);
1308
1309 for (i = 0; i < numbufs; i++) {
1310 uint64_t tocpy;
1311 int64_t bufoff;
1312 dmu_buf_t *db = dbp[i];
1313
1314 ASSERT(size > 0);
1315
1316 bufoff = zfs_uio_offset(uio) - db->db_offset;
1317 tocpy = MIN(db->db_size - bufoff, size);
1318
1319 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
1320
1321 if (tocpy == db->db_size)
1322 dmu_buf_will_fill(db, tx);
1323 else
1324 dmu_buf_will_dirty(db, tx);
1325
1326 /*
1327 * XXX zfs_uiomove could block forever (eg.nfs-backed
1328 * pages). There needs to be a uiolockdown() function
1329 * to lock the pages in memory, so that zfs_uiomove won't
1330 * block.
1331 */
1332 err = zfs_uio_fault_move((char *)db->db_data + bufoff,
1333 tocpy, UIO_WRITE, uio);
1334
1335 if (tocpy == db->db_size)
1336 dmu_buf_fill_done(db, tx);
1337
1338 if (err)
1339 break;
1340
1341 size -= tocpy;
1342 }
1343
1344 dmu_buf_rele_array(dbp, numbufs, FTAG);
1345 return (err);
1346 }
1347
1348 /*
1349 * Write 'size' bytes from the uio buffer.
1350 * To object zdb->db_object.
1351 * Starting at offset zfs_uio_offset(uio).
1352 *
1353 * If the caller already has a dbuf in the target object
1354 * (e.g. its bonus buffer), this routine is faster than dmu_write_uio(),
1355 * because we don't have to find the dnode_t for the object.
1356 */
1357 int
1358 dmu_write_uio_dbuf(dmu_buf_t *zdb, zfs_uio_t *uio, uint64_t size,
1359 dmu_tx_t *tx)
1360 {
1361 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb;
1362 dnode_t *dn;
1363 int err;
1364
1365 if (size == 0)
1366 return (0);
1367
1368 DB_DNODE_ENTER(db);
1369 dn = DB_DNODE(db);
1370 err = dmu_write_uio_dnode(dn, uio, size, tx);
1371 DB_DNODE_EXIT(db);
1372
1373 return (err);
1374 }
1375
1376 /*
1377 * Write 'size' bytes from the uio buffer.
1378 * To the specified object.
1379 * Starting at offset zfs_uio_offset(uio).
1380 */
1381 int
1382 dmu_write_uio(objset_t *os, uint64_t object, zfs_uio_t *uio, uint64_t size,
1383 dmu_tx_t *tx)
1384 {
1385 dnode_t *dn;
1386 int err;
1387
1388 if (size == 0)
1389 return (0);
1390
1391 err = dnode_hold(os, object, FTAG, &dn);
1392 if (err)
1393 return (err);
1394
1395 err = dmu_write_uio_dnode(dn, uio, size, tx);
1396
1397 dnode_rele(dn, FTAG);
1398
1399 return (err);
1400 }
1401 #endif /* _KERNEL */
1402
1403 /*
1404 * Allocate a loaned anonymous arc buffer.
1405 */
1406 arc_buf_t *
1407 dmu_request_arcbuf(dmu_buf_t *handle, int size)
1408 {
1409 dmu_buf_impl_t *db = (dmu_buf_impl_t *)handle;
1410
1411 return (arc_loan_buf(db->db_objset->os_spa, B_FALSE, size));
1412 }
1413
1414 /*
1415 * Free a loaned arc buffer.
1416 */
1417 void
1418 dmu_return_arcbuf(arc_buf_t *buf)
1419 {
1420 arc_return_buf(buf, FTAG);
1421 arc_buf_destroy(buf, FTAG);
1422 }
1423
1424 /*
1425 * A "lightweight" write is faster than a regular write (e.g.
1426 * dmu_write_by_dnode() or dmu_assign_arcbuf_by_dnode()), because it avoids the
1427 * CPU cost of creating a dmu_buf_impl_t and arc_buf_[hdr_]_t. However, the
1428 * data can not be read or overwritten until the transaction's txg has been
1429 * synced. This makes it appropriate for workloads that are known to be
1430 * (temporarily) write-only, like "zfs receive".
1431 *
1432 * A single block is written, starting at the specified offset in bytes. If
1433 * the call is successful, it returns 0 and the provided abd has been
1434 * consumed (the caller should not free it).
1435 */
1436 int
1437 dmu_lightweight_write_by_dnode(dnode_t *dn, uint64_t offset, abd_t *abd,
1438 const zio_prop_t *zp, enum zio_flag flags, dmu_tx_t *tx)
1439 {
1440 dbuf_dirty_record_t *dr =
1441 dbuf_dirty_lightweight(dn, dbuf_whichblock(dn, 0, offset), tx);
1442 if (dr == NULL)
1443 return (SET_ERROR(EIO));
1444 dr->dt.dll.dr_abd = abd;
1445 dr->dt.dll.dr_props = *zp;
1446 dr->dt.dll.dr_flags = flags;
1447 return (0);
1448 }
1449
1450 /*
1451 * When possible directly assign passed loaned arc buffer to a dbuf.
1452 * If this is not possible copy the contents of passed arc buf via
1453 * dmu_write().
1454 */
1455 int
1456 dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset, arc_buf_t *buf,
1457 dmu_tx_t *tx)
1458 {
1459 dmu_buf_impl_t *db;
1460 objset_t *os = dn->dn_objset;
1461 uint64_t object = dn->dn_object;
1462 uint32_t blksz = (uint32_t)arc_buf_lsize(buf);
1463 uint64_t blkid;
1464
1465 rw_enter(&dn->dn_struct_rwlock, RW_READER);
1466 blkid = dbuf_whichblock(dn, 0, offset);
1467 db = dbuf_hold(dn, blkid, FTAG);
1468 if (db == NULL)
1469 return (SET_ERROR(EIO));
1470 rw_exit(&dn->dn_struct_rwlock);
1471
1472 /*
1473 * We can only assign if the offset is aligned and the arc buf is the
1474 * same size as the dbuf.
1475 */
1476 if (offset == db->db.db_offset && blksz == db->db.db_size) {
1477 zfs_racct_write(blksz, 1);
1478 dbuf_assign_arcbuf(db, buf, tx);
1479 dbuf_rele(db, FTAG);
1480 } else {
1481 /* compressed bufs must always be assignable to their dbuf */
1482 ASSERT3U(arc_get_compression(buf), ==, ZIO_COMPRESS_OFF);
1483 ASSERT(!(buf->b_flags & ARC_BUF_FLAG_COMPRESSED));
1484
1485 dbuf_rele(db, FTAG);
1486 dmu_write(os, object, offset, blksz, buf->b_data, tx);
1487 dmu_return_arcbuf(buf);
1488 }
1489
1490 return (0);
1491 }
1492
1493 int
1494 dmu_assign_arcbuf_by_dbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf,
1495 dmu_tx_t *tx)
1496 {
1497 int err;
1498 dmu_buf_impl_t *dbuf = (dmu_buf_impl_t *)handle;
1499
1500 DB_DNODE_ENTER(dbuf);
1501 err = dmu_assign_arcbuf_by_dnode(DB_DNODE(dbuf), offset, buf, tx);
1502 DB_DNODE_EXIT(dbuf);
1503
1504 return (err);
1505 }
1506
1507 typedef struct {
1508 dbuf_dirty_record_t *dsa_dr;
1509 dmu_sync_cb_t *dsa_done;
1510 zgd_t *dsa_zgd;
1511 dmu_tx_t *dsa_tx;
1512 } dmu_sync_arg_t;
1513
1514 static void
1515 dmu_sync_ready(zio_t *zio, arc_buf_t *buf, void *varg)
1516 {
1517 (void) buf;
1518 dmu_sync_arg_t *dsa = varg;
1519 dmu_buf_t *db = dsa->dsa_zgd->zgd_db;
1520 blkptr_t *bp = zio->io_bp;
1521
1522 if (zio->io_error == 0) {
1523 if (BP_IS_HOLE(bp)) {
1524 /*
1525 * A block of zeros may compress to a hole, but the
1526 * block size still needs to be known for replay.
1527 */
1528 BP_SET_LSIZE(bp, db->db_size);
1529 } else if (!BP_IS_EMBEDDED(bp)) {
1530 ASSERT(BP_GET_LEVEL(bp) == 0);
1531 BP_SET_FILL(bp, 1);
1532 }
1533 }
1534 }
1535
1536 static void
1537 dmu_sync_late_arrival_ready(zio_t *zio)
1538 {
1539 dmu_sync_ready(zio, NULL, zio->io_private);
1540 }
1541
1542 static void
1543 dmu_sync_done(zio_t *zio, arc_buf_t *buf, void *varg)
1544 {
1545 (void) buf;
1546 dmu_sync_arg_t *dsa = varg;
1547 dbuf_dirty_record_t *dr = dsa->dsa_dr;
1548 dmu_buf_impl_t *db = dr->dr_dbuf;
1549 zgd_t *zgd = dsa->dsa_zgd;
1550
1551 /*
1552 * Record the vdev(s) backing this blkptr so they can be flushed after
1553 * the writes for the lwb have completed.
1554 */
1555 if (zio->io_error == 0) {
1556 zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp);
1557 }
1558
1559 mutex_enter(&db->db_mtx);
1560 ASSERT(dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC);
1561 if (zio->io_error == 0) {
1562 dr->dt.dl.dr_nopwrite = !!(zio->io_flags & ZIO_FLAG_NOPWRITE);
1563 if (dr->dt.dl.dr_nopwrite) {
1564 blkptr_t *bp = zio->io_bp;
1565 blkptr_t *bp_orig = &zio->io_bp_orig;
1566 uint8_t chksum = BP_GET_CHECKSUM(bp_orig);
1567
1568 ASSERT(BP_EQUAL(bp, bp_orig));
1569 VERIFY(BP_EQUAL(bp, db->db_blkptr));
1570 ASSERT(zio->io_prop.zp_compress != ZIO_COMPRESS_OFF);
1571 VERIFY(zio_checksum_table[chksum].ci_flags &
1572 ZCHECKSUM_FLAG_NOPWRITE);
1573 }
1574 dr->dt.dl.dr_overridden_by = *zio->io_bp;
1575 dr->dt.dl.dr_override_state = DR_OVERRIDDEN;
1576 dr->dt.dl.dr_copies = zio->io_prop.zp_copies;
1577
1578 /*
1579 * Old style holes are filled with all zeros, whereas
1580 * new-style holes maintain their lsize, type, level,
1581 * and birth time (see zio_write_compress). While we
1582 * need to reset the BP_SET_LSIZE() call that happened
1583 * in dmu_sync_ready for old style holes, we do *not*
1584 * want to wipe out the information contained in new
1585 * style holes. Thus, only zero out the block pointer if
1586 * it's an old style hole.
1587 */
1588 if (BP_IS_HOLE(&dr->dt.dl.dr_overridden_by) &&
1589 dr->dt.dl.dr_overridden_by.blk_birth == 0)
1590 BP_ZERO(&dr->dt.dl.dr_overridden_by);
1591 } else {
1592 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
1593 }
1594 cv_broadcast(&db->db_changed);
1595 mutex_exit(&db->db_mtx);
1596
1597 dsa->dsa_done(dsa->dsa_zgd, zio->io_error);
1598
1599 kmem_free(dsa, sizeof (*dsa));
1600 }
1601
1602 static void
1603 dmu_sync_late_arrival_done(zio_t *zio)
1604 {
1605 blkptr_t *bp = zio->io_bp;
1606 dmu_sync_arg_t *dsa = zio->io_private;
1607 zgd_t *zgd = dsa->dsa_zgd;
1608
1609 if (zio->io_error == 0) {
1610 /*
1611 * Record the vdev(s) backing this blkptr so they can be
1612 * flushed after the writes for the lwb have completed.
1613 */
1614 zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp);
1615
1616 if (!BP_IS_HOLE(bp)) {
1617 blkptr_t *bp_orig __maybe_unused = &zio->io_bp_orig;
1618 ASSERT(!(zio->io_flags & ZIO_FLAG_NOPWRITE));
1619 ASSERT(BP_IS_HOLE(bp_orig) || !BP_EQUAL(bp, bp_orig));
1620 ASSERT(zio->io_bp->blk_birth == zio->io_txg);
1621 ASSERT(zio->io_txg > spa_syncing_txg(zio->io_spa));
1622 zio_free(zio->io_spa, zio->io_txg, zio->io_bp);
1623 }
1624 }
1625
1626 dmu_tx_commit(dsa->dsa_tx);
1627
1628 dsa->dsa_done(dsa->dsa_zgd, zio->io_error);
1629
1630 abd_free(zio->io_abd);
1631 kmem_free(dsa, sizeof (*dsa));
1632 }
1633
1634 static int
1635 dmu_sync_late_arrival(zio_t *pio, objset_t *os, dmu_sync_cb_t *done, zgd_t *zgd,
1636 zio_prop_t *zp, zbookmark_phys_t *zb)
1637 {
1638 dmu_sync_arg_t *dsa;
1639 dmu_tx_t *tx;
1640
1641 tx = dmu_tx_create(os);
1642 dmu_tx_hold_space(tx, zgd->zgd_db->db_size);
1643 if (dmu_tx_assign(tx, TXG_WAIT) != 0) {
1644 dmu_tx_abort(tx);
1645 /* Make zl_get_data do txg_waited_synced() */
1646 return (SET_ERROR(EIO));
1647 }
1648
1649 /*
1650 * In order to prevent the zgd's lwb from being free'd prior to
1651 * dmu_sync_late_arrival_done() being called, we have to ensure
1652 * the lwb's "max txg" takes this tx's txg into account.
1653 */
1654 zil_lwb_add_txg(zgd->zgd_lwb, dmu_tx_get_txg(tx));
1655
1656 dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
1657 dsa->dsa_dr = NULL;
1658 dsa->dsa_done = done;
1659 dsa->dsa_zgd = zgd;
1660 dsa->dsa_tx = tx;
1661
1662 /*
1663 * Since we are currently syncing this txg, it's nontrivial to
1664 * determine what BP to nopwrite against, so we disable nopwrite.
1665 *
1666 * When syncing, the db_blkptr is initially the BP of the previous
1667 * txg. We can not nopwrite against it because it will be changed
1668 * (this is similar to the non-late-arrival case where the dbuf is
1669 * dirty in a future txg).
1670 *
1671 * Then dbuf_write_ready() sets bp_blkptr to the location we will write.
1672 * We can not nopwrite against it because although the BP will not
1673 * (typically) be changed, the data has not yet been persisted to this
1674 * location.
1675 *
1676 * Finally, when dbuf_write_done() is called, it is theoretically
1677 * possible to always nopwrite, because the data that was written in
1678 * this txg is the same data that we are trying to write. However we
1679 * would need to check that this dbuf is not dirty in any future
1680 * txg's (as we do in the normal dmu_sync() path). For simplicity, we
1681 * don't nopwrite in this case.
1682 */
1683 zp->zp_nopwrite = B_FALSE;
1684
1685 zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), zgd->zgd_bp,
1686 abd_get_from_buf(zgd->zgd_db->db_data, zgd->zgd_db->db_size),
1687 zgd->zgd_db->db_size, zgd->zgd_db->db_size, zp,
1688 dmu_sync_late_arrival_ready, NULL, NULL, dmu_sync_late_arrival_done,
1689 dsa, ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, zb));
1690
1691 return (0);
1692 }
1693
1694 /*
1695 * Intent log support: sync the block associated with db to disk.
1696 * N.B. and XXX: the caller is responsible for making sure that the
1697 * data isn't changing while dmu_sync() is writing it.
1698 *
1699 * Return values:
1700 *
1701 * EEXIST: this txg has already been synced, so there's nothing to do.
1702 * The caller should not log the write.
1703 *
1704 * ENOENT: the block was dbuf_free_range()'d, so there's nothing to do.
1705 * The caller should not log the write.
1706 *
1707 * EALREADY: this block is already in the process of being synced.
1708 * The caller should track its progress (somehow).
1709 *
1710 * EIO: could not do the I/O.
1711 * The caller should do a txg_wait_synced().
1712 *
1713 * 0: the I/O has been initiated.
1714 * The caller should log this blkptr in the done callback.
1715 * It is possible that the I/O will fail, in which case
1716 * the error will be reported to the done callback and
1717 * propagated to pio from zio_done().
1718 */
1719 int
1720 dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd)
1721 {
1722 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zgd->zgd_db;
1723 objset_t *os = db->db_objset;
1724 dsl_dataset_t *ds = os->os_dsl_dataset;
1725 dbuf_dirty_record_t *dr, *dr_next;
1726 dmu_sync_arg_t *dsa;
1727 zbookmark_phys_t zb;
1728 zio_prop_t zp;
1729 dnode_t *dn;
1730
1731 ASSERT(pio != NULL);
1732 ASSERT(txg != 0);
1733
1734 SET_BOOKMARK(&zb, ds->ds_object,
1735 db->db.db_object, db->db_level, db->db_blkid);
1736
1737 DB_DNODE_ENTER(db);
1738 dn = DB_DNODE(db);
1739 dmu_write_policy(os, dn, db->db_level, WP_DMU_SYNC, &zp);
1740 DB_DNODE_EXIT(db);
1741
1742 /*
1743 * If we're frozen (running ziltest), we always need to generate a bp.
1744 */
1745 if (txg > spa_freeze_txg(os->os_spa))
1746 return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb));
1747
1748 /*
1749 * Grabbing db_mtx now provides a barrier between dbuf_sync_leaf()
1750 * and us. If we determine that this txg is not yet syncing,
1751 * but it begins to sync a moment later, that's OK because the
1752 * sync thread will block in dbuf_sync_leaf() until we drop db_mtx.
1753 */
1754 mutex_enter(&db->db_mtx);
1755
1756 if (txg <= spa_last_synced_txg(os->os_spa)) {
1757 /*
1758 * This txg has already synced. There's nothing to do.
1759 */
1760 mutex_exit(&db->db_mtx);
1761 return (SET_ERROR(EEXIST));
1762 }
1763
1764 if (txg <= spa_syncing_txg(os->os_spa)) {
1765 /*
1766 * This txg is currently syncing, so we can't mess with
1767 * the dirty record anymore; just write a new log block.
1768 */
1769 mutex_exit(&db->db_mtx);
1770 return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb));
1771 }
1772
1773 dr = dbuf_find_dirty_eq(db, txg);
1774
1775 if (dr == NULL) {
1776 /*
1777 * There's no dr for this dbuf, so it must have been freed.
1778 * There's no need to log writes to freed blocks, so we're done.
1779 */
1780 mutex_exit(&db->db_mtx);
1781 return (SET_ERROR(ENOENT));
1782 }
1783
1784 dr_next = list_next(&db->db_dirty_records, dr);
1785 ASSERT(dr_next == NULL || dr_next->dr_txg < txg);
1786
1787 if (db->db_blkptr != NULL) {
1788 /*
1789 * We need to fill in zgd_bp with the current blkptr so that
1790 * the nopwrite code can check if we're writing the same
1791 * data that's already on disk. We can only nopwrite if we
1792 * are sure that after making the copy, db_blkptr will not
1793 * change until our i/o completes. We ensure this by
1794 * holding the db_mtx, and only allowing nopwrite if the
1795 * block is not already dirty (see below). This is verified
1796 * by dmu_sync_done(), which VERIFYs that the db_blkptr has
1797 * not changed.
1798 */
1799 *zgd->zgd_bp = *db->db_blkptr;
1800 }
1801
1802 /*
1803 * Assume the on-disk data is X, the current syncing data (in
1804 * txg - 1) is Y, and the current in-memory data is Z (currently
1805 * in dmu_sync).
1806 *
1807 * We usually want to perform a nopwrite if X and Z are the
1808 * same. However, if Y is different (i.e. the BP is going to
1809 * change before this write takes effect), then a nopwrite will
1810 * be incorrect - we would override with X, which could have
1811 * been freed when Y was written.
1812 *
1813 * (Note that this is not a concern when we are nop-writing from
1814 * syncing context, because X and Y must be identical, because
1815 * all previous txgs have been synced.)
1816 *
1817 * Therefore, we disable nopwrite if the current BP could change
1818 * before this TXG. There are two ways it could change: by
1819 * being dirty (dr_next is non-NULL), or by being freed
1820 * (dnode_block_freed()). This behavior is verified by
1821 * zio_done(), which VERIFYs that the override BP is identical
1822 * to the on-disk BP.
1823 */
1824 DB_DNODE_ENTER(db);
1825 dn = DB_DNODE(db);
1826 if (dr_next != NULL || dnode_block_freed(dn, db->db_blkid))
1827 zp.zp_nopwrite = B_FALSE;
1828 DB_DNODE_EXIT(db);
1829
1830 ASSERT(dr->dr_txg == txg);
1831 if (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC ||
1832 dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
1833 /*
1834 * We have already issued a sync write for this buffer,
1835 * or this buffer has already been synced. It could not
1836 * have been dirtied since, or we would have cleared the state.
1837 */
1838 mutex_exit(&db->db_mtx);
1839 return (SET_ERROR(EALREADY));
1840 }
1841
1842 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
1843 dr->dt.dl.dr_override_state = DR_IN_DMU_SYNC;
1844 mutex_exit(&db->db_mtx);
1845
1846 dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
1847 dsa->dsa_dr = dr;
1848 dsa->dsa_done = done;
1849 dsa->dsa_zgd = zgd;
1850 dsa->dsa_tx = NULL;
1851
1852 zio_nowait(arc_write(pio, os->os_spa, txg,
1853 zgd->zgd_bp, dr->dt.dl.dr_data, dbuf_is_l2cacheable(db),
1854 &zp, dmu_sync_ready, NULL, NULL, dmu_sync_done, dsa,
1855 ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, &zb));
1856
1857 return (0);
1858 }
1859
1860 int
1861 dmu_object_set_nlevels(objset_t *os, uint64_t object, int nlevels, dmu_tx_t *tx)
1862 {
1863 dnode_t *dn;
1864 int err;
1865
1866 err = dnode_hold(os, object, FTAG, &dn);
1867 if (err)
1868 return (err);
1869 err = dnode_set_nlevels(dn, nlevels, tx);
1870 dnode_rele(dn, FTAG);
1871 return (err);
1872 }
1873
1874 int
1875 dmu_object_set_blocksize(objset_t *os, uint64_t object, uint64_t size, int ibs,
1876 dmu_tx_t *tx)
1877 {
1878 dnode_t *dn;
1879 int err;
1880
1881 err = dnode_hold(os, object, FTAG, &dn);
1882 if (err)
1883 return (err);
1884 err = dnode_set_blksz(dn, size, ibs, tx);
1885 dnode_rele(dn, FTAG);
1886 return (err);
1887 }
1888
1889 int
1890 dmu_object_set_maxblkid(objset_t *os, uint64_t object, uint64_t maxblkid,
1891 dmu_tx_t *tx)
1892 {
1893 dnode_t *dn;
1894 int err;
1895
1896 err = dnode_hold(os, object, FTAG, &dn);
1897 if (err)
1898 return (err);
1899 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
1900 dnode_new_blkid(dn, maxblkid, tx, B_FALSE, B_TRUE);
1901 rw_exit(&dn->dn_struct_rwlock);
1902 dnode_rele(dn, FTAG);
1903 return (0);
1904 }
1905
1906 void
1907 dmu_object_set_checksum(objset_t *os, uint64_t object, uint8_t checksum,
1908 dmu_tx_t *tx)
1909 {
1910 dnode_t *dn;
1911
1912 /*
1913 * Send streams include each object's checksum function. This
1914 * check ensures that the receiving system can understand the
1915 * checksum function transmitted.
1916 */
1917 ASSERT3U(checksum, <, ZIO_CHECKSUM_LEGACY_FUNCTIONS);
1918
1919 VERIFY0(dnode_hold(os, object, FTAG, &dn));
1920 ASSERT3U(checksum, <, ZIO_CHECKSUM_FUNCTIONS);
1921 dn->dn_checksum = checksum;
1922 dnode_setdirty(dn, tx);
1923 dnode_rele(dn, FTAG);
1924 }
1925
1926 void
1927 dmu_object_set_compress(objset_t *os, uint64_t object, uint8_t compress,
1928 dmu_tx_t *tx)
1929 {
1930 dnode_t *dn;
1931
1932 /*
1933 * Send streams include each object's compression function. This
1934 * check ensures that the receiving system can understand the
1935 * compression function transmitted.
1936 */
1937 ASSERT3U(compress, <, ZIO_COMPRESS_LEGACY_FUNCTIONS);
1938
1939 VERIFY0(dnode_hold(os, object, FTAG, &dn));
1940 dn->dn_compress = compress;
1941 dnode_setdirty(dn, tx);
1942 dnode_rele(dn, FTAG);
1943 }
1944
1945 /*
1946 * When the "redundant_metadata" property is set to "most", only indirect
1947 * blocks of this level and higher will have an additional ditto block.
1948 */
1949 static const int zfs_redundant_metadata_most_ditto_level = 2;
1950
1951 void
1952 dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp)
1953 {
1954 dmu_object_type_t type = dn ? dn->dn_type : DMU_OT_OBJSET;
1955 boolean_t ismd = (level > 0 || DMU_OT_IS_METADATA(type) ||
1956 (wp & WP_SPILL));
1957 enum zio_checksum checksum = os->os_checksum;
1958 enum zio_compress compress = os->os_compress;
1959 uint8_t complevel = os->os_complevel;
1960 enum zio_checksum dedup_checksum = os->os_dedup_checksum;
1961 boolean_t dedup = B_FALSE;
1962 boolean_t nopwrite = B_FALSE;
1963 boolean_t dedup_verify = os->os_dedup_verify;
1964 boolean_t encrypt = B_FALSE;
1965 int copies = os->os_copies;
1966
1967 /*
1968 * We maintain different write policies for each of the following
1969 * types of data:
1970 * 1. metadata
1971 * 2. preallocated blocks (i.e. level-0 blocks of a dump device)
1972 * 3. all other level 0 blocks
1973 */
1974 if (ismd) {
1975 /*
1976 * XXX -- we should design a compression algorithm
1977 * that specializes in arrays of bps.
1978 */
1979 compress = zio_compress_select(os->os_spa,
1980 ZIO_COMPRESS_ON, ZIO_COMPRESS_ON);
1981
1982 /*
1983 * Metadata always gets checksummed. If the data
1984 * checksum is multi-bit correctable, and it's not a
1985 * ZBT-style checksum, then it's suitable for metadata
1986 * as well. Otherwise, the metadata checksum defaults
1987 * to fletcher4.
1988 */
1989 if (!(zio_checksum_table[checksum].ci_flags &
1990 ZCHECKSUM_FLAG_METADATA) ||
1991 (zio_checksum_table[checksum].ci_flags &
1992 ZCHECKSUM_FLAG_EMBEDDED))
1993 checksum = ZIO_CHECKSUM_FLETCHER_4;
1994
1995 if (os->os_redundant_metadata == ZFS_REDUNDANT_METADATA_ALL ||
1996 (os->os_redundant_metadata ==
1997 ZFS_REDUNDANT_METADATA_MOST &&
1998 (level >= zfs_redundant_metadata_most_ditto_level ||
1999 DMU_OT_IS_METADATA(type) || (wp & WP_SPILL))))
2000 copies++;
2001 } else if (wp & WP_NOFILL) {
2002 ASSERT(level == 0);
2003
2004 /*
2005 * If we're writing preallocated blocks, we aren't actually
2006 * writing them so don't set any policy properties. These
2007 * blocks are currently only used by an external subsystem
2008 * outside of zfs (i.e. dump) and not written by the zio
2009 * pipeline.
2010 */
2011 compress = ZIO_COMPRESS_OFF;
2012 checksum = ZIO_CHECKSUM_OFF;
2013 } else {
2014 compress = zio_compress_select(os->os_spa, dn->dn_compress,
2015 compress);
2016 complevel = zio_complevel_select(os->os_spa, compress,
2017 complevel, complevel);
2018
2019 checksum = (dedup_checksum == ZIO_CHECKSUM_OFF) ?
2020 zio_checksum_select(dn->dn_checksum, checksum) :
2021 dedup_checksum;
2022
2023 /*
2024 * Determine dedup setting. If we are in dmu_sync(),
2025 * we won't actually dedup now because that's all
2026 * done in syncing context; but we do want to use the
2027 * dedup checksum. If the checksum is not strong
2028 * enough to ensure unique signatures, force
2029 * dedup_verify.
2030 */
2031 if (dedup_checksum != ZIO_CHECKSUM_OFF) {
2032 dedup = (wp & WP_DMU_SYNC) ? B_FALSE : B_TRUE;
2033 if (!(zio_checksum_table[checksum].ci_flags &
2034 ZCHECKSUM_FLAG_DEDUP))
2035 dedup_verify = B_TRUE;
2036 }
2037
2038 /*
2039 * Enable nopwrite if we have secure enough checksum
2040 * algorithm (see comment in zio_nop_write) and
2041 * compression is enabled. We don't enable nopwrite if
2042 * dedup is enabled as the two features are mutually
2043 * exclusive.
2044 */
2045 nopwrite = (!dedup && (zio_checksum_table[checksum].ci_flags &
2046 ZCHECKSUM_FLAG_NOPWRITE) &&
2047 compress != ZIO_COMPRESS_OFF && zfs_nopwrite_enabled);
2048 }
2049
2050 /*
2051 * All objects in an encrypted objset are protected from modification
2052 * via a MAC. Encrypted objects store their IV and salt in the last DVA
2053 * in the bp, so we cannot use all copies. Encrypted objects are also
2054 * not subject to nopwrite since writing the same data will still
2055 * result in a new ciphertext. Only encrypted blocks can be dedup'd
2056 * to avoid ambiguity in the dedup code since the DDT does not store
2057 * object types.
2058 */
2059 if (os->os_encrypted && (wp & WP_NOFILL) == 0) {
2060 encrypt = B_TRUE;
2061
2062 if (DMU_OT_IS_ENCRYPTED(type)) {
2063 copies = MIN(copies, SPA_DVAS_PER_BP - 1);
2064 nopwrite = B_FALSE;
2065 } else {
2066 dedup = B_FALSE;
2067 }
2068
2069 if (level <= 0 &&
2070 (type == DMU_OT_DNODE || type == DMU_OT_OBJSET)) {
2071 compress = ZIO_COMPRESS_EMPTY;
2072 }
2073 }
2074
2075 zp->zp_compress = compress;
2076 zp->zp_complevel = complevel;
2077 zp->zp_checksum = checksum;
2078 zp->zp_type = (wp & WP_SPILL) ? dn->dn_bonustype : type;
2079 zp->zp_level = level;
2080 zp->zp_copies = MIN(copies, spa_max_replication(os->os_spa));
2081 zp->zp_dedup = dedup;
2082 zp->zp_dedup_verify = dedup && dedup_verify;
2083 zp->zp_nopwrite = nopwrite;
2084 zp->zp_encrypt = encrypt;
2085 zp->zp_byteorder = ZFS_HOST_BYTEORDER;
2086 memset(zp->zp_salt, 0, ZIO_DATA_SALT_LEN);
2087 memset(zp->zp_iv, 0, ZIO_DATA_IV_LEN);
2088 memset(zp->zp_mac, 0, ZIO_DATA_MAC_LEN);
2089 zp->zp_zpl_smallblk = DMU_OT_IS_FILE(zp->zp_type) ?
2090 os->os_zpl_special_smallblock : 0;
2091
2092 ASSERT3U(zp->zp_compress, !=, ZIO_COMPRESS_INHERIT);
2093 }
2094
2095 /*
2096 * This function is only called from zfs_holey_common() for zpl_llseek()
2097 * in order to determine the location of holes. In order to accurately
2098 * report holes all dirty data must be synced to disk. This causes extremely
2099 * poor performance when seeking for holes in a dirty file. As a compromise,
2100 * only provide hole data when the dnode is clean. When a dnode is dirty
2101 * report the dnode as having no holes which is always a safe thing to do.
2102 */
2103 int
2104 dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off)
2105 {
2106 dnode_t *dn;
2107 int err;
2108
2109 restart:
2110 err = dnode_hold(os, object, FTAG, &dn);
2111 if (err)
2112 return (err);
2113
2114 rw_enter(&dn->dn_struct_rwlock, RW_READER);
2115
2116 if (dnode_is_dirty(dn)) {
2117 /*
2118 * If the zfs_dmu_offset_next_sync module option is enabled
2119 * then strict hole reporting has been requested. Dirty
2120 * dnodes must be synced to disk to accurately report all
2121 * holes. When disabled dirty dnodes are reported to not
2122 * have any holes which is always safe.
2123 *
2124 * When called by zfs_holey_common() the zp->z_rangelock
2125 * is held to prevent zfs_write() and mmap writeback from
2126 * re-dirtying the dnode after txg_wait_synced().
2127 */
2128 if (zfs_dmu_offset_next_sync) {
2129 rw_exit(&dn->dn_struct_rwlock);
2130 dnode_rele(dn, FTAG);
2131 txg_wait_synced(dmu_objset_pool(os), 0);
2132 goto restart;
2133 }
2134
2135 err = SET_ERROR(EBUSY);
2136 } else {
2137 err = dnode_next_offset(dn, DNODE_FIND_HAVELOCK |
2138 (hole ? DNODE_FIND_HOLE : 0), off, 1, 1, 0);
2139 }
2140
2141 rw_exit(&dn->dn_struct_rwlock);
2142 dnode_rele(dn, FTAG);
2143
2144 return (err);
2145 }
2146
2147 void
2148 __dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi)
2149 {
2150 dnode_phys_t *dnp = dn->dn_phys;
2151
2152 doi->doi_data_block_size = dn->dn_datablksz;
2153 doi->doi_metadata_block_size = dn->dn_indblkshift ?
2154 1ULL << dn->dn_indblkshift : 0;
2155 doi->doi_type = dn->dn_type;
2156 doi->doi_bonus_type = dn->dn_bonustype;
2157 doi->doi_bonus_size = dn->dn_bonuslen;
2158 doi->doi_dnodesize = dn->dn_num_slots << DNODE_SHIFT;
2159 doi->doi_indirection = dn->dn_nlevels;
2160 doi->doi_checksum = dn->dn_checksum;
2161 doi->doi_compress = dn->dn_compress;
2162 doi->doi_nblkptr = dn->dn_nblkptr;
2163 doi->doi_physical_blocks_512 = (DN_USED_BYTES(dnp) + 256) >> 9;
2164 doi->doi_max_offset = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
2165 doi->doi_fill_count = 0;
2166 for (int i = 0; i < dnp->dn_nblkptr; i++)
2167 doi->doi_fill_count += BP_GET_FILL(&dnp->dn_blkptr[i]);
2168 }
2169
2170 void
2171 dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi)
2172 {
2173 rw_enter(&dn->dn_struct_rwlock, RW_READER);
2174 mutex_enter(&dn->dn_mtx);
2175
2176 __dmu_object_info_from_dnode(dn, doi);
2177
2178 mutex_exit(&dn->dn_mtx);
2179 rw_exit(&dn->dn_struct_rwlock);
2180 }
2181
2182 /*
2183 * Get information on a DMU object.
2184 * If doi is NULL, just indicates whether the object exists.
2185 */
2186 int
2187 dmu_object_info(objset_t *os, uint64_t object, dmu_object_info_t *doi)
2188 {
2189 dnode_t *dn;
2190 int err = dnode_hold(os, object, FTAG, &dn);
2191
2192 if (err)
2193 return (err);
2194
2195 if (doi != NULL)
2196 dmu_object_info_from_dnode(dn, doi);
2197
2198 dnode_rele(dn, FTAG);
2199 return (0);
2200 }
2201
2202 /*
2203 * As above, but faster; can be used when you have a held dbuf in hand.
2204 */
2205 void
2206 dmu_object_info_from_db(dmu_buf_t *db_fake, dmu_object_info_t *doi)
2207 {
2208 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2209
2210 DB_DNODE_ENTER(db);
2211 dmu_object_info_from_dnode(DB_DNODE(db), doi);
2212 DB_DNODE_EXIT(db);
2213 }
2214
2215 /*
2216 * Faster still when you only care about the size.
2217 */
2218 void
2219 dmu_object_size_from_db(dmu_buf_t *db_fake, uint32_t *blksize,
2220 u_longlong_t *nblk512)
2221 {
2222 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2223 dnode_t *dn;
2224
2225 DB_DNODE_ENTER(db);
2226 dn = DB_DNODE(db);
2227
2228 *blksize = dn->dn_datablksz;
2229 /* add in number of slots used for the dnode itself */
2230 *nblk512 = ((DN_USED_BYTES(dn->dn_phys) + SPA_MINBLOCKSIZE/2) >>
2231 SPA_MINBLOCKSHIFT) + dn->dn_num_slots;
2232 DB_DNODE_EXIT(db);
2233 }
2234
2235 void
2236 dmu_object_dnsize_from_db(dmu_buf_t *db_fake, int *dnsize)
2237 {
2238 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2239 dnode_t *dn;
2240
2241 DB_DNODE_ENTER(db);
2242 dn = DB_DNODE(db);
2243 *dnsize = dn->dn_num_slots << DNODE_SHIFT;
2244 DB_DNODE_EXIT(db);
2245 }
2246
2247 void
2248 byteswap_uint64_array(void *vbuf, size_t size)
2249 {
2250 uint64_t *buf = vbuf;
2251 size_t count = size >> 3;
2252 int i;
2253
2254 ASSERT((size & 7) == 0);
2255
2256 for (i = 0; i < count; i++)
2257 buf[i] = BSWAP_64(buf[i]);
2258 }
2259
2260 void
2261 byteswap_uint32_array(void *vbuf, size_t size)
2262 {
2263 uint32_t *buf = vbuf;
2264 size_t count = size >> 2;
2265 int i;
2266
2267 ASSERT((size & 3) == 0);
2268
2269 for (i = 0; i < count; i++)
2270 buf[i] = BSWAP_32(buf[i]);
2271 }
2272
2273 void
2274 byteswap_uint16_array(void *vbuf, size_t size)
2275 {
2276 uint16_t *buf = vbuf;
2277 size_t count = size >> 1;
2278 int i;
2279
2280 ASSERT((size & 1) == 0);
2281
2282 for (i = 0; i < count; i++)
2283 buf[i] = BSWAP_16(buf[i]);
2284 }
2285
2286 void
2287 byteswap_uint8_array(void *vbuf, size_t size)
2288 {
2289 (void) vbuf, (void) size;
2290 }
2291
2292 void
2293 dmu_init(void)
2294 {
2295 abd_init();
2296 zfs_dbgmsg_init();
2297 sa_cache_init();
2298 dmu_objset_init();
2299 dnode_init();
2300 zfetch_init();
2301 dmu_tx_init();
2302 l2arc_init();
2303 arc_init();
2304 dbuf_init();
2305 }
2306
2307 void
2308 dmu_fini(void)
2309 {
2310 arc_fini(); /* arc depends on l2arc, so arc must go first */
2311 l2arc_fini();
2312 dmu_tx_fini();
2313 zfetch_fini();
2314 dbuf_fini();
2315 dnode_fini();
2316 dmu_objset_fini();
2317 sa_cache_fini();
2318 zfs_dbgmsg_fini();
2319 abd_fini();
2320 }
2321
2322 EXPORT_SYMBOL(dmu_bonus_hold);
2323 EXPORT_SYMBOL(dmu_bonus_hold_by_dnode);
2324 EXPORT_SYMBOL(dmu_buf_hold_array_by_bonus);
2325 EXPORT_SYMBOL(dmu_buf_rele_array);
2326 EXPORT_SYMBOL(dmu_prefetch);
2327 EXPORT_SYMBOL(dmu_free_range);
2328 EXPORT_SYMBOL(dmu_free_long_range);
2329 EXPORT_SYMBOL(dmu_free_long_object);
2330 EXPORT_SYMBOL(dmu_read);
2331 EXPORT_SYMBOL(dmu_read_by_dnode);
2332 EXPORT_SYMBOL(dmu_write);
2333 EXPORT_SYMBOL(dmu_write_by_dnode);
2334 EXPORT_SYMBOL(dmu_prealloc);
2335 EXPORT_SYMBOL(dmu_object_info);
2336 EXPORT_SYMBOL(dmu_object_info_from_dnode);
2337 EXPORT_SYMBOL(dmu_object_info_from_db);
2338 EXPORT_SYMBOL(dmu_object_size_from_db);
2339 EXPORT_SYMBOL(dmu_object_dnsize_from_db);
2340 EXPORT_SYMBOL(dmu_object_set_nlevels);
2341 EXPORT_SYMBOL(dmu_object_set_blocksize);
2342 EXPORT_SYMBOL(dmu_object_set_maxblkid);
2343 EXPORT_SYMBOL(dmu_object_set_checksum);
2344 EXPORT_SYMBOL(dmu_object_set_compress);
2345 EXPORT_SYMBOL(dmu_offset_next);
2346 EXPORT_SYMBOL(dmu_write_policy);
2347 EXPORT_SYMBOL(dmu_sync);
2348 EXPORT_SYMBOL(dmu_request_arcbuf);
2349 EXPORT_SYMBOL(dmu_return_arcbuf);
2350 EXPORT_SYMBOL(dmu_assign_arcbuf_by_dnode);
2351 EXPORT_SYMBOL(dmu_assign_arcbuf_by_dbuf);
2352 EXPORT_SYMBOL(dmu_buf_hold);
2353 EXPORT_SYMBOL(dmu_ot);
2354
2355 ZFS_MODULE_PARAM(zfs, zfs_, nopwrite_enabled, INT, ZMOD_RW,
2356 "Enable NOP writes");
2357
2358 ZFS_MODULE_PARAM(zfs, zfs_, per_txg_dirty_frees_percent, ULONG, ZMOD_RW,
2359 "Percentage of dirtied blocks from frees in one TXG");
2360
2361 ZFS_MODULE_PARAM(zfs, zfs_, dmu_offset_next_sync, INT, ZMOD_RW,
2362 "Enable forcing txg sync to find holes");
2363
2364 /* CSTYLED */
2365 ZFS_MODULE_PARAM(zfs, , dmu_prefetch_max, INT, ZMOD_RW,
2366 "Limit one prefetch call to this size");
ZFS On Linux mirror for PVE