]> git.proxmox.com Git - mirror_zfs.git/blob - module/zfs/dmu.c
projects / mirror_zfs.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Fix typo/etc in module/zfs/zfs_ctldir.c
[mirror_zfs.git] / module / zfs / dmu.c
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
24 * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
25 * Copyright (c) 2013, Joyent, Inc. All rights reserved.
26 * Copyright (c) 2016, Nexenta Systems, Inc. All rights reserved.
27 * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
28 * Copyright (c) 2019 Datto Inc.
29 */
30
31 #include <sys/dmu.h>
32 #include <sys/dmu_impl.h>
33 #include <sys/dmu_tx.h>
34 #include <sys/dbuf.h>
35 #include <sys/dnode.h>
36 #include <sys/zfs_context.h>
37 #include <sys/dmu_objset.h>
38 #include <sys/dmu_traverse.h>
39 #include <sys/dsl_dataset.h>
40 #include <sys/dsl_dir.h>
41 #include <sys/dsl_pool.h>
42 #include <sys/dsl_synctask.h>
43 #include <sys/dsl_prop.h>
44 #include <sys/dmu_zfetch.h>
45 #include <sys/zfs_ioctl.h>
46 #include <sys/zap.h>
47 #include <sys/zio_checksum.h>
48 #include <sys/zio_compress.h>
49 #include <sys/sa.h>
50 #include <sys/zfeature.h>
51 #include <sys/abd.h>
52 #include <sys/trace_dmu.h>
53 #include <sys/zfs_rlock.h>
54 #ifdef _KERNEL
55 #include <sys/vmsystm.h>
56 #include <sys/zfs_znode.h>
57 #endif
58
59 /*
60 * Enable/disable nopwrite feature.
61 */
62 int zfs_nopwrite_enabled = 1;
63
64 /*
65 * Tunable to control percentage of dirtied L1 blocks from frees allowed into
66 * one TXG. After this threshold is crossed, additional dirty blocks from frees
67 * will wait until the next TXG.
68 * A value of zero will disable this throttle.
69 */
70 unsigned long zfs_per_txg_dirty_frees_percent = 5;
71
72 /*
73 * Enable/disable forcing txg sync when dirty in dmu_offset_next.
74 */
75 int zfs_dmu_offset_next_sync = 0;
76
77 /*
78 * This can be used for testing, to ensure that certain actions happen
79 * while in the middle of a remap (which might otherwise complete too
80 * quickly). Used by ztest(8).
81 */
82 int zfs_object_remap_one_indirect_delay_ms = 0;
83
84 const dmu_object_type_info_t dmu_ot[DMU_OT_NUMTYPES] = {
85 {DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "unallocated" },
86 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "object directory" },
87 {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "object array" },
88 {DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "packed nvlist" },
89 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "packed nvlist size" },
90 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj" },
91 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj header" },
92 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA space map header" },
93 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA space map" },
94 {DMU_BSWAP_UINT64, TRUE, FALSE, TRUE, "ZIL intent log" },
95 {DMU_BSWAP_DNODE, TRUE, FALSE, TRUE, "DMU dnode" },
96 {DMU_BSWAP_OBJSET, TRUE, TRUE, FALSE, "DMU objset" },
97 {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL directory" },
98 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL directory child map"},
99 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dataset snap map" },
100 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL props" },
101 {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL dataset" },
102 {DMU_BSWAP_ZNODE, TRUE, FALSE, FALSE, "ZFS znode" },
103 {DMU_BSWAP_OLDACL, TRUE, FALSE, TRUE, "ZFS V0 ACL" },
104 {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "ZFS plain file" },
105 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS directory" },
106 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "ZFS master node" },
107 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS delete queue" },
108 {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "zvol object" },
109 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "zvol prop" },
110 {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "other uint8[]" },
111 {DMU_BSWAP_UINT64, FALSE, FALSE, TRUE, "other uint64[]" },
112 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "other ZAP" },
113 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "persistent error log" },
114 {DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "SPA history" },
115 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA history offsets" },
116 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "Pool properties" },
117 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL permissions" },
118 {DMU_BSWAP_ACL, TRUE, FALSE, TRUE, "ZFS ACL" },
119 {DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "ZFS SYSACL" },
120 {DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "FUID table" },
121 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "FUID table size" },
122 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dataset next clones"},
123 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "scan work queue" },
124 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS user/group/project used" },
125 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS user/group/project quota"},
126 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "snapshot refcount tags"},
127 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "DDT ZAP algorithm" },
128 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "DDT statistics" },
129 {DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "System attributes" },
130 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA master node" },
131 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA attr registration" },
132 {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA attr layouts" },
133 {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "scan translations" },
134 {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "deduplicated block" },
135 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL deadlist map" },
136 {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL deadlist map hdr" },
137 {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dir clones" },
138 {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj subobj" }
139 };
140
141 const dmu_object_byteswap_info_t dmu_ot_byteswap[DMU_BSWAP_NUMFUNCS] = {
142 { byteswap_uint8_array, "uint8" },
143 { byteswap_uint16_array, "uint16" },
144 { byteswap_uint32_array, "uint32" },
145 { byteswap_uint64_array, "uint64" },
146 { zap_byteswap, "zap" },
147 { dnode_buf_byteswap, "dnode" },
148 { dmu_objset_byteswap, "objset" },
149 { zfs_znode_byteswap, "znode" },
150 { zfs_oldacl_byteswap, "oldacl" },
151 { zfs_acl_byteswap, "acl" }
152 };
153
154 int
155 dmu_buf_hold_noread_by_dnode(dnode_t *dn, uint64_t offset,
156 void *tag, dmu_buf_t **dbp)
157 {
158 uint64_t blkid;
159 dmu_buf_impl_t *db;
160
161 blkid = dbuf_whichblock(dn, 0, offset);
162 rw_enter(&dn->dn_struct_rwlock, RW_READER);
163 db = dbuf_hold(dn, blkid, tag);
164 rw_exit(&dn->dn_struct_rwlock);
165
166 if (db == NULL) {
167 *dbp = NULL;
168 return (SET_ERROR(EIO));
169 }
170
171 *dbp = &db->db;
172 return (0);
173 }
174 int
175 dmu_buf_hold_noread(objset_t *os, uint64_t object, uint64_t offset,
176 void *tag, dmu_buf_t **dbp)
177 {
178 dnode_t *dn;
179 uint64_t blkid;
180 dmu_buf_impl_t *db;
181 int err;
182
183 err = dnode_hold(os, object, FTAG, &dn);
184 if (err)
185 return (err);
186 blkid = dbuf_whichblock(dn, 0, offset);
187 rw_enter(&dn->dn_struct_rwlock, RW_READER);
188 db = dbuf_hold(dn, blkid, tag);
189 rw_exit(&dn->dn_struct_rwlock);
190 dnode_rele(dn, FTAG);
191
192 if (db == NULL) {
193 *dbp = NULL;
194 return (SET_ERROR(EIO));
195 }
196
197 *dbp = &db->db;
198 return (err);
199 }
200
201 int
202 dmu_buf_hold_by_dnode(dnode_t *dn, uint64_t offset,
203 void *tag, dmu_buf_t **dbp, int flags)
204 {
205 int err;
206 int db_flags = DB_RF_CANFAIL;
207
208 if (flags & DMU_READ_NO_PREFETCH)
209 db_flags |= DB_RF_NOPREFETCH;
210 if (flags & DMU_READ_NO_DECRYPT)
211 db_flags |= DB_RF_NO_DECRYPT;
212
213 err = dmu_buf_hold_noread_by_dnode(dn, offset, tag, dbp);
214 if (err == 0) {
215 dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp);
216 err = dbuf_read(db, NULL, db_flags);
217 if (err != 0) {
218 dbuf_rele(db, tag);
219 *dbp = NULL;
220 }
221 }
222
223 return (err);
224 }
225
226 int
227 dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset,
228 void *tag, dmu_buf_t **dbp, int flags)
229 {
230 int err;
231 int db_flags = DB_RF_CANFAIL;
232
233 if (flags & DMU_READ_NO_PREFETCH)
234 db_flags |= DB_RF_NOPREFETCH;
235 if (flags & DMU_READ_NO_DECRYPT)
236 db_flags |= DB_RF_NO_DECRYPT;
237
238 err = dmu_buf_hold_noread(os, object, offset, tag, dbp);
239 if (err == 0) {
240 dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp);
241 err = dbuf_read(db, NULL, db_flags);
242 if (err != 0) {
243 dbuf_rele(db, tag);
244 *dbp = NULL;
245 }
246 }
247
248 return (err);
249 }
250
251 int
252 dmu_bonus_max(void)
253 {
254 return (DN_OLD_MAX_BONUSLEN);
255 }
256
257 int
258 dmu_set_bonus(dmu_buf_t *db_fake, int newsize, dmu_tx_t *tx)
259 {
260 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
261 dnode_t *dn;
262 int error;
263
264 DB_DNODE_ENTER(db);
265 dn = DB_DNODE(db);
266
267 if (dn->dn_bonus != db) {
268 error = SET_ERROR(EINVAL);
269 } else if (newsize < 0 || newsize > db_fake->db_size) {
270 error = SET_ERROR(EINVAL);
271 } else {
272 dnode_setbonuslen(dn, newsize, tx);
273 error = 0;
274 }
275
276 DB_DNODE_EXIT(db);
277 return (error);
278 }
279
280 int
281 dmu_set_bonustype(dmu_buf_t *db_fake, dmu_object_type_t type, dmu_tx_t *tx)
282 {
283 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
284 dnode_t *dn;
285 int error;
286
287 DB_DNODE_ENTER(db);
288 dn = DB_DNODE(db);
289
290 if (!DMU_OT_IS_VALID(type)) {
291 error = SET_ERROR(EINVAL);
292 } else if (dn->dn_bonus != db) {
293 error = SET_ERROR(EINVAL);
294 } else {
295 dnode_setbonus_type(dn, type, tx);
296 error = 0;
297 }
298
299 DB_DNODE_EXIT(db);
300 return (error);
301 }
302
303 dmu_object_type_t
304 dmu_get_bonustype(dmu_buf_t *db_fake)
305 {
306 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
307 dnode_t *dn;
308 dmu_object_type_t type;
309
310 DB_DNODE_ENTER(db);
311 dn = DB_DNODE(db);
312 type = dn->dn_bonustype;
313 DB_DNODE_EXIT(db);
314
315 return (type);
316 }
317
318 int
319 dmu_rm_spill(objset_t *os, uint64_t object, dmu_tx_t *tx)
320 {
321 dnode_t *dn;
322 int error;
323
324 error = dnode_hold(os, object, FTAG, &dn);
325 dbuf_rm_spill(dn, tx);
326 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
327 dnode_rm_spill(dn, tx);
328 rw_exit(&dn->dn_struct_rwlock);
329 dnode_rele(dn, FTAG);
330 return (error);
331 }
332
333 /*
334 * Lookup and hold the bonus buffer for the provided dnode. If the dnode
335 * has not yet been allocated a new bonus dbuf a will be allocated.
336 * Returns ENOENT, EIO, or 0.
337 */
338 int dmu_bonus_hold_by_dnode(dnode_t *dn, void *tag, dmu_buf_t **dbp,
339 uint32_t flags)
340 {
341 dmu_buf_impl_t *db;
342 int error;
343 uint32_t db_flags = DB_RF_MUST_SUCCEED;
344
345 if (flags & DMU_READ_NO_PREFETCH)
346 db_flags |= DB_RF_NOPREFETCH;
347 if (flags & DMU_READ_NO_DECRYPT)
348 db_flags |= DB_RF_NO_DECRYPT;
349
350 rw_enter(&dn->dn_struct_rwlock, RW_READER);
351 if (dn->dn_bonus == NULL) {
352 rw_exit(&dn->dn_struct_rwlock);
353 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
354 if (dn->dn_bonus == NULL)
355 dbuf_create_bonus(dn);
356 }
357 db = dn->dn_bonus;
358
359 /* as long as the bonus buf is held, the dnode will be held */
360 if (zfs_refcount_add(&db->db_holds, tag) == 1) {
361 VERIFY(dnode_add_ref(dn, db));
362 atomic_inc_32(&dn->dn_dbufs_count);
363 }
364
365 /*
366 * Wait to drop dn_struct_rwlock until after adding the bonus dbuf's
367 * hold and incrementing the dbuf count to ensure that dnode_move() sees
368 * a dnode hold for every dbuf.
369 */
370 rw_exit(&dn->dn_struct_rwlock);
371
372 error = dbuf_read(db, NULL, db_flags);
373 if (error) {
374 dnode_evict_bonus(dn);
375 dbuf_rele(db, tag);
376 *dbp = NULL;
377 return (error);
378 }
379
380 *dbp = &db->db;
381 return (0);
382 }
383
384 int
385 dmu_bonus_hold(objset_t *os, uint64_t object, void *tag, dmu_buf_t **dbp)
386 {
387 dnode_t *dn;
388 int error;
389
390 error = dnode_hold(os, object, FTAG, &dn);
391 if (error)
392 return (error);
393
394 error = dmu_bonus_hold_by_dnode(dn, tag, dbp, DMU_READ_NO_PREFETCH);
395 dnode_rele(dn, FTAG);
396
397 return (error);
398 }
399
400 /*
401 * returns ENOENT, EIO, or 0.
402 *
403 * This interface will allocate a blank spill dbuf when a spill blk
404 * doesn't already exist on the dnode.
405 *
406 * if you only want to find an already existing spill db, then
407 * dmu_spill_hold_existing() should be used.
408 */
409 int
410 dmu_spill_hold_by_dnode(dnode_t *dn, uint32_t flags, void *tag, dmu_buf_t **dbp)
411 {
412 dmu_buf_impl_t *db = NULL;
413 int err;
414
415 if ((flags & DB_RF_HAVESTRUCT) == 0)
416 rw_enter(&dn->dn_struct_rwlock, RW_READER);
417
418 db = dbuf_hold(dn, DMU_SPILL_BLKID, tag);
419
420 if ((flags & DB_RF_HAVESTRUCT) == 0)
421 rw_exit(&dn->dn_struct_rwlock);
422
423 if (db == NULL) {
424 *dbp = NULL;
425 return (SET_ERROR(EIO));
426 }
427 err = dbuf_read(db, NULL, flags);
428 if (err == 0)
429 *dbp = &db->db;
430 else {
431 dbuf_rele(db, tag);
432 *dbp = NULL;
433 }
434 return (err);
435 }
436
437 int
438 dmu_spill_hold_existing(dmu_buf_t *bonus, void *tag, dmu_buf_t **dbp)
439 {
440 dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus;
441 dnode_t *dn;
442 int err;
443
444 DB_DNODE_ENTER(db);
445 dn = DB_DNODE(db);
446
447 if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_SA) {
448 err = SET_ERROR(EINVAL);
449 } else {
450 rw_enter(&dn->dn_struct_rwlock, RW_READER);
451
452 if (!dn->dn_have_spill) {
453 err = SET_ERROR(ENOENT);
454 } else {
455 err = dmu_spill_hold_by_dnode(dn,
456 DB_RF_HAVESTRUCT | DB_RF_CANFAIL, tag, dbp);
457 }
458
459 rw_exit(&dn->dn_struct_rwlock);
460 }
461
462 DB_DNODE_EXIT(db);
463 return (err);
464 }
465
466 int
467 dmu_spill_hold_by_bonus(dmu_buf_t *bonus, uint32_t flags, void *tag,
468 dmu_buf_t **dbp)
469 {
470 dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus;
471 dnode_t *dn;
472 int err;
473 uint32_t db_flags = DB_RF_CANFAIL;
474
475 if (flags & DMU_READ_NO_DECRYPT)
476 db_flags |= DB_RF_NO_DECRYPT;
477
478 DB_DNODE_ENTER(db);
479 dn = DB_DNODE(db);
480 err = dmu_spill_hold_by_dnode(dn, db_flags, tag, dbp);
481 DB_DNODE_EXIT(db);
482
483 return (err);
484 }
485
486 /*
487 * Note: longer-term, we should modify all of the dmu_buf_*() interfaces
488 * to take a held dnode rather than <os, object> -- the lookup is wasteful,
489 * and can induce severe lock contention when writing to several files
490 * whose dnodes are in the same block.
491 */
492 static int
493 dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length,
494 boolean_t read, void *tag, int *numbufsp, dmu_buf_t ***dbpp, uint32_t flags)
495 {
496 dmu_buf_t **dbp;
497 uint64_t blkid, nblks, i;
498 uint32_t dbuf_flags;
499 int err;
500 zio_t *zio;
501
502 ASSERT(length <= DMU_MAX_ACCESS);
503
504 /*
505 * Note: We directly notify the prefetch code of this read, so that
506 * we can tell it about the multi-block read. dbuf_read() only knows
507 * about the one block it is accessing.
508 */
509 dbuf_flags = DB_RF_CANFAIL | DB_RF_NEVERWAIT | DB_RF_HAVESTRUCT |
510 DB_RF_NOPREFETCH;
511
512 rw_enter(&dn->dn_struct_rwlock, RW_READER);
513 if (dn->dn_datablkshift) {
514 int blkshift = dn->dn_datablkshift;
515 nblks = (P2ROUNDUP(offset + length, 1ULL << blkshift) -
516 P2ALIGN(offset, 1ULL << blkshift)) >> blkshift;
517 } else {
518 if (offset + length > dn->dn_datablksz) {
519 zfs_panic_recover("zfs: accessing past end of object "
520 "%llx/%llx (size=%u access=%llu+%llu)",
521 (longlong_t)dn->dn_objset->
522 os_dsl_dataset->ds_object,
523 (longlong_t)dn->dn_object, dn->dn_datablksz,
524 (longlong_t)offset, (longlong_t)length);
525 rw_exit(&dn->dn_struct_rwlock);
526 return (SET_ERROR(EIO));
527 }
528 nblks = 1;
529 }
530 dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_SLEEP);
531
532 zio = zio_root(dn->dn_objset->os_spa, NULL, NULL, ZIO_FLAG_CANFAIL);
533 blkid = dbuf_whichblock(dn, 0, offset);
534 for (i = 0; i < nblks; i++) {
535 dmu_buf_impl_t *db = dbuf_hold(dn, blkid + i, tag);
536 if (db == NULL) {
537 rw_exit(&dn->dn_struct_rwlock);
538 dmu_buf_rele_array(dbp, nblks, tag);
539 zio_nowait(zio);
540 return (SET_ERROR(EIO));
541 }
542
543 /* initiate async i/o */
544 if (read)
545 (void) dbuf_read(db, zio, dbuf_flags);
546 dbp[i] = &db->db;
547 }
548
549 if ((flags & DMU_READ_NO_PREFETCH) == 0 &&
550 DNODE_META_IS_CACHEABLE(dn) && length <= zfetch_array_rd_sz) {
551 dmu_zfetch(&dn->dn_zfetch, blkid, nblks,
552 read && DNODE_IS_CACHEABLE(dn));
553 }
554 rw_exit(&dn->dn_struct_rwlock);
555
556 /* wait for async i/o */
557 err = zio_wait(zio);
558 if (err) {
559 dmu_buf_rele_array(dbp, nblks, tag);
560 return (err);
561 }
562
563 /* wait for other io to complete */
564 if (read) {
565 for (i = 0; i < nblks; i++) {
566 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp[i];
567 mutex_enter(&db->db_mtx);
568 while (db->db_state == DB_READ ||
569 db->db_state == DB_FILL)
570 cv_wait(&db->db_changed, &db->db_mtx);
571 if (db->db_state == DB_UNCACHED)
572 err = SET_ERROR(EIO);
573 mutex_exit(&db->db_mtx);
574 if (err) {
575 dmu_buf_rele_array(dbp, nblks, tag);
576 return (err);
577 }
578 }
579 }
580
581 *numbufsp = nblks;
582 *dbpp = dbp;
583 return (0);
584 }
585
586 static int
587 dmu_buf_hold_array(objset_t *os, uint64_t object, uint64_t offset,
588 uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp)
589 {
590 dnode_t *dn;
591 int err;
592
593 err = dnode_hold(os, object, FTAG, &dn);
594 if (err)
595 return (err);
596
597 err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
598 numbufsp, dbpp, DMU_READ_PREFETCH);
599
600 dnode_rele(dn, FTAG);
601
602 return (err);
603 }
604
605 int
606 dmu_buf_hold_array_by_bonus(dmu_buf_t *db_fake, uint64_t offset,
607 uint64_t length, boolean_t read, void *tag, int *numbufsp,
608 dmu_buf_t ***dbpp)
609 {
610 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
611 dnode_t *dn;
612 int err;
613
614 DB_DNODE_ENTER(db);
615 dn = DB_DNODE(db);
616 err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag,
617 numbufsp, dbpp, DMU_READ_PREFETCH);
618 DB_DNODE_EXIT(db);
619
620 return (err);
621 }
622
623 void
624 dmu_buf_rele_array(dmu_buf_t **dbp_fake, int numbufs, void *tag)
625 {
626 int i;
627 dmu_buf_impl_t **dbp = (dmu_buf_impl_t **)dbp_fake;
628
629 if (numbufs == 0)
630 return;
631
632 for (i = 0; i < numbufs; i++) {
633 if (dbp[i])
634 dbuf_rele(dbp[i], tag);
635 }
636
637 kmem_free(dbp, sizeof (dmu_buf_t *) * numbufs);
638 }
639
640 /*
641 * Issue prefetch i/os for the given blocks. If level is greater than 0, the
642 * indirect blocks prefeteched will be those that point to the blocks containing
643 * the data starting at offset, and continuing to offset + len.
644 *
645 * Note that if the indirect blocks above the blocks being prefetched are not
646 * in cache, they will be asychronously read in.
647 */
648 void
649 dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset,
650 uint64_t len, zio_priority_t pri)
651 {
652 dnode_t *dn;
653 uint64_t blkid;
654 int nblks, err;
655
656 if (len == 0) { /* they're interested in the bonus buffer */
657 dn = DMU_META_DNODE(os);
658
659 if (object == 0 || object >= DN_MAX_OBJECT)
660 return;
661
662 rw_enter(&dn->dn_struct_rwlock, RW_READER);
663 blkid = dbuf_whichblock(dn, level,
664 object * sizeof (dnode_phys_t));
665 dbuf_prefetch(dn, level, blkid, pri, 0);
666 rw_exit(&dn->dn_struct_rwlock);
667 return;
668 }
669
670 /*
671 * XXX - Note, if the dnode for the requested object is not
672 * already cached, we will do a *synchronous* read in the
673 * dnode_hold() call. The same is true for any indirects.
674 */
675 err = dnode_hold(os, object, FTAG, &dn);
676 if (err != 0)
677 return;
678
679 rw_enter(&dn->dn_struct_rwlock, RW_READER);
680 /*
681 * offset + len - 1 is the last byte we want to prefetch for, and offset
682 * is the first. Then dbuf_whichblk(dn, level, off + len - 1) is the
683 * last block we want to prefetch, and dbuf_whichblock(dn, level,
684 * offset) is the first. Then the number we need to prefetch is the
685 * last - first + 1.
686 */
687 if (level > 0 || dn->dn_datablkshift != 0) {
688 nblks = dbuf_whichblock(dn, level, offset + len - 1) -
689 dbuf_whichblock(dn, level, offset) + 1;
690 } else {
691 nblks = (offset < dn->dn_datablksz);
692 }
693
694 if (nblks != 0) {
695 blkid = dbuf_whichblock(dn, level, offset);
696 for (int i = 0; i < nblks; i++)
697 dbuf_prefetch(dn, level, blkid + i, pri, 0);
698 }
699
700 rw_exit(&dn->dn_struct_rwlock);
701
702 dnode_rele(dn, FTAG);
703 }
704
705 /*
706 * Get the next "chunk" of file data to free. We traverse the file from
707 * the end so that the file gets shorter over time (if we crashes in the
708 * middle, this will leave us in a better state). We find allocated file
709 * data by simply searching the allocated level 1 indirects.
710 *
711 * On input, *start should be the first offset that does not need to be
712 * freed (e.g. "offset + length"). On return, *start will be the first
713 * offset that should be freed and l1blks is set to the number of level 1
714 * indirect blocks found within the chunk.
715 */
716 static int
717 get_next_chunk(dnode_t *dn, uint64_t *start, uint64_t minimum, uint64_t *l1blks)
718 {
719 uint64_t blks;
720 uint64_t maxblks = DMU_MAX_ACCESS >> (dn->dn_indblkshift + 1);
721 /* bytes of data covered by a level-1 indirect block */
722 uint64_t iblkrange =
723 dn->dn_datablksz * EPB(dn->dn_indblkshift, SPA_BLKPTRSHIFT);
724
725 ASSERT3U(minimum, <=, *start);
726
727 /*
728 * Check if we can free the entire range assuming that all of the
729 * L1 blocks in this range have data. If we can, we use this
730 * worst case value as an estimate so we can avoid having to look
731 * at the object's actual data.
732 */
733 uint64_t total_l1blks =
734 (roundup(*start, iblkrange) - (minimum / iblkrange * iblkrange)) /
735 iblkrange;
736 if (total_l1blks <= maxblks) {
737 *l1blks = total_l1blks;
738 *start = minimum;
739 return (0);
740 }
741 ASSERT(ISP2(iblkrange));
742
743 for (blks = 0; *start > minimum && blks < maxblks; blks++) {
744 int err;
745
746 /*
747 * dnode_next_offset(BACKWARDS) will find an allocated L1
748 * indirect block at or before the input offset. We must
749 * decrement *start so that it is at the end of the region
750 * to search.
751 */
752 (*start)--;
753
754 err = dnode_next_offset(dn,
755 DNODE_FIND_BACKWARDS, start, 2, 1, 0);
756
757 /* if there are no indirect blocks before start, we are done */
758 if (err == ESRCH) {
759 *start = minimum;
760 break;
761 } else if (err != 0) {
762 *l1blks = blks;
763 return (err);
764 }
765
766 /* set start to the beginning of this L1 indirect */
767 *start = P2ALIGN(*start, iblkrange);
768 }
769 if (*start < minimum)
770 *start = minimum;
771 *l1blks = blks;
772
773 return (0);
774 }
775
776 /*
777 * If this objset is of type OST_ZFS return true if vfs's unmounted flag is set,
778 * otherwise return false.
779 * Used below in dmu_free_long_range_impl() to enable abort when unmounting
780 */
781 /*ARGSUSED*/
782 static boolean_t
783 dmu_objset_zfs_unmounting(objset_t *os)
784 {
785 #ifdef _KERNEL
786 if (dmu_objset_type(os) == DMU_OST_ZFS)
787 return (zfs_get_vfs_flag_unmounted(os));
788 #endif
789 return (B_FALSE);
790 }
791
792 static int
793 dmu_free_long_range_impl(objset_t *os, dnode_t *dn, uint64_t offset,
794 uint64_t length)
795 {
796 uint64_t object_size;
797 int err;
798 uint64_t dirty_frees_threshold;
799 dsl_pool_t *dp = dmu_objset_pool(os);
800
801 if (dn == NULL)
802 return (SET_ERROR(EINVAL));
803
804 object_size = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
805 if (offset >= object_size)
806 return (0);
807
808 if (zfs_per_txg_dirty_frees_percent <= 100)
809 dirty_frees_threshold =
810 zfs_per_txg_dirty_frees_percent * zfs_dirty_data_max / 100;
811 else
812 dirty_frees_threshold = zfs_dirty_data_max / 20;
813
814 if (length == DMU_OBJECT_END || offset + length > object_size)
815 length = object_size - offset;
816
817 while (length != 0) {
818 uint64_t chunk_end, chunk_begin, chunk_len;
819 uint64_t l1blks;
820 dmu_tx_t *tx;
821
822 if (dmu_objset_zfs_unmounting(dn->dn_objset))
823 return (SET_ERROR(EINTR));
824
825 chunk_end = chunk_begin = offset + length;
826
827 /* move chunk_begin backwards to the beginning of this chunk */
828 err = get_next_chunk(dn, &chunk_begin, offset, &l1blks);
829 if (err)
830 return (err);
831 ASSERT3U(chunk_begin, >=, offset);
832 ASSERT3U(chunk_begin, <=, chunk_end);
833
834 chunk_len = chunk_end - chunk_begin;
835
836 tx = dmu_tx_create(os);
837 dmu_tx_hold_free(tx, dn->dn_object, chunk_begin, chunk_len);
838
839 /*
840 * Mark this transaction as typically resulting in a net
841 * reduction in space used.
842 */
843 dmu_tx_mark_netfree(tx);
844 err = dmu_tx_assign(tx, TXG_WAIT);
845 if (err) {
846 dmu_tx_abort(tx);
847 return (err);
848 }
849
850 uint64_t txg = dmu_tx_get_txg(tx);
851
852 mutex_enter(&dp->dp_lock);
853 uint64_t long_free_dirty =
854 dp->dp_long_free_dirty_pertxg[txg & TXG_MASK];
855 mutex_exit(&dp->dp_lock);
856
857 /*
858 * To avoid filling up a TXG with just frees, wait for
859 * the next TXG to open before freeing more chunks if
860 * we have reached the threshold of frees.
861 */
862 if (dirty_frees_threshold != 0 &&
863 long_free_dirty >= dirty_frees_threshold) {
864 DMU_TX_STAT_BUMP(dmu_tx_dirty_frees_delay);
865 dmu_tx_commit(tx);
866 txg_wait_open(dp, 0, B_TRUE);
867 continue;
868 }
869
870 /*
871 * In order to prevent unnecessary write throttling, for each
872 * TXG, we track the cumulative size of L1 blocks being dirtied
873 * in dnode_free_range() below. We compare this number to a
874 * tunable threshold, past which we prevent new L1 dirty freeing
875 * blocks from being added into the open TXG. See
876 * dmu_free_long_range_impl() for details. The threshold
877 * prevents write throttle activation due to dirty freeing L1
878 * blocks taking up a large percentage of zfs_dirty_data_max.
879 */
880 mutex_enter(&dp->dp_lock);
881 dp->dp_long_free_dirty_pertxg[txg & TXG_MASK] +=
882 l1blks << dn->dn_indblkshift;
883 mutex_exit(&dp->dp_lock);
884 DTRACE_PROBE3(free__long__range,
885 uint64_t, long_free_dirty, uint64_t, chunk_len,
886 uint64_t, txg);
887 dnode_free_range(dn, chunk_begin, chunk_len, tx);
888
889 dmu_tx_commit(tx);
890
891 length -= chunk_len;
892 }
893 return (0);
894 }
895
896 int
897 dmu_free_long_range(objset_t *os, uint64_t object,
898 uint64_t offset, uint64_t length)
899 {
900 dnode_t *dn;
901 int err;
902
903 err = dnode_hold(os, object, FTAG, &dn);
904 if (err != 0)
905 return (err);
906 err = dmu_free_long_range_impl(os, dn, offset, length);
907
908 /*
909 * It is important to zero out the maxblkid when freeing the entire
910 * file, so that (a) subsequent calls to dmu_free_long_range_impl()
911 * will take the fast path, and (b) dnode_reallocate() can verify
912 * that the entire file has been freed.
913 */
914 if (err == 0 && offset == 0 && length == DMU_OBJECT_END)
915 dn->dn_maxblkid = 0;
916
917 dnode_rele(dn, FTAG);
918 return (err);
919 }
920
921 int
922 dmu_free_long_object(objset_t *os, uint64_t object)
923 {
924 dmu_tx_t *tx;
925 int err;
926
927 err = dmu_free_long_range(os, object, 0, DMU_OBJECT_END);
928 if (err != 0)
929 return (err);
930
931 tx = dmu_tx_create(os);
932 dmu_tx_hold_bonus(tx, object);
933 dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END);
934 dmu_tx_mark_netfree(tx);
935 err = dmu_tx_assign(tx, TXG_WAIT);
936 if (err == 0) {
937 if (err == 0)
938 err = dmu_object_free(os, object, tx);
939
940 dmu_tx_commit(tx);
941 } else {
942 dmu_tx_abort(tx);
943 }
944
945 return (err);
946 }
947
948 int
949 dmu_free_range(objset_t *os, uint64_t object, uint64_t offset,
950 uint64_t size, dmu_tx_t *tx)
951 {
952 dnode_t *dn;
953 int err = dnode_hold(os, object, FTAG, &dn);
954 if (err)
955 return (err);
956 ASSERT(offset < UINT64_MAX);
957 ASSERT(size == DMU_OBJECT_END || size <= UINT64_MAX - offset);
958 dnode_free_range(dn, offset, size, tx);
959 dnode_rele(dn, FTAG);
960 return (0);
961 }
962
963 static int
964 dmu_read_impl(dnode_t *dn, uint64_t offset, uint64_t size,
965 void *buf, uint32_t flags)
966 {
967 dmu_buf_t **dbp;
968 int numbufs, err = 0;
969
970 /*
971 * Deal with odd block sizes, where there can't be data past the first
972 * block. If we ever do the tail block optimization, we will need to
973 * handle that here as well.
974 */
975 if (dn->dn_maxblkid == 0) {
976 uint64_t newsz = offset > dn->dn_datablksz ? 0 :
977 MIN(size, dn->dn_datablksz - offset);
978 bzero((char *)buf + newsz, size - newsz);
979 size = newsz;
980 }
981
982 while (size > 0) {
983 uint64_t mylen = MIN(size, DMU_MAX_ACCESS / 2);
984 int i;
985
986 /*
987 * NB: we could do this block-at-a-time, but it's nice
988 * to be reading in parallel.
989 */
990 err = dmu_buf_hold_array_by_dnode(dn, offset, mylen,
991 TRUE, FTAG, &numbufs, &dbp, flags);
992 if (err)
993 break;
994
995 for (i = 0; i < numbufs; i++) {
996 uint64_t tocpy;
997 int64_t bufoff;
998 dmu_buf_t *db = dbp[i];
999
1000 ASSERT(size > 0);
1001
1002 bufoff = offset - db->db_offset;
1003 tocpy = MIN(db->db_size - bufoff, size);
1004
1005 (void) memcpy(buf, (char *)db->db_data + bufoff, tocpy);
1006
1007 offset += tocpy;
1008 size -= tocpy;
1009 buf = (char *)buf + tocpy;
1010 }
1011 dmu_buf_rele_array(dbp, numbufs, FTAG);
1012 }
1013 return (err);
1014 }
1015
1016 int
1017 dmu_read(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
1018 void *buf, uint32_t flags)
1019 {
1020 dnode_t *dn;
1021 int err;
1022
1023 err = dnode_hold(os, object, FTAG, &dn);
1024 if (err != 0)
1025 return (err);
1026
1027 err = dmu_read_impl(dn, offset, size, buf, flags);
1028 dnode_rele(dn, FTAG);
1029 return (err);
1030 }
1031
1032 int
1033 dmu_read_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size, void *buf,
1034 uint32_t flags)
1035 {
1036 return (dmu_read_impl(dn, offset, size, buf, flags));
1037 }
1038
1039 static void
1040 dmu_write_impl(dmu_buf_t **dbp, int numbufs, uint64_t offset, uint64_t size,
1041 const void *buf, dmu_tx_t *tx)
1042 {
1043 int i;
1044
1045 for (i = 0; i < numbufs; i++) {
1046 uint64_t tocpy;
1047 int64_t bufoff;
1048 dmu_buf_t *db = dbp[i];
1049
1050 ASSERT(size > 0);
1051
1052 bufoff = offset - db->db_offset;
1053 tocpy = MIN(db->db_size - bufoff, size);
1054
1055 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
1056
1057 if (tocpy == db->db_size)
1058 dmu_buf_will_fill(db, tx);
1059 else
1060 dmu_buf_will_dirty(db, tx);
1061
1062 (void) memcpy((char *)db->db_data + bufoff, buf, tocpy);
1063
1064 if (tocpy == db->db_size)
1065 dmu_buf_fill_done(db, tx);
1066
1067 offset += tocpy;
1068 size -= tocpy;
1069 buf = (char *)buf + tocpy;
1070 }
1071 }
1072
1073 void
1074 dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
1075 const void *buf, dmu_tx_t *tx)
1076 {
1077 dmu_buf_t **dbp;
1078 int numbufs;
1079
1080 if (size == 0)
1081 return;
1082
1083 VERIFY0(dmu_buf_hold_array(os, object, offset, size,
1084 FALSE, FTAG, &numbufs, &dbp));
1085 dmu_write_impl(dbp, numbufs, offset, size, buf, tx);
1086 dmu_buf_rele_array(dbp, numbufs, FTAG);
1087 }
1088
1089 void
1090 dmu_write_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size,
1091 const void *buf, dmu_tx_t *tx)
1092 {
1093 dmu_buf_t **dbp;
1094 int numbufs;
1095
1096 if (size == 0)
1097 return;
1098
1099 VERIFY0(dmu_buf_hold_array_by_dnode(dn, offset, size,
1100 FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH));
1101 dmu_write_impl(dbp, numbufs, offset, size, buf, tx);
1102 dmu_buf_rele_array(dbp, numbufs, FTAG);
1103 }
1104
1105 static int
1106 dmu_object_remap_one_indirect(objset_t *os, dnode_t *dn,
1107 uint64_t last_removal_txg, uint64_t offset)
1108 {
1109 uint64_t l1blkid = dbuf_whichblock(dn, 1, offset);
1110 dnode_t *dn_tx;
1111 int err = 0;
1112
1113 rw_enter(&dn->dn_struct_rwlock, RW_READER);
1114 dmu_buf_impl_t *dbuf = dbuf_hold_level(dn, 1, l1blkid, FTAG);
1115 ASSERT3P(dbuf, !=, NULL);
1116
1117 /*
1118 * If the block hasn't been written yet, this default will ensure
1119 * we don't try to remap it.
1120 */
1121 uint64_t birth = UINT64_MAX;
1122 ASSERT3U(last_removal_txg, !=, UINT64_MAX);
1123 if (dbuf->db_blkptr != NULL)
1124 birth = dbuf->db_blkptr->blk_birth;
1125 rw_exit(&dn->dn_struct_rwlock);
1126
1127 /*
1128 * If this L1 was already written after the last removal, then we've
1129 * already tried to remap it. An additional hold is taken after the
1130 * dmu_tx_assign() to handle the case where the dnode is freed while
1131 * waiting for the next open txg.
1132 */
1133 if (birth <= last_removal_txg &&
1134 dbuf_read(dbuf, NULL, DB_RF_MUST_SUCCEED) == 0 &&
1135 dbuf_can_remap(dbuf)) {
1136 dmu_tx_t *tx = dmu_tx_create(os);
1137 dmu_tx_hold_remap_l1indirect(tx, dn->dn_object);
1138 err = dmu_tx_assign(tx, TXG_WAIT);
1139 if (err == 0) {
1140 err = dnode_hold(os, dn->dn_object, FTAG, &dn_tx);
1141 if (err == 0) {
1142 (void) dbuf_dirty(dbuf, tx);
1143 dnode_rele(dn_tx, FTAG);
1144 }
1145 dmu_tx_commit(tx);
1146 } else {
1147 dmu_tx_abort(tx);
1148 }
1149 }
1150
1151 dbuf_rele(dbuf, FTAG);
1152
1153 delay(MSEC_TO_TICK(zfs_object_remap_one_indirect_delay_ms));
1154
1155 return (err);
1156 }
1157
1158 /*
1159 * Remap all blockpointers in the object, if possible, so that they reference
1160 * only concrete vdevs.
1161 *
1162 * To do this, iterate over the L0 blockpointers and remap any that reference
1163 * an indirect vdev. Note that we only examine L0 blockpointers; since we
1164 * cannot guarantee that we can remap all blockpointer anyways (due to split
1165 * blocks), we do not want to make the code unnecessarily complicated to
1166 * catch the unlikely case that there is an L1 block on an indirect vdev that
1167 * contains no indirect blockpointers.
1168 */
1169 int
1170 dmu_object_remap_indirects(objset_t *os, uint64_t object,
1171 uint64_t last_removal_txg)
1172 {
1173 uint64_t offset, l1span;
1174 int err;
1175 dnode_t *dn, *dn_tx;
1176
1177 err = dnode_hold(os, object, FTAG, &dn);
1178 if (err != 0) {
1179 return (err);
1180 }
1181
1182 if (dn->dn_nlevels <= 1) {
1183 if (issig(JUSTLOOKING) && issig(FORREAL)) {
1184 err = SET_ERROR(EINTR);
1185 }
1186
1187 /*
1188 * If the dnode has no indirect blocks, we cannot dirty them.
1189 * We still want to remap the blkptr(s) in the dnode if
1190 * appropriate, so mark it as dirty. An additional hold is
1191 * taken after the dmu_tx_assign() to handle the case where
1192 * the dnode is freed while waiting for the next open txg.
1193 */
1194 if (err == 0 && dnode_needs_remap(dn)) {
1195 dmu_tx_t *tx = dmu_tx_create(os);
1196 dmu_tx_hold_bonus(tx, object);
1197 err = dmu_tx_assign(tx, TXG_WAIT);
1198 if (err == 0) {
1199 err = dnode_hold(os, object, FTAG, &dn_tx);
1200 if (err == 0) {
1201 dnode_setdirty(dn_tx, tx);
1202 dnode_rele(dn_tx, FTAG);
1203 }
1204 dmu_tx_commit(tx);
1205 } else {
1206 dmu_tx_abort(tx);
1207 }
1208 }
1209
1210 dnode_rele(dn, FTAG);
1211 return (err);
1212 }
1213
1214 offset = 0;
1215 l1span = 1ULL << (dn->dn_indblkshift - SPA_BLKPTRSHIFT +
1216 dn->dn_datablkshift);
1217 /*
1218 * Find the next L1 indirect that is not a hole.
1219 */
1220 while (dnode_next_offset(dn, 0, &offset, 2, 1, 0) == 0) {
1221 if (issig(JUSTLOOKING) && issig(FORREAL)) {
1222 err = SET_ERROR(EINTR);
1223 break;
1224 }
1225 if ((err = dmu_object_remap_one_indirect(os, dn,
1226 last_removal_txg, offset)) != 0) {
1227 break;
1228 }
1229 offset += l1span;
1230 }
1231
1232 dnode_rele(dn, FTAG);
1233 return (err);
1234 }
1235
1236 void
1237 dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size,
1238 dmu_tx_t *tx)
1239 {
1240 dmu_buf_t **dbp;
1241 int numbufs, i;
1242
1243 if (size == 0)
1244 return;
1245
1246 VERIFY(0 == dmu_buf_hold_array(os, object, offset, size,
1247 FALSE, FTAG, &numbufs, &dbp));
1248
1249 for (i = 0; i < numbufs; i++) {
1250 dmu_buf_t *db = dbp[i];
1251
1252 dmu_buf_will_not_fill(db, tx);
1253 }
1254 dmu_buf_rele_array(dbp, numbufs, FTAG);
1255 }
1256
1257 void
1258 dmu_write_embedded(objset_t *os, uint64_t object, uint64_t offset,
1259 void *data, uint8_t etype, uint8_t comp, int uncompressed_size,
1260 int compressed_size, int byteorder, dmu_tx_t *tx)
1261 {
1262 dmu_buf_t *db;
1263
1264 ASSERT3U(etype, <, NUM_BP_EMBEDDED_TYPES);
1265 ASSERT3U(comp, <, ZIO_COMPRESS_FUNCTIONS);
1266 VERIFY0(dmu_buf_hold_noread(os, object, offset,
1267 FTAG, &db));
1268
1269 dmu_buf_write_embedded(db,
1270 data, (bp_embedded_type_t)etype, (enum zio_compress)comp,
1271 uncompressed_size, compressed_size, byteorder, tx);
1272
1273 dmu_buf_rele(db, FTAG);
1274 }
1275
1276 /*
1277 * DMU support for xuio
1278 */
1279 kstat_t *xuio_ksp = NULL;
1280
1281 typedef struct xuio_stats {
1282 /* loaned yet not returned arc_buf */
1283 kstat_named_t xuiostat_onloan_rbuf;
1284 kstat_named_t xuiostat_onloan_wbuf;
1285 /* whether a copy is made when loaning out a read buffer */
1286 kstat_named_t xuiostat_rbuf_copied;
1287 kstat_named_t xuiostat_rbuf_nocopy;
1288 /* whether a copy is made when assigning a write buffer */
1289 kstat_named_t xuiostat_wbuf_copied;
1290 kstat_named_t xuiostat_wbuf_nocopy;
1291 } xuio_stats_t;
1292
1293 static xuio_stats_t xuio_stats = {
1294 { "onloan_read_buf", KSTAT_DATA_UINT64 },
1295 { "onloan_write_buf", KSTAT_DATA_UINT64 },
1296 { "read_buf_copied", KSTAT_DATA_UINT64 },
1297 { "read_buf_nocopy", KSTAT_DATA_UINT64 },
1298 { "write_buf_copied", KSTAT_DATA_UINT64 },
1299 { "write_buf_nocopy", KSTAT_DATA_UINT64 }
1300 };
1301
1302 #define XUIOSTAT_INCR(stat, val) \
1303 atomic_add_64(&xuio_stats.stat.value.ui64, (val))
1304 #define XUIOSTAT_BUMP(stat) XUIOSTAT_INCR(stat, 1)
1305
1306 #ifdef HAVE_UIO_ZEROCOPY
1307 int
1308 dmu_xuio_init(xuio_t *xuio, int nblk)
1309 {
1310 dmu_xuio_t *priv;
1311 uio_t *uio = &xuio->xu_uio;
1312
1313 uio->uio_iovcnt = nblk;
1314 uio->uio_iov = kmem_zalloc(nblk * sizeof (iovec_t), KM_SLEEP);
1315
1316 priv = kmem_zalloc(sizeof (dmu_xuio_t), KM_SLEEP);
1317 priv->cnt = nblk;
1318 priv->bufs = kmem_zalloc(nblk * sizeof (arc_buf_t *), KM_SLEEP);
1319 priv->iovp = (iovec_t *)uio->uio_iov;
1320 XUIO_XUZC_PRIV(xuio) = priv;
1321
1322 if (XUIO_XUZC_RW(xuio) == UIO_READ)
1323 XUIOSTAT_INCR(xuiostat_onloan_rbuf, nblk);
1324 else
1325 XUIOSTAT_INCR(xuiostat_onloan_wbuf, nblk);
1326
1327 return (0);
1328 }
1329
1330 void
1331 dmu_xuio_fini(xuio_t *xuio)
1332 {
1333 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1334 int nblk = priv->cnt;
1335
1336 kmem_free(priv->iovp, nblk * sizeof (iovec_t));
1337 kmem_free(priv->bufs, nblk * sizeof (arc_buf_t *));
1338 kmem_free(priv, sizeof (dmu_xuio_t));
1339
1340 if (XUIO_XUZC_RW(xuio) == UIO_READ)
1341 XUIOSTAT_INCR(xuiostat_onloan_rbuf, -nblk);
1342 else
1343 XUIOSTAT_INCR(xuiostat_onloan_wbuf, -nblk);
1344 }
1345
1346 /*
1347 * Initialize iov[priv->next] and priv->bufs[priv->next] with { off, n, abuf }
1348 * and increase priv->next by 1.
1349 */
1350 int
1351 dmu_xuio_add(xuio_t *xuio, arc_buf_t *abuf, offset_t off, size_t n)
1352 {
1353 struct iovec *iov;
1354 uio_t *uio = &xuio->xu_uio;
1355 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1356 int i = priv->next++;
1357
1358 ASSERT(i < priv->cnt);
1359 ASSERT(off + n <= arc_buf_lsize(abuf));
1360 iov = (iovec_t *)uio->uio_iov + i;
1361 iov->iov_base = (char *)abuf->b_data + off;
1362 iov->iov_len = n;
1363 priv->bufs[i] = abuf;
1364 return (0);
1365 }
1366
1367 int
1368 dmu_xuio_cnt(xuio_t *xuio)
1369 {
1370 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1371 return (priv->cnt);
1372 }
1373
1374 arc_buf_t *
1375 dmu_xuio_arcbuf(xuio_t *xuio, int i)
1376 {
1377 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1378
1379 ASSERT(i < priv->cnt);
1380 return (priv->bufs[i]);
1381 }
1382
1383 void
1384 dmu_xuio_clear(xuio_t *xuio, int i)
1385 {
1386 dmu_xuio_t *priv = XUIO_XUZC_PRIV(xuio);
1387
1388 ASSERT(i < priv->cnt);
1389 priv->bufs[i] = NULL;
1390 }
1391 #endif /* HAVE_UIO_ZEROCOPY */
1392
1393 static void
1394 xuio_stat_init(void)
1395 {
1396 xuio_ksp = kstat_create("zfs", 0, "xuio_stats", "misc",
1397 KSTAT_TYPE_NAMED, sizeof (xuio_stats) / sizeof (kstat_named_t),
1398 KSTAT_FLAG_VIRTUAL);
1399 if (xuio_ksp != NULL) {
1400 xuio_ksp->ks_data = &xuio_stats;
1401 kstat_install(xuio_ksp);
1402 }
1403 }
1404
1405 static void
1406 xuio_stat_fini(void)
1407 {
1408 if (xuio_ksp != NULL) {
1409 kstat_delete(xuio_ksp);
1410 xuio_ksp = NULL;
1411 }
1412 }
1413
1414 void
1415 xuio_stat_wbuf_copied(void)
1416 {
1417 XUIOSTAT_BUMP(xuiostat_wbuf_copied);
1418 }
1419
1420 void
1421 xuio_stat_wbuf_nocopy(void)
1422 {
1423 XUIOSTAT_BUMP(xuiostat_wbuf_nocopy);
1424 }
1425
1426 #ifdef _KERNEL
1427 int
1428 dmu_read_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size)
1429 {
1430 dmu_buf_t **dbp;
1431 int numbufs, i, err;
1432 #ifdef HAVE_UIO_ZEROCOPY
1433 xuio_t *xuio = NULL;
1434 #endif
1435
1436 /*
1437 * NB: we could do this block-at-a-time, but it's nice
1438 * to be reading in parallel.
1439 */
1440 err = dmu_buf_hold_array_by_dnode(dn, uio->uio_loffset, size,
1441 TRUE, FTAG, &numbufs, &dbp, 0);
1442 if (err)
1443 return (err);
1444
1445 for (i = 0; i < numbufs; i++) {
1446 uint64_t tocpy;
1447 int64_t bufoff;
1448 dmu_buf_t *db = dbp[i];
1449
1450 ASSERT(size > 0);
1451
1452 bufoff = uio->uio_loffset - db->db_offset;
1453 tocpy = MIN(db->db_size - bufoff, size);
1454
1455 #ifdef HAVE_UIO_ZEROCOPY
1456 if (xuio) {
1457 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
1458 arc_buf_t *dbuf_abuf = dbi->db_buf;
1459 arc_buf_t *abuf = dbuf_loan_arcbuf(dbi);
1460 err = dmu_xuio_add(xuio, abuf, bufoff, tocpy);
1461 if (!err) {
1462 uio->uio_resid -= tocpy;
1463 uio->uio_loffset += tocpy;
1464 }
1465
1466 if (abuf == dbuf_abuf)
1467 XUIOSTAT_BUMP(xuiostat_rbuf_nocopy);
1468 else
1469 XUIOSTAT_BUMP(xuiostat_rbuf_copied);
1470 } else
1471 #endif
1472 err = uiomove((char *)db->db_data + bufoff, tocpy,
1473 UIO_READ, uio);
1474 if (err)
1475 break;
1476
1477 size -= tocpy;
1478 }
1479 dmu_buf_rele_array(dbp, numbufs, FTAG);
1480
1481 return (err);
1482 }
1483
1484 /*
1485 * Read 'size' bytes into the uio buffer.
1486 * From object zdb->db_object.
1487 * Starting at offset uio->uio_loffset.
1488 *
1489 * If the caller already has a dbuf in the target object
1490 * (e.g. its bonus buffer), this routine is faster than dmu_read_uio(),
1491 * because we don't have to find the dnode_t for the object.
1492 */
1493 int
1494 dmu_read_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size)
1495 {
1496 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb;
1497 dnode_t *dn;
1498 int err;
1499
1500 if (size == 0)
1501 return (0);
1502
1503 DB_DNODE_ENTER(db);
1504 dn = DB_DNODE(db);
1505 err = dmu_read_uio_dnode(dn, uio, size);
1506 DB_DNODE_EXIT(db);
1507
1508 return (err);
1509 }
1510
1511 /*
1512 * Read 'size' bytes into the uio buffer.
1513 * From the specified object
1514 * Starting at offset uio->uio_loffset.
1515 */
1516 int
1517 dmu_read_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size)
1518 {
1519 dnode_t *dn;
1520 int err;
1521
1522 if (size == 0)
1523 return (0);
1524
1525 err = dnode_hold(os, object, FTAG, &dn);
1526 if (err)
1527 return (err);
1528
1529 err = dmu_read_uio_dnode(dn, uio, size);
1530
1531 dnode_rele(dn, FTAG);
1532
1533 return (err);
1534 }
1535
1536 int
1537 dmu_write_uio_dnode(dnode_t *dn, uio_t *uio, uint64_t size, dmu_tx_t *tx)
1538 {
1539 dmu_buf_t **dbp;
1540 int numbufs;
1541 int err = 0;
1542 int i;
1543
1544 err = dmu_buf_hold_array_by_dnode(dn, uio->uio_loffset, size,
1545 FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH);
1546 if (err)
1547 return (err);
1548
1549 for (i = 0; i < numbufs; i++) {
1550 uint64_t tocpy;
1551 int64_t bufoff;
1552 dmu_buf_t *db = dbp[i];
1553
1554 ASSERT(size > 0);
1555
1556 bufoff = uio->uio_loffset - db->db_offset;
1557 tocpy = MIN(db->db_size - bufoff, size);
1558
1559 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size);
1560
1561 if (tocpy == db->db_size)
1562 dmu_buf_will_fill(db, tx);
1563 else
1564 dmu_buf_will_dirty(db, tx);
1565
1566 /*
1567 * XXX uiomove could block forever (eg.nfs-backed
1568 * pages). There needs to be a uiolockdown() function
1569 * to lock the pages in memory, so that uiomove won't
1570 * block.
1571 */
1572 err = uiomove((char *)db->db_data + bufoff, tocpy,
1573 UIO_WRITE, uio);
1574
1575 if (tocpy == db->db_size)
1576 dmu_buf_fill_done(db, tx);
1577
1578 if (err)
1579 break;
1580
1581 size -= tocpy;
1582 }
1583
1584 dmu_buf_rele_array(dbp, numbufs, FTAG);
1585 return (err);
1586 }
1587
1588 /*
1589 * Write 'size' bytes from the uio buffer.
1590 * To object zdb->db_object.
1591 * Starting at offset uio->uio_loffset.
1592 *
1593 * If the caller already has a dbuf in the target object
1594 * (e.g. its bonus buffer), this routine is faster than dmu_write_uio(),
1595 * because we don't have to find the dnode_t for the object.
1596 */
1597 int
1598 dmu_write_uio_dbuf(dmu_buf_t *zdb, uio_t *uio, uint64_t size,
1599 dmu_tx_t *tx)
1600 {
1601 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb;
1602 dnode_t *dn;
1603 int err;
1604
1605 if (size == 0)
1606 return (0);
1607
1608 DB_DNODE_ENTER(db);
1609 dn = DB_DNODE(db);
1610 err = dmu_write_uio_dnode(dn, uio, size, tx);
1611 DB_DNODE_EXIT(db);
1612
1613 return (err);
1614 }
1615
1616 /*
1617 * Write 'size' bytes from the uio buffer.
1618 * To the specified object.
1619 * Starting at offset uio->uio_loffset.
1620 */
1621 int
1622 dmu_write_uio(objset_t *os, uint64_t object, uio_t *uio, uint64_t size,
1623 dmu_tx_t *tx)
1624 {
1625 dnode_t *dn;
1626 int err;
1627
1628 if (size == 0)
1629 return (0);
1630
1631 err = dnode_hold(os, object, FTAG, &dn);
1632 if (err)
1633 return (err);
1634
1635 err = dmu_write_uio_dnode(dn, uio, size, tx);
1636
1637 dnode_rele(dn, FTAG);
1638
1639 return (err);
1640 }
1641 #endif /* _KERNEL */
1642
1643 /*
1644 * Allocate a loaned anonymous arc buffer.
1645 */
1646 arc_buf_t *
1647 dmu_request_arcbuf(dmu_buf_t *handle, int size)
1648 {
1649 dmu_buf_impl_t *db = (dmu_buf_impl_t *)handle;
1650
1651 return (arc_loan_buf(db->db_objset->os_spa, B_FALSE, size));
1652 }
1653
1654 /*
1655 * Free a loaned arc buffer.
1656 */
1657 void
1658 dmu_return_arcbuf(arc_buf_t *buf)
1659 {
1660 arc_return_buf(buf, FTAG);
1661 arc_buf_destroy(buf, FTAG);
1662 }
1663
1664 void
1665 dmu_copy_from_buf(objset_t *os, uint64_t object, uint64_t offset,
1666 dmu_buf_t *handle, dmu_tx_t *tx)
1667 {
1668 dmu_buf_t *dst_handle;
1669 dmu_buf_impl_t *dstdb;
1670 dmu_buf_impl_t *srcdb = (dmu_buf_impl_t *)handle;
1671 dmu_object_type_t type;
1672 arc_buf_t *abuf;
1673 uint64_t datalen;
1674 boolean_t byteorder;
1675 uint8_t salt[ZIO_DATA_SALT_LEN];
1676 uint8_t iv[ZIO_DATA_IV_LEN];
1677 uint8_t mac[ZIO_DATA_MAC_LEN];
1678
1679 ASSERT3P(srcdb->db_buf, !=, NULL);
1680
1681 /* hold the db that we want to write to */
1682 VERIFY0(dmu_buf_hold(os, object, offset, FTAG, &dst_handle,
1683 DMU_READ_NO_DECRYPT));
1684 dstdb = (dmu_buf_impl_t *)dst_handle;
1685 datalen = arc_buf_size(srcdb->db_buf);
1686
1687 DB_DNODE_ENTER(dstdb);
1688 type = DB_DNODE(dstdb)->dn_type;
1689 DB_DNODE_EXIT(dstdb);
1690
1691 /* allocated an arc buffer that matches the type of srcdb->db_buf */
1692 if (arc_is_encrypted(srcdb->db_buf)) {
1693 arc_get_raw_params(srcdb->db_buf, &byteorder, salt, iv, mac);
1694 abuf = arc_loan_raw_buf(os->os_spa, dmu_objset_id(os),
1695 byteorder, salt, iv, mac, type,
1696 datalen, arc_buf_lsize(srcdb->db_buf),
1697 arc_get_compression(srcdb->db_buf));
1698 } else {
1699 /* we won't get a compressed db back from dmu_buf_hold() */
1700 ASSERT3U(arc_get_compression(srcdb->db_buf),
1701 ==, ZIO_COMPRESS_OFF);
1702 abuf = arc_loan_buf(os->os_spa,
1703 DMU_OT_IS_METADATA(type), datalen);
1704 }
1705
1706 ASSERT3U(datalen, ==, arc_buf_size(abuf));
1707
1708 /* copy the data to the new buffer and assign it to the dstdb */
1709 bcopy(srcdb->db_buf->b_data, abuf->b_data, datalen);
1710 dbuf_assign_arcbuf(dstdb, abuf, tx);
1711 dmu_buf_rele(dst_handle, FTAG);
1712 }
1713
1714 /*
1715 * When possible directly assign passed loaned arc buffer to a dbuf.
1716 * If this is not possible copy the contents of passed arc buf via
1717 * dmu_write().
1718 */
1719 int
1720 dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset, arc_buf_t *buf,
1721 dmu_tx_t *tx)
1722 {
1723 dmu_buf_impl_t *db;
1724 objset_t *os = dn->dn_objset;
1725 uint64_t object = dn->dn_object;
1726 uint32_t blksz = (uint32_t)arc_buf_lsize(buf);
1727 uint64_t blkid;
1728
1729 rw_enter(&dn->dn_struct_rwlock, RW_READER);
1730 blkid = dbuf_whichblock(dn, 0, offset);
1731 db = dbuf_hold(dn, blkid, FTAG);
1732 if (db == NULL)
1733 return (SET_ERROR(EIO));
1734 rw_exit(&dn->dn_struct_rwlock);
1735
1736 /*
1737 * We can only assign if the offset is aligned, the arc buf is the
1738 * same size as the dbuf, and the dbuf is not metadata.
1739 */
1740 if (offset == db->db.db_offset && blksz == db->db.db_size) {
1741 dbuf_assign_arcbuf(db, buf, tx);
1742 dbuf_rele(db, FTAG);
1743 } else {
1744 /* compressed bufs must always be assignable to their dbuf */
1745 ASSERT3U(arc_get_compression(buf), ==, ZIO_COMPRESS_OFF);
1746 ASSERT(!(buf->b_flags & ARC_BUF_FLAG_COMPRESSED));
1747
1748 dbuf_rele(db, FTAG);
1749 dmu_write(os, object, offset, blksz, buf->b_data, tx);
1750 dmu_return_arcbuf(buf);
1751 XUIOSTAT_BUMP(xuiostat_wbuf_copied);
1752 }
1753
1754 return (0);
1755 }
1756
1757 int
1758 dmu_assign_arcbuf_by_dbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf,
1759 dmu_tx_t *tx)
1760 {
1761 int err;
1762 dmu_buf_impl_t *dbuf = (dmu_buf_impl_t *)handle;
1763
1764 DB_DNODE_ENTER(dbuf);
1765 err = dmu_assign_arcbuf_by_dnode(DB_DNODE(dbuf), offset, buf, tx);
1766 DB_DNODE_EXIT(dbuf);
1767
1768 return (err);
1769 }
1770
1771 typedef struct {
1772 dbuf_dirty_record_t *dsa_dr;
1773 dmu_sync_cb_t *dsa_done;
1774 zgd_t *dsa_zgd;
1775 dmu_tx_t *dsa_tx;
1776 } dmu_sync_arg_t;
1777
1778 /* ARGSUSED */
1779 static void
1780 dmu_sync_ready(zio_t *zio, arc_buf_t *buf, void *varg)
1781 {
1782 dmu_sync_arg_t *dsa = varg;
1783 dmu_buf_t *db = dsa->dsa_zgd->zgd_db;
1784 blkptr_t *bp = zio->io_bp;
1785
1786 if (zio->io_error == 0) {
1787 if (BP_IS_HOLE(bp)) {
1788 /*
1789 * A block of zeros may compress to a hole, but the
1790 * block size still needs to be known for replay.
1791 */
1792 BP_SET_LSIZE(bp, db->db_size);
1793 } else if (!BP_IS_EMBEDDED(bp)) {
1794 ASSERT(BP_GET_LEVEL(bp) == 0);
1795 BP_SET_FILL(bp, 1);
1796 }
1797 }
1798 }
1799
1800 static void
1801 dmu_sync_late_arrival_ready(zio_t *zio)
1802 {
1803 dmu_sync_ready(zio, NULL, zio->io_private);
1804 }
1805
1806 /* ARGSUSED */
1807 static void
1808 dmu_sync_done(zio_t *zio, arc_buf_t *buf, void *varg)
1809 {
1810 dmu_sync_arg_t *dsa = varg;
1811 dbuf_dirty_record_t *dr = dsa->dsa_dr;
1812 dmu_buf_impl_t *db = dr->dr_dbuf;
1813 zgd_t *zgd = dsa->dsa_zgd;
1814
1815 /*
1816 * Record the vdev(s) backing this blkptr so they can be flushed after
1817 * the writes for the lwb have completed.
1818 */
1819 if (zio->io_error == 0) {
1820 zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp);
1821 }
1822
1823 mutex_enter(&db->db_mtx);
1824 ASSERT(dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC);
1825 if (zio->io_error == 0) {
1826 dr->dt.dl.dr_nopwrite = !!(zio->io_flags & ZIO_FLAG_NOPWRITE);
1827 if (dr->dt.dl.dr_nopwrite) {
1828 blkptr_t *bp = zio->io_bp;
1829 blkptr_t *bp_orig = &zio->io_bp_orig;
1830 uint8_t chksum = BP_GET_CHECKSUM(bp_orig);
1831
1832 ASSERT(BP_EQUAL(bp, bp_orig));
1833 VERIFY(BP_EQUAL(bp, db->db_blkptr));
1834 ASSERT(zio->io_prop.zp_compress != ZIO_COMPRESS_OFF);
1835 VERIFY(zio_checksum_table[chksum].ci_flags &
1836 ZCHECKSUM_FLAG_NOPWRITE);
1837 }
1838 dr->dt.dl.dr_overridden_by = *zio->io_bp;
1839 dr->dt.dl.dr_override_state = DR_OVERRIDDEN;
1840 dr->dt.dl.dr_copies = zio->io_prop.zp_copies;
1841
1842 /*
1843 * Old style holes are filled with all zeros, whereas
1844 * new-style holes maintain their lsize, type, level,
1845 * and birth time (see zio_write_compress). While we
1846 * need to reset the BP_SET_LSIZE() call that happened
1847 * in dmu_sync_ready for old style holes, we do *not*
1848 * want to wipe out the information contained in new
1849 * style holes. Thus, only zero out the block pointer if
1850 * it's an old style hole.
1851 */
1852 if (BP_IS_HOLE(&dr->dt.dl.dr_overridden_by) &&
1853 dr->dt.dl.dr_overridden_by.blk_birth == 0)
1854 BP_ZERO(&dr->dt.dl.dr_overridden_by);
1855 } else {
1856 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN;
1857 }
1858 cv_broadcast(&db->db_changed);
1859 mutex_exit(&db->db_mtx);
1860
1861 dsa->dsa_done(dsa->dsa_zgd, zio->io_error);
1862
1863 kmem_free(dsa, sizeof (*dsa));
1864 }
1865
1866 static void
1867 dmu_sync_late_arrival_done(zio_t *zio)
1868 {
1869 blkptr_t *bp = zio->io_bp;
1870 dmu_sync_arg_t *dsa = zio->io_private;
1871 zgd_t *zgd = dsa->dsa_zgd;
1872
1873 if (zio->io_error == 0) {
1874 /*
1875 * Record the vdev(s) backing this blkptr so they can be
1876 * flushed after the writes for the lwb have completed.
1877 */
1878 zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp);
1879
1880 if (!BP_IS_HOLE(bp)) {
1881 ASSERTV(blkptr_t *bp_orig = &zio->io_bp_orig);
1882 ASSERT(!(zio->io_flags & ZIO_FLAG_NOPWRITE));
1883 ASSERT(BP_IS_HOLE(bp_orig) || !BP_EQUAL(bp, bp_orig));
1884 ASSERT(zio->io_bp->blk_birth == zio->io_txg);
1885 ASSERT(zio->io_txg > spa_syncing_txg(zio->io_spa));
1886 zio_free(zio->io_spa, zio->io_txg, zio->io_bp);
1887 }
1888 }
1889
1890 dmu_tx_commit(dsa->dsa_tx);
1891
1892 dsa->dsa_done(dsa->dsa_zgd, zio->io_error);
1893
1894 abd_put(zio->io_abd);
1895 kmem_free(dsa, sizeof (*dsa));
1896 }
1897
1898 static int
1899 dmu_sync_late_arrival(zio_t *pio, objset_t *os, dmu_sync_cb_t *done, zgd_t *zgd,
1900 zio_prop_t *zp, zbookmark_phys_t *zb)
1901 {
1902 dmu_sync_arg_t *dsa;
1903 dmu_tx_t *tx;
1904
1905 tx = dmu_tx_create(os);
1906 dmu_tx_hold_space(tx, zgd->zgd_db->db_size);
1907 if (dmu_tx_assign(tx, TXG_WAIT) != 0) {
1908 dmu_tx_abort(tx);
1909 /* Make zl_get_data do txg_waited_synced() */
1910 return (SET_ERROR(EIO));
1911 }
1912
1913 /*
1914 * In order to prevent the zgd's lwb from being free'd prior to
1915 * dmu_sync_late_arrival_done() being called, we have to ensure
1916 * the lwb's "max txg" takes this tx's txg into account.
1917 */
1918 zil_lwb_add_txg(zgd->zgd_lwb, dmu_tx_get_txg(tx));
1919
1920 dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
1921 dsa->dsa_dr = NULL;
1922 dsa->dsa_done = done;
1923 dsa->dsa_zgd = zgd;
1924 dsa->dsa_tx = tx;
1925
1926 /*
1927 * Since we are currently syncing this txg, it's nontrivial to
1928 * determine what BP to nopwrite against, so we disable nopwrite.
1929 *
1930 * When syncing, the db_blkptr is initially the BP of the previous
1931 * txg. We can not nopwrite against it because it will be changed
1932 * (this is similar to the non-late-arrival case where the dbuf is
1933 * dirty in a future txg).
1934 *
1935 * Then dbuf_write_ready() sets bp_blkptr to the location we will write.
1936 * We can not nopwrite against it because although the BP will not
1937 * (typically) be changed, the data has not yet been persisted to this
1938 * location.
1939 *
1940 * Finally, when dbuf_write_done() is called, it is theoretically
1941 * possible to always nopwrite, because the data that was written in
1942 * this txg is the same data that we are trying to write. However we
1943 * would need to check that this dbuf is not dirty in any future
1944 * txg's (as we do in the normal dmu_sync() path). For simplicity, we
1945 * don't nopwrite in this case.
1946 */
1947 zp->zp_nopwrite = B_FALSE;
1948
1949 zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), zgd->zgd_bp,
1950 abd_get_from_buf(zgd->zgd_db->db_data, zgd->zgd_db->db_size),
1951 zgd->zgd_db->db_size, zgd->zgd_db->db_size, zp,
1952 dmu_sync_late_arrival_ready, NULL, NULL, dmu_sync_late_arrival_done,
1953 dsa, ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, zb));
1954
1955 return (0);
1956 }
1957
1958 /*
1959 * Intent log support: sync the block associated with db to disk.
1960 * N.B. and XXX: the caller is responsible for making sure that the
1961 * data isn't changing while dmu_sync() is writing it.
1962 *
1963 * Return values:
1964 *
1965 * EEXIST: this txg has already been synced, so there's nothing to do.
1966 * The caller should not log the write.
1967 *
1968 * ENOENT: the block was dbuf_free_range()'d, so there's nothing to do.
1969 * The caller should not log the write.
1970 *
1971 * EALREADY: this block is already in the process of being synced.
1972 * The caller should track its progress (somehow).
1973 *
1974 * EIO: could not do the I/O.
1975 * The caller should do a txg_wait_synced().
1976 *
1977 * 0: the I/O has been initiated.
1978 * The caller should log this blkptr in the done callback.
1979 * It is possible that the I/O will fail, in which case
1980 * the error will be reported to the done callback and
1981 * propagated to pio from zio_done().
1982 */
1983 int
1984 dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd)
1985 {
1986 dmu_buf_impl_t *db = (dmu_buf_impl_t *)zgd->zgd_db;
1987 objset_t *os = db->db_objset;
1988 dsl_dataset_t *ds = os->os_dsl_dataset;
1989 dbuf_dirty_record_t *dr;
1990 dmu_sync_arg_t *dsa;
1991 zbookmark_phys_t zb;
1992 zio_prop_t zp;
1993 dnode_t *dn;
1994
1995 ASSERT(pio != NULL);
1996 ASSERT(txg != 0);
1997
1998 SET_BOOKMARK(&zb, ds->ds_object,
1999 db->db.db_object, db->db_level, db->db_blkid);
2000
2001 DB_DNODE_ENTER(db);
2002 dn = DB_DNODE(db);
2003 dmu_write_policy(os, dn, db->db_level, WP_DMU_SYNC, &zp);
2004 DB_DNODE_EXIT(db);
2005
2006 /*
2007 * If we're frozen (running ziltest), we always need to generate a bp.
2008 */
2009 if (txg > spa_freeze_txg(os->os_spa))
2010 return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb));
2011
2012 /*
2013 * Grabbing db_mtx now provides a barrier between dbuf_sync_leaf()
2014 * and us. If we determine that this txg is not yet syncing,
2015 * but it begins to sync a moment later, that's OK because the
2016 * sync thread will block in dbuf_sync_leaf() until we drop db_mtx.
2017 */
2018 mutex_enter(&db->db_mtx);
2019
2020 if (txg <= spa_last_synced_txg(os->os_spa)) {
2021 /*
2022 * This txg has already synced. There's nothing to do.
2023 */
2024 mutex_exit(&db->db_mtx);
2025 return (SET_ERROR(EEXIST));
2026 }
2027
2028 if (txg <= spa_syncing_txg(os->os_spa)) {
2029 /*
2030 * This txg is currently syncing, so we can't mess with
2031 * the dirty record anymore; just write a new log block.
2032 */
2033 mutex_exit(&db->db_mtx);
2034 return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb));
2035 }
2036
2037 dr = db->db_last_dirty;
2038 while (dr && dr->dr_txg != txg)
2039 dr = dr->dr_next;
2040
2041 if (dr == NULL) {
2042 /*
2043 * There's no dr for this dbuf, so it must have been freed.
2044 * There's no need to log writes to freed blocks, so we're done.
2045 */
2046 mutex_exit(&db->db_mtx);
2047 return (SET_ERROR(ENOENT));
2048 }
2049
2050 ASSERT(dr->dr_next == NULL || dr->dr_next->dr_txg < txg);
2051
2052 if (db->db_blkptr != NULL) {
2053 /*
2054 * We need to fill in zgd_bp with the current blkptr so that
2055 * the nopwrite code can check if we're writing the same
2056 * data that's already on disk. We can only nopwrite if we
2057 * are sure that after making the copy, db_blkptr will not
2058 * change until our i/o completes. We ensure this by
2059 * holding the db_mtx, and only allowing nopwrite if the
2060 * block is not already dirty (see below). This is verified
2061 * by dmu_sync_done(), which VERIFYs that the db_blkptr has
2062 * not changed.
2063 */
2064 *zgd->zgd_bp = *db->db_blkptr;
2065 }
2066
2067 /*
2068 * Assume the on-disk data is X, the current syncing data (in
2069 * txg - 1) is Y, and the current in-memory data is Z (currently
2070 * in dmu_sync).
2071 *
2072 * We usually want to perform a nopwrite if X and Z are the
2073 * same. However, if Y is different (i.e. the BP is going to
2074 * change before this write takes effect), then a nopwrite will
2075 * be incorrect - we would override with X, which could have
2076 * been freed when Y was written.
2077 *
2078 * (Note that this is not a concern when we are nop-writing from
2079 * syncing context, because X and Y must be identical, because
2080 * all previous txgs have been synced.)
2081 *
2082 * Therefore, we disable nopwrite if the current BP could change
2083 * before this TXG. There are two ways it could change: by
2084 * being dirty (dr_next is non-NULL), or by being freed
2085 * (dnode_block_freed()). This behavior is verified by
2086 * zio_done(), which VERIFYs that the override BP is identical
2087 * to the on-disk BP.
2088 */
2089 DB_DNODE_ENTER(db);
2090 dn = DB_DNODE(db);
2091 if (dr->dr_next != NULL || dnode_block_freed(dn, db->db_blkid))
2092 zp.zp_nopwrite = B_FALSE;
2093 DB_DNODE_EXIT(db);
2094
2095 ASSERT(dr->dr_txg == txg);
2096 if (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC ||
2097 dr->dt.dl.dr_override_state == DR_OVERRIDDEN) {
2098 /*
2099 * We have already issued a sync write for this buffer,
2100 * or this buffer has already been synced. It could not
2101 * have been dirtied since, or we would have cleared the state.
2102 */
2103 mutex_exit(&db->db_mtx);
2104 return (SET_ERROR(EALREADY));
2105 }
2106
2107 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN);
2108 dr->dt.dl.dr_override_state = DR_IN_DMU_SYNC;
2109 mutex_exit(&db->db_mtx);
2110
2111 dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP);
2112 dsa->dsa_dr = dr;
2113 dsa->dsa_done = done;
2114 dsa->dsa_zgd = zgd;
2115 dsa->dsa_tx = NULL;
2116
2117 zio_nowait(arc_write(pio, os->os_spa, txg,
2118 zgd->zgd_bp, dr->dt.dl.dr_data, DBUF_IS_L2CACHEABLE(db),
2119 &zp, dmu_sync_ready, NULL, NULL, dmu_sync_done, dsa,
2120 ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, &zb));
2121
2122 return (0);
2123 }
2124
2125 int
2126 dmu_object_set_nlevels(objset_t *os, uint64_t object, int nlevels, dmu_tx_t *tx)
2127 {
2128 dnode_t *dn;
2129 int err;
2130
2131 err = dnode_hold(os, object, FTAG, &dn);
2132 if (err)
2133 return (err);
2134 err = dnode_set_nlevels(dn, nlevels, tx);
2135 dnode_rele(dn, FTAG);
2136 return (err);
2137 }
2138
2139 int
2140 dmu_object_set_blocksize(objset_t *os, uint64_t object, uint64_t size, int ibs,
2141 dmu_tx_t *tx)
2142 {
2143 dnode_t *dn;
2144 int err;
2145
2146 err = dnode_hold(os, object, FTAG, &dn);
2147 if (err)
2148 return (err);
2149 err = dnode_set_blksz(dn, size, ibs, tx);
2150 dnode_rele(dn, FTAG);
2151 return (err);
2152 }
2153
2154 int
2155 dmu_object_set_maxblkid(objset_t *os, uint64_t object, uint64_t maxblkid,
2156 dmu_tx_t *tx)
2157 {
2158 dnode_t *dn;
2159 int err;
2160
2161 err = dnode_hold(os, object, FTAG, &dn);
2162 if (err)
2163 return (err);
2164 rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2165 dnode_new_blkid(dn, maxblkid, tx, B_FALSE, B_TRUE);
2166 rw_exit(&dn->dn_struct_rwlock);
2167 dnode_rele(dn, FTAG);
2168 return (0);
2169 }
2170
2171 void
2172 dmu_object_set_checksum(objset_t *os, uint64_t object, uint8_t checksum,
2173 dmu_tx_t *tx)
2174 {
2175 dnode_t *dn;
2176
2177 /*
2178 * Send streams include each object's checksum function. This
2179 * check ensures that the receiving system can understand the
2180 * checksum function transmitted.
2181 */
2182 ASSERT3U(checksum, <, ZIO_CHECKSUM_LEGACY_FUNCTIONS);
2183
2184 VERIFY0(dnode_hold(os, object, FTAG, &dn));
2185 ASSERT3U(checksum, <, ZIO_CHECKSUM_FUNCTIONS);
2186 dn->dn_checksum = checksum;
2187 dnode_setdirty(dn, tx);
2188 dnode_rele(dn, FTAG);
2189 }
2190
2191 void
2192 dmu_object_set_compress(objset_t *os, uint64_t object, uint8_t compress,
2193 dmu_tx_t *tx)
2194 {
2195 dnode_t *dn;
2196
2197 /*
2198 * Send streams include each object's compression function. This
2199 * check ensures that the receiving system can understand the
2200 * compression function transmitted.
2201 */
2202 ASSERT3U(compress, <, ZIO_COMPRESS_LEGACY_FUNCTIONS);
2203
2204 VERIFY0(dnode_hold(os, object, FTAG, &dn));
2205 dn->dn_compress = compress;
2206 dnode_setdirty(dn, tx);
2207 dnode_rele(dn, FTAG);
2208 }
2209
2210 /*
2211 * When the "redundant_metadata" property is set to "most", only indirect
2212 * blocks of this level and higher will have an additional ditto block.
2213 */
2214 int zfs_redundant_metadata_most_ditto_level = 2;
2215
2216 void
2217 dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp)
2218 {
2219 dmu_object_type_t type = dn ? dn->dn_type : DMU_OT_OBJSET;
2220 boolean_t ismd = (level > 0 || DMU_OT_IS_METADATA(type) ||
2221 (wp & WP_SPILL));
2222 enum zio_checksum checksum = os->os_checksum;
2223 enum zio_compress compress = os->os_compress;
2224 enum zio_checksum dedup_checksum = os->os_dedup_checksum;
2225 boolean_t dedup = B_FALSE;
2226 boolean_t nopwrite = B_FALSE;
2227 boolean_t dedup_verify = os->os_dedup_verify;
2228 boolean_t encrypt = B_FALSE;
2229 int copies = os->os_copies;
2230
2231 /*
2232 * We maintain different write policies for each of the following
2233 * types of data:
2234 * 1. metadata
2235 * 2. preallocated blocks (i.e. level-0 blocks of a dump device)
2236 * 3. all other level 0 blocks
2237 */
2238 if (ismd) {
2239 /*
2240 * XXX -- we should design a compression algorithm
2241 * that specializes in arrays of bps.
2242 */
2243 compress = zio_compress_select(os->os_spa,
2244 ZIO_COMPRESS_ON, ZIO_COMPRESS_ON);
2245
2246 /*
2247 * Metadata always gets checksummed. If the data
2248 * checksum is multi-bit correctable, and it's not a
2249 * ZBT-style checksum, then it's suitable for metadata
2250 * as well. Otherwise, the metadata checksum defaults
2251 * to fletcher4.
2252 */
2253 if (!(zio_checksum_table[checksum].ci_flags &
2254 ZCHECKSUM_FLAG_METADATA) ||
2255 (zio_checksum_table[checksum].ci_flags &
2256 ZCHECKSUM_FLAG_EMBEDDED))
2257 checksum = ZIO_CHECKSUM_FLETCHER_4;
2258
2259 if (os->os_redundant_metadata == ZFS_REDUNDANT_METADATA_ALL ||
2260 (os->os_redundant_metadata ==
2261 ZFS_REDUNDANT_METADATA_MOST &&
2262 (level >= zfs_redundant_metadata_most_ditto_level ||
2263 DMU_OT_IS_METADATA(type) || (wp & WP_SPILL))))
2264 copies++;
2265 } else if (wp & WP_NOFILL) {
2266 ASSERT(level == 0);
2267
2268 /*
2269 * If we're writing preallocated blocks, we aren't actually
2270 * writing them so don't set any policy properties. These
2271 * blocks are currently only used by an external subsystem
2272 * outside of zfs (i.e. dump) and not written by the zio
2273 * pipeline.
2274 */
2275 compress = ZIO_COMPRESS_OFF;
2276 checksum = ZIO_CHECKSUM_OFF;
2277 } else {
2278 compress = zio_compress_select(os->os_spa, dn->dn_compress,
2279 compress);
2280
2281 checksum = (dedup_checksum == ZIO_CHECKSUM_OFF) ?
2282 zio_checksum_select(dn->dn_checksum, checksum) :
2283 dedup_checksum;
2284
2285 /*
2286 * Determine dedup setting. If we are in dmu_sync(),
2287 * we won't actually dedup now because that's all
2288 * done in syncing context; but we do want to use the
2289 * dedup checkum. If the checksum is not strong
2290 * enough to ensure unique signatures, force
2291 * dedup_verify.
2292 */
2293 if (dedup_checksum != ZIO_CHECKSUM_OFF) {
2294 dedup = (wp & WP_DMU_SYNC) ? B_FALSE : B_TRUE;
2295 if (!(zio_checksum_table[checksum].ci_flags &
2296 ZCHECKSUM_FLAG_DEDUP))
2297 dedup_verify = B_TRUE;
2298 }
2299
2300 /*
2301 * Enable nopwrite if we have secure enough checksum
2302 * algorithm (see comment in zio_nop_write) and
2303 * compression is enabled. We don't enable nopwrite if
2304 * dedup is enabled as the two features are mutually
2305 * exclusive.
2306 */
2307 nopwrite = (!dedup && (zio_checksum_table[checksum].ci_flags &
2308 ZCHECKSUM_FLAG_NOPWRITE) &&
2309 compress != ZIO_COMPRESS_OFF && zfs_nopwrite_enabled);
2310 }
2311
2312 /*
2313 * All objects in an encrypted objset are protected from modification
2314 * via a MAC. Encrypted objects store their IV and salt in the last DVA
2315 * in the bp, so we cannot use all copies. Encrypted objects are also
2316 * not subject to nopwrite since writing the same data will still
2317 * result in a new ciphertext. Only encrypted blocks can be dedup'd
2318 * to avoid ambiguity in the dedup code since the DDT does not store
2319 * object types.
2320 */
2321 if (os->os_encrypted && (wp & WP_NOFILL) == 0) {
2322 encrypt = B_TRUE;
2323
2324 if (DMU_OT_IS_ENCRYPTED(type)) {
2325 copies = MIN(copies, SPA_DVAS_PER_BP - 1);
2326 nopwrite = B_FALSE;
2327 } else {
2328 dedup = B_FALSE;
2329 }
2330
2331 if (level <= 0 &&
2332 (type == DMU_OT_DNODE || type == DMU_OT_OBJSET)) {
2333 compress = ZIO_COMPRESS_EMPTY;
2334 }
2335 }
2336
2337 zp->zp_compress = compress;
2338 zp->zp_checksum = checksum;
2339 zp->zp_type = (wp & WP_SPILL) ? dn->dn_bonustype : type;
2340 zp->zp_level = level;
2341 zp->zp_copies = MIN(copies, spa_max_replication(os->os_spa));
2342 zp->zp_dedup = dedup;
2343 zp->zp_dedup_verify = dedup && dedup_verify;
2344 zp->zp_nopwrite = nopwrite;
2345 zp->zp_encrypt = encrypt;
2346 zp->zp_byteorder = ZFS_HOST_BYTEORDER;
2347 bzero(zp->zp_salt, ZIO_DATA_SALT_LEN);
2348 bzero(zp->zp_iv, ZIO_DATA_IV_LEN);
2349 bzero(zp->zp_mac, ZIO_DATA_MAC_LEN);
2350 zp->zp_zpl_smallblk = DMU_OT_IS_FILE(zp->zp_type) ?
2351 os->os_zpl_special_smallblock : 0;
2352
2353 ASSERT3U(zp->zp_compress, !=, ZIO_COMPRESS_INHERIT);
2354 }
2355
2356 /*
2357 * This function is only called from zfs_holey_common() for zpl_llseek()
2358 * in order to determine the location of holes. In order to accurately
2359 * report holes all dirty data must be synced to disk. This causes extremely
2360 * poor performance when seeking for holes in a dirty file. As a compromise,
2361 * only provide hole data when the dnode is clean. When a dnode is dirty
2362 * report the dnode as having no holes which is always a safe thing to do.
2363 */
2364 int
2365 dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off)
2366 {
2367 dnode_t *dn;
2368 int i, err;
2369 boolean_t clean = B_TRUE;
2370
2371 err = dnode_hold(os, object, FTAG, &dn);
2372 if (err)
2373 return (err);
2374
2375 /*
2376 * Check if there are dirty data blocks or frees which have not been
2377 * synced. Dirty spill and bonus blocks which are external to the
2378 * object can ignored when reporting holes.
2379 */
2380 mutex_enter(&dn->dn_mtx);
2381 for (i = 0; i < TXG_SIZE; i++) {
2382 if (multilist_link_active(&dn->dn_dirty_link[i])) {
2383
2384 if (dn->dn_free_ranges[i] != NULL) {
2385 clean = B_FALSE;
2386 break;
2387 }
2388
2389 list_t *list = &dn->dn_dirty_records[i];
2390 dbuf_dirty_record_t *dr;
2391
2392 for (dr = list_head(list); dr != NULL;
2393 dr = list_next(list, dr)) {
2394 dmu_buf_impl_t *db = dr->dr_dbuf;
2395
2396 if (db->db_blkid == DMU_SPILL_BLKID ||
2397 db->db_blkid == DMU_BONUS_BLKID)
2398 continue;
2399
2400 clean = B_FALSE;
2401 break;
2402 }
2403 }
2404
2405 if (clean == B_FALSE)
2406 break;
2407 }
2408 mutex_exit(&dn->dn_mtx);
2409
2410 /*
2411 * If compatibility option is on, sync any current changes before
2412 * we go trundling through the block pointers.
2413 */
2414 if (!clean && zfs_dmu_offset_next_sync) {
2415 clean = B_TRUE;
2416 dnode_rele(dn, FTAG);
2417 txg_wait_synced(dmu_objset_pool(os), 0);
2418 err = dnode_hold(os, object, FTAG, &dn);
2419 if (err)
2420 return (err);
2421 }
2422
2423 if (clean)
2424 err = dnode_next_offset(dn,
2425 (hole ? DNODE_FIND_HOLE : 0), off, 1, 1, 0);
2426 else
2427 err = SET_ERROR(EBUSY);
2428
2429 dnode_rele(dn, FTAG);
2430
2431 return (err);
2432 }
2433
2434 void
2435 __dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi)
2436 {
2437 dnode_phys_t *dnp = dn->dn_phys;
2438
2439 doi->doi_data_block_size = dn->dn_datablksz;
2440 doi->doi_metadata_block_size = dn->dn_indblkshift ?
2441 1ULL << dn->dn_indblkshift : 0;
2442 doi->doi_type = dn->dn_type;
2443 doi->doi_bonus_type = dn->dn_bonustype;
2444 doi->doi_bonus_size = dn->dn_bonuslen;
2445 doi->doi_dnodesize = dn->dn_num_slots << DNODE_SHIFT;
2446 doi->doi_indirection = dn->dn_nlevels;
2447 doi->doi_checksum = dn->dn_checksum;
2448 doi->doi_compress = dn->dn_compress;
2449 doi->doi_nblkptr = dn->dn_nblkptr;
2450 doi->doi_physical_blocks_512 = (DN_USED_BYTES(dnp) + 256) >> 9;
2451 doi->doi_max_offset = (dn->dn_maxblkid + 1) * dn->dn_datablksz;
2452 doi->doi_fill_count = 0;
2453 for (int i = 0; i < dnp->dn_nblkptr; i++)
2454 doi->doi_fill_count += BP_GET_FILL(&dnp->dn_blkptr[i]);
2455 }
2456
2457 void
2458 dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi)
2459 {
2460 rw_enter(&dn->dn_struct_rwlock, RW_READER);
2461 mutex_enter(&dn->dn_mtx);
2462
2463 __dmu_object_info_from_dnode(dn, doi);
2464
2465 mutex_exit(&dn->dn_mtx);
2466 rw_exit(&dn->dn_struct_rwlock);
2467 }
2468
2469 /*
2470 * Get information on a DMU object.
2471 * If doi is NULL, just indicates whether the object exists.
2472 */
2473 int
2474 dmu_object_info(objset_t *os, uint64_t object, dmu_object_info_t *doi)
2475 {
2476 dnode_t *dn;
2477 int err = dnode_hold(os, object, FTAG, &dn);
2478
2479 if (err)
2480 return (err);
2481
2482 if (doi != NULL)
2483 dmu_object_info_from_dnode(dn, doi);
2484
2485 dnode_rele(dn, FTAG);
2486 return (0);
2487 }
2488
2489 /*
2490 * As above, but faster; can be used when you have a held dbuf in hand.
2491 */
2492 void
2493 dmu_object_info_from_db(dmu_buf_t *db_fake, dmu_object_info_t *doi)
2494 {
2495 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2496
2497 DB_DNODE_ENTER(db);
2498 dmu_object_info_from_dnode(DB_DNODE(db), doi);
2499 DB_DNODE_EXIT(db);
2500 }
2501
2502 /*
2503 * Faster still when you only care about the size.
2504 * This is specifically optimized for zfs_getattr().
2505 */
2506 void
2507 dmu_object_size_from_db(dmu_buf_t *db_fake, uint32_t *blksize,
2508 u_longlong_t *nblk512)
2509 {
2510 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2511 dnode_t *dn;
2512
2513 DB_DNODE_ENTER(db);
2514 dn = DB_DNODE(db);
2515
2516 *blksize = dn->dn_datablksz;
2517 /* add in number of slots used for the dnode itself */
2518 *nblk512 = ((DN_USED_BYTES(dn->dn_phys) + SPA_MINBLOCKSIZE/2) >>
2519 SPA_MINBLOCKSHIFT) + dn->dn_num_slots;
2520 DB_DNODE_EXIT(db);
2521 }
2522
2523 void
2524 dmu_object_dnsize_from_db(dmu_buf_t *db_fake, int *dnsize)
2525 {
2526 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2527 dnode_t *dn;
2528
2529 DB_DNODE_ENTER(db);
2530 dn = DB_DNODE(db);
2531 *dnsize = dn->dn_num_slots << DNODE_SHIFT;
2532 DB_DNODE_EXIT(db);
2533 }
2534
2535 void
2536 byteswap_uint64_array(void *vbuf, size_t size)
2537 {
2538 uint64_t *buf = vbuf;
2539 size_t count = size >> 3;
2540 int i;
2541
2542 ASSERT((size & 7) == 0);
2543
2544 for (i = 0; i < count; i++)
2545 buf[i] = BSWAP_64(buf[i]);
2546 }
2547
2548 void
2549 byteswap_uint32_array(void *vbuf, size_t size)
2550 {
2551 uint32_t *buf = vbuf;
2552 size_t count = size >> 2;
2553 int i;
2554
2555 ASSERT((size & 3) == 0);
2556
2557 for (i = 0; i < count; i++)
2558 buf[i] = BSWAP_32(buf[i]);
2559 }
2560
2561 void
2562 byteswap_uint16_array(void *vbuf, size_t size)
2563 {
2564 uint16_t *buf = vbuf;
2565 size_t count = size >> 1;
2566 int i;
2567
2568 ASSERT((size & 1) == 0);
2569
2570 for (i = 0; i < count; i++)
2571 buf[i] = BSWAP_16(buf[i]);
2572 }
2573
2574 /* ARGSUSED */
2575 void
2576 byteswap_uint8_array(void *vbuf, size_t size)
2577 {
2578 }
2579
2580 void
2581 dmu_init(void)
2582 {
2583 abd_init();
2584 zfs_dbgmsg_init();
2585 sa_cache_init();
2586 xuio_stat_init();
2587 dmu_objset_init();
2588 dnode_init();
2589 zfetch_init();
2590 dmu_tx_init();
2591 l2arc_init();
2592 arc_init();
2593 dbuf_init();
2594 }
2595
2596 void
2597 dmu_fini(void)
2598 {
2599 arc_fini(); /* arc depends on l2arc, so arc must go first */
2600 l2arc_fini();
2601 dmu_tx_fini();
2602 zfetch_fini();
2603 dbuf_fini();
2604 dnode_fini();
2605 dmu_objset_fini();
2606 xuio_stat_fini();
2607 sa_cache_fini();
2608 zfs_dbgmsg_fini();
2609 abd_fini();
2610 }
2611
2612 #if defined(_KERNEL)
2613 EXPORT_SYMBOL(dmu_bonus_hold);
2614 EXPORT_SYMBOL(dmu_bonus_hold_by_dnode);
2615 EXPORT_SYMBOL(dmu_buf_hold_array_by_bonus);
2616 EXPORT_SYMBOL(dmu_buf_rele_array);
2617 EXPORT_SYMBOL(dmu_prefetch);
2618 EXPORT_SYMBOL(dmu_free_range);
2619 EXPORT_SYMBOL(dmu_free_long_range);
2620 EXPORT_SYMBOL(dmu_free_long_object);
2621 EXPORT_SYMBOL(dmu_read);
2622 EXPORT_SYMBOL(dmu_read_by_dnode);
2623 EXPORT_SYMBOL(dmu_write);
2624 EXPORT_SYMBOL(dmu_write_by_dnode);
2625 EXPORT_SYMBOL(dmu_prealloc);
2626 EXPORT_SYMBOL(dmu_object_info);
2627 EXPORT_SYMBOL(dmu_object_info_from_dnode);
2628 EXPORT_SYMBOL(dmu_object_info_from_db);
2629 EXPORT_SYMBOL(dmu_object_size_from_db);
2630 EXPORT_SYMBOL(dmu_object_dnsize_from_db);
2631 EXPORT_SYMBOL(dmu_object_set_nlevels);
2632 EXPORT_SYMBOL(dmu_object_set_blocksize);
2633 EXPORT_SYMBOL(dmu_object_set_maxblkid);
2634 EXPORT_SYMBOL(dmu_object_set_checksum);
2635 EXPORT_SYMBOL(dmu_object_set_compress);
2636 EXPORT_SYMBOL(dmu_write_policy);
2637 EXPORT_SYMBOL(dmu_sync);
2638 EXPORT_SYMBOL(dmu_request_arcbuf);
2639 EXPORT_SYMBOL(dmu_return_arcbuf);
2640 EXPORT_SYMBOL(dmu_assign_arcbuf_by_dnode);
2641 EXPORT_SYMBOL(dmu_assign_arcbuf_by_dbuf);
2642 EXPORT_SYMBOL(dmu_buf_hold);
2643 EXPORT_SYMBOL(dmu_ot);
2644
2645 /* BEGIN CSTYLED */
2646 module_param(zfs_nopwrite_enabled, int, 0644);
2647 MODULE_PARM_DESC(zfs_nopwrite_enabled, "Enable NOP writes");
2648
2649 module_param(zfs_per_txg_dirty_frees_percent, ulong, 0644);
2650 MODULE_PARM_DESC(zfs_per_txg_dirty_frees_percent,
2651 "percentage of dirtied blocks from frees in one TXG");
2652
2653 module_param(zfs_dmu_offset_next_sync, int, 0644);
2654 MODULE_PARM_DESC(zfs_dmu_offset_next_sync,
2655 "Enable forcing txg sync to find holes");
2656
2657 /* END CSTYLED */
2658
2659 #endif
ZFS On Linux mirror for PVE