4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
27 #include <sys/dmu_impl.h>
28 #include <sys/dmu_tx.h>
30 #include <sys/dnode.h>
31 #include <sys/zfs_context.h>
32 #include <sys/dmu_objset.h>
33 #include <sys/dmu_traverse.h>
34 #include <sys/dsl_dataset.h>
35 #include <sys/dsl_dir.h>
36 #include <sys/dsl_pool.h>
37 #include <sys/dsl_synctask.h>
38 #include <sys/dsl_prop.h>
39 #include <sys/dmu_zfetch.h>
40 #include <sys/zfs_ioctl.h>
42 #include <sys/zio_checksum.h>
44 #include <sys/vmsystm.h>
45 #include <sys/zfs_znode.h>
48 const dmu_object_type_info_t dmu_ot
[DMU_OT_NUMTYPES
] = {
49 { byteswap_uint8_array
, TRUE
, "unallocated" },
50 { zap_byteswap
, TRUE
, "object directory" },
51 { byteswap_uint64_array
, TRUE
, "object array" },
52 { byteswap_uint8_array
, TRUE
, "packed nvlist" },
53 { byteswap_uint64_array
, TRUE
, "packed nvlist size" },
54 { byteswap_uint64_array
, TRUE
, "bplist" },
55 { byteswap_uint64_array
, TRUE
, "bplist header" },
56 { byteswap_uint64_array
, TRUE
, "SPA space map header" },
57 { byteswap_uint64_array
, TRUE
, "SPA space map" },
58 { byteswap_uint64_array
, TRUE
, "ZIL intent log" },
59 { dnode_buf_byteswap
, TRUE
, "DMU dnode" },
60 { dmu_objset_byteswap
, TRUE
, "DMU objset" },
61 { byteswap_uint64_array
, TRUE
, "DSL directory" },
62 { zap_byteswap
, TRUE
, "DSL directory child map"},
63 { zap_byteswap
, TRUE
, "DSL dataset snap map" },
64 { zap_byteswap
, TRUE
, "DSL props" },
65 { byteswap_uint64_array
, TRUE
, "DSL dataset" },
66 { zfs_znode_byteswap
, TRUE
, "ZFS znode" },
67 { zfs_oldacl_byteswap
, TRUE
, "ZFS V0 ACL" },
68 { byteswap_uint8_array
, FALSE
, "ZFS plain file" },
69 { zap_byteswap
, TRUE
, "ZFS directory" },
70 { zap_byteswap
, TRUE
, "ZFS master node" },
71 { zap_byteswap
, TRUE
, "ZFS delete queue" },
72 { byteswap_uint8_array
, FALSE
, "zvol object" },
73 { zap_byteswap
, TRUE
, "zvol prop" },
74 { byteswap_uint8_array
, FALSE
, "other uint8[]" },
75 { byteswap_uint64_array
, FALSE
, "other uint64[]" },
76 { zap_byteswap
, TRUE
, "other ZAP" },
77 { zap_byteswap
, TRUE
, "persistent error log" },
78 { byteswap_uint8_array
, TRUE
, "SPA history" },
79 { byteswap_uint64_array
, TRUE
, "SPA history offsets" },
80 { zap_byteswap
, TRUE
, "Pool properties" },
81 { zap_byteswap
, TRUE
, "DSL permissions" },
82 { zfs_acl_byteswap
, TRUE
, "ZFS ACL" },
83 { byteswap_uint8_array
, TRUE
, "ZFS SYSACL" },
84 { byteswap_uint8_array
, TRUE
, "FUID table" },
85 { byteswap_uint64_array
, TRUE
, "FUID table size" },
86 { zap_byteswap
, TRUE
, "DSL dataset next clones"},
87 { zap_byteswap
, TRUE
, "scrub work queue" },
88 { zap_byteswap
, TRUE
, "ZFS user/group used" },
89 { zap_byteswap
, TRUE
, "ZFS user/group quota" },
93 dmu_buf_hold(objset_t
*os
, uint64_t object
, uint64_t offset
,
94 void *tag
, dmu_buf_t
**dbp
)
101 err
= dnode_hold(os
->os
, object
, FTAG
, &dn
);
104 blkid
= dbuf_whichblock(dn
, offset
);
105 rw_enter(&dn
->dn_struct_rwlock
, RW_READER
);
106 db
= dbuf_hold(dn
, blkid
, tag
);
107 rw_exit(&dn
->dn_struct_rwlock
);
111 err
= dbuf_read(db
, NULL
, DB_RF_CANFAIL
);
118 dnode_rele(dn
, FTAG
);
126 return (DN_MAX_BONUSLEN
);
130 dmu_set_bonus(dmu_buf_t
*db
, int newsize
, dmu_tx_t
*tx
)
132 dnode_t
*dn
= ((dmu_buf_impl_t
*)db
)->db_dnode
;
134 if (dn
->dn_bonus
!= (dmu_buf_impl_t
*)db
)
136 if (newsize
< 0 || newsize
> db
->db_size
)
138 dnode_setbonuslen(dn
, newsize
, tx
);
143 * returns ENOENT, EIO, or 0.
146 dmu_bonus_hold(objset_t
*os
, uint64_t object
, void *tag
, dmu_buf_t
**dbp
)
152 error
= dnode_hold(os
->os
, object
, FTAG
, &dn
);
156 rw_enter(&dn
->dn_struct_rwlock
, RW_READER
);
157 if (dn
->dn_bonus
== NULL
) {
158 rw_exit(&dn
->dn_struct_rwlock
);
159 rw_enter(&dn
->dn_struct_rwlock
, RW_WRITER
);
160 if (dn
->dn_bonus
== NULL
)
161 dbuf_create_bonus(dn
);
164 rw_exit(&dn
->dn_struct_rwlock
);
166 /* as long as the bonus buf is held, the dnode will be held */
167 if (refcount_add(&db
->db_holds
, tag
) == 1)
168 VERIFY(dnode_add_ref(dn
, db
));
170 dnode_rele(dn
, FTAG
);
172 VERIFY(0 == dbuf_read(db
, NULL
, DB_RF_MUST_SUCCEED
));
179 * Note: longer-term, we should modify all of the dmu_buf_*() interfaces
180 * to take a held dnode rather than <os, object> -- the lookup is wasteful,
181 * and can induce severe lock contention when writing to several files
182 * whose dnodes are in the same block.
185 dmu_buf_hold_array_by_dnode(dnode_t
*dn
, uint64_t offset
, uint64_t length
,
186 int read
, void *tag
, int *numbufsp
, dmu_buf_t
***dbpp
, uint32_t flags
)
188 dsl_pool_t
*dp
= NULL
;
190 uint64_t blkid
, nblks
, i
;
196 ASSERT(length
<= DMU_MAX_ACCESS
);
198 dbuf_flags
= DB_RF_CANFAIL
| DB_RF_NEVERWAIT
;
199 if (flags
& DMU_READ_NO_PREFETCH
|| length
> zfetch_array_rd_sz
)
200 dbuf_flags
|= DB_RF_NOPREFETCH
;
202 rw_enter(&dn
->dn_struct_rwlock
, RW_READER
);
203 if (dn
->dn_datablkshift
) {
204 int blkshift
= dn
->dn_datablkshift
;
205 nblks
= (P2ROUNDUP(offset
+length
, 1ULL<<blkshift
) -
206 P2ALIGN(offset
, 1ULL<<blkshift
)) >> blkshift
;
208 if (offset
+ length
> dn
->dn_datablksz
) {
209 zfs_panic_recover("zfs: accessing past end of object "
210 "%llx/%llx (size=%u access=%llu+%llu)",
211 (longlong_t
)dn
->dn_objset
->
212 os_dsl_dataset
->ds_object
,
213 (longlong_t
)dn
->dn_object
, dn
->dn_datablksz
,
214 (longlong_t
)offset
, (longlong_t
)length
);
219 dbp
= kmem_zalloc(sizeof (dmu_buf_t
*) * nblks
, KM_SLEEP
);
221 if (dn
->dn_objset
->os_dsl_dataset
)
222 dp
= dn
->dn_objset
->os_dsl_dataset
->ds_dir
->dd_pool
;
223 if (dp
&& dsl_pool_sync_context(dp
))
225 zio
= zio_root(dn
->dn_objset
->os_spa
, NULL
, NULL
, ZIO_FLAG_CANFAIL
);
226 blkid
= dbuf_whichblock(dn
, offset
);
227 for (i
= 0; i
< nblks
; i
++) {
228 dmu_buf_impl_t
*db
= dbuf_hold(dn
, blkid
+i
, tag
);
230 rw_exit(&dn
->dn_struct_rwlock
);
231 dmu_buf_rele_array(dbp
, nblks
, tag
);
235 /* initiate async i/o */
237 rw_exit(&dn
->dn_struct_rwlock
);
238 (void) dbuf_read(db
, zio
, dbuf_flags
);
239 rw_enter(&dn
->dn_struct_rwlock
, RW_READER
);
243 rw_exit(&dn
->dn_struct_rwlock
);
245 /* wait for async i/o */
247 /* track read overhead when we are in sync context */
248 if (dp
&& dsl_pool_sync_context(dp
))
249 dp
->dp_read_overhead
+= gethrtime() - start
;
251 dmu_buf_rele_array(dbp
, nblks
, tag
);
255 /* wait for other io to complete */
257 for (i
= 0; i
< nblks
; i
++) {
258 dmu_buf_impl_t
*db
= (dmu_buf_impl_t
*)dbp
[i
];
259 mutex_enter(&db
->db_mtx
);
260 while (db
->db_state
== DB_READ
||
261 db
->db_state
== DB_FILL
)
262 cv_wait(&db
->db_changed
, &db
->db_mtx
);
263 if (db
->db_state
== DB_UNCACHED
)
265 mutex_exit(&db
->db_mtx
);
267 dmu_buf_rele_array(dbp
, nblks
, tag
);
279 dmu_buf_hold_array(objset_t
*os
, uint64_t object
, uint64_t offset
,
280 uint64_t length
, int read
, void *tag
, int *numbufsp
, dmu_buf_t
***dbpp
)
285 err
= dnode_hold(os
->os
, object
, FTAG
, &dn
);
289 err
= dmu_buf_hold_array_by_dnode(dn
, offset
, length
, read
, tag
,
290 numbufsp
, dbpp
, DMU_READ_PREFETCH
);
292 dnode_rele(dn
, FTAG
);
298 dmu_buf_hold_array_by_bonus(dmu_buf_t
*db
, uint64_t offset
,
299 uint64_t length
, int read
, void *tag
, int *numbufsp
, dmu_buf_t
***dbpp
)
301 dnode_t
*dn
= ((dmu_buf_impl_t
*)db
)->db_dnode
;
304 err
= dmu_buf_hold_array_by_dnode(dn
, offset
, length
, read
, tag
,
305 numbufsp
, dbpp
, DMU_READ_PREFETCH
);
311 dmu_buf_rele_array(dmu_buf_t
**dbp_fake
, int numbufs
, void *tag
)
314 dmu_buf_impl_t
**dbp
= (dmu_buf_impl_t
**)dbp_fake
;
319 for (i
= 0; i
< numbufs
; i
++) {
321 dbuf_rele(dbp
[i
], tag
);
324 kmem_free(dbp
, sizeof (dmu_buf_t
*) * numbufs
);
328 dmu_prefetch(objset_t
*os
, uint64_t object
, uint64_t offset
, uint64_t len
)
334 if (zfs_prefetch_disable
)
337 if (len
== 0) { /* they're interested in the bonus buffer */
338 dn
= os
->os
->os_meta_dnode
;
340 if (object
== 0 || object
>= DN_MAX_OBJECT
)
343 rw_enter(&dn
->dn_struct_rwlock
, RW_READER
);
344 blkid
= dbuf_whichblock(dn
, object
* sizeof (dnode_phys_t
));
345 dbuf_prefetch(dn
, blkid
);
346 rw_exit(&dn
->dn_struct_rwlock
);
351 * XXX - Note, if the dnode for the requested object is not
352 * already cached, we will do a *synchronous* read in the
353 * dnode_hold() call. The same is true for any indirects.
355 err
= dnode_hold(os
->os
, object
, FTAG
, &dn
);
359 rw_enter(&dn
->dn_struct_rwlock
, RW_READER
);
360 if (dn
->dn_datablkshift
) {
361 int blkshift
= dn
->dn_datablkshift
;
362 nblks
= (P2ROUNDUP(offset
+len
, 1<<blkshift
) -
363 P2ALIGN(offset
, 1<<blkshift
)) >> blkshift
;
365 nblks
= (offset
< dn
->dn_datablksz
);
369 blkid
= dbuf_whichblock(dn
, offset
);
370 for (i
= 0; i
< nblks
; i
++)
371 dbuf_prefetch(dn
, blkid
+i
);
374 rw_exit(&dn
->dn_struct_rwlock
);
376 dnode_rele(dn
, FTAG
);
380 get_next_chunk(dnode_t
*dn
, uint64_t *offset
, uint64_t limit
)
382 uint64_t len
= *offset
- limit
;
383 uint64_t chunk_len
= dn
->dn_datablksz
* DMU_MAX_DELETEBLKCNT
;
385 dn
->dn_datablksz
* EPB(dn
->dn_indblkshift
, SPA_BLKPTRSHIFT
);
387 ASSERT(limit
<= *offset
);
389 if (len
<= chunk_len
) {
394 ASSERT(ISP2(subchunk
));
396 while (*offset
> limit
) {
397 uint64_t initial_offset
= P2ROUNDUP(*offset
, subchunk
);
401 /* skip over allocated data */
402 err
= dnode_next_offset(dn
,
403 DNODE_FIND_HOLE
|DNODE_FIND_BACKWARDS
, offset
, 1, 1, 0);
409 ASSERT3U(*offset
, <=, initial_offset
);
410 *offset
= P2ALIGN(*offset
, subchunk
);
411 delta
= initial_offset
- *offset
;
412 if (delta
>= chunk_len
) {
413 *offset
+= delta
- chunk_len
;
418 /* skip over unallocated data */
419 err
= dnode_next_offset(dn
,
420 DNODE_FIND_BACKWARDS
, offset
, 1, 1, 0);
428 ASSERT3U(*offset
, <, initial_offset
);
434 dmu_free_long_range_impl(objset_t
*os
, dnode_t
*dn
, uint64_t offset
,
435 uint64_t length
, boolean_t free_dnode
)
438 uint64_t object_size
, start
, end
, len
;
439 boolean_t trunc
= (length
== DMU_OBJECT_END
);
442 align
= 1 << dn
->dn_datablkshift
;
444 object_size
= align
== 1 ? dn
->dn_datablksz
:
445 (dn
->dn_maxblkid
+ 1) << dn
->dn_datablkshift
;
447 end
= offset
+ length
;
448 if (trunc
|| end
> object_size
)
452 length
= end
- offset
;
456 /* assert(offset <= start) */
457 err
= get_next_chunk(dn
, &start
, offset
);
460 len
= trunc
? DMU_OBJECT_END
: end
- start
;
462 tx
= dmu_tx_create(os
);
463 dmu_tx_hold_free(tx
, dn
->dn_object
, start
, len
);
464 err
= dmu_tx_assign(tx
, TXG_WAIT
);
470 dnode_free_range(dn
, start
, trunc
? -1 : len
, tx
);
472 if (start
== 0 && free_dnode
) {
477 length
-= end
- start
;
486 dmu_free_long_range(objset_t
*os
, uint64_t object
,
487 uint64_t offset
, uint64_t length
)
492 err
= dnode_hold(os
->os
, object
, FTAG
, &dn
);
495 err
= dmu_free_long_range_impl(os
, dn
, offset
, length
, FALSE
);
496 dnode_rele(dn
, FTAG
);
501 dmu_free_object(objset_t
*os
, uint64_t object
)
507 err
= dnode_hold_impl(os
->os
, object
, DNODE_MUST_BE_ALLOCATED
,
511 if (dn
->dn_nlevels
== 1) {
512 tx
= dmu_tx_create(os
);
513 dmu_tx_hold_bonus(tx
, object
);
514 dmu_tx_hold_free(tx
, dn
->dn_object
, 0, DMU_OBJECT_END
);
515 err
= dmu_tx_assign(tx
, TXG_WAIT
);
517 dnode_free_range(dn
, 0, DMU_OBJECT_END
, tx
);
524 err
= dmu_free_long_range_impl(os
, dn
, 0, DMU_OBJECT_END
, TRUE
);
526 dnode_rele(dn
, FTAG
);
531 dmu_free_range(objset_t
*os
, uint64_t object
, uint64_t offset
,
532 uint64_t size
, dmu_tx_t
*tx
)
535 int err
= dnode_hold(os
->os
, object
, FTAG
, &dn
);
538 ASSERT(offset
< UINT64_MAX
);
539 ASSERT(size
== -1ULL || size
<= UINT64_MAX
- offset
);
540 dnode_free_range(dn
, offset
, size
, tx
);
541 dnode_rele(dn
, FTAG
);
546 dmu_read(objset_t
*os
, uint64_t object
, uint64_t offset
, uint64_t size
,
547 void *buf
, uint32_t flags
)
553 err
= dnode_hold(os
->os
, object
, FTAG
, &dn
);
558 * Deal with odd block sizes, where there can't be data past the first
559 * block. If we ever do the tail block optimization, we will need to
560 * handle that here as well.
562 if (dn
->dn_datablkshift
== 0) {
563 int newsz
= offset
> dn
->dn_datablksz
? 0 :
564 MIN(size
, dn
->dn_datablksz
- offset
);
565 bzero((char *)buf
+ newsz
, size
- newsz
);
570 uint64_t mylen
= MIN(size
, DMU_MAX_ACCESS
/ 2);
573 * NB: we could do this block-at-a-time, but it's nice
574 * to be reading in parallel.
576 err
= dmu_buf_hold_array_by_dnode(dn
, offset
, mylen
,
577 TRUE
, FTAG
, &numbufs
, &dbp
, flags
);
581 for (i
= 0; i
< numbufs
; i
++) {
584 dmu_buf_t
*db
= dbp
[i
];
588 bufoff
= offset
- db
->db_offset
;
589 tocpy
= (int)MIN(db
->db_size
- bufoff
, size
);
591 bcopy((char *)db
->db_data
+ bufoff
, buf
, tocpy
);
595 buf
= (char *)buf
+ tocpy
;
597 dmu_buf_rele_array(dbp
, numbufs
, FTAG
);
599 dnode_rele(dn
, FTAG
);
604 dmu_write(objset_t
*os
, uint64_t object
, uint64_t offset
, uint64_t size
,
605 const void *buf
, dmu_tx_t
*tx
)
613 VERIFY(0 == dmu_buf_hold_array(os
, object
, offset
, size
,
614 FALSE
, FTAG
, &numbufs
, &dbp
));
616 for (i
= 0; i
< numbufs
; i
++) {
619 dmu_buf_t
*db
= dbp
[i
];
623 bufoff
= offset
- db
->db_offset
;
624 tocpy
= (int)MIN(db
->db_size
- bufoff
, size
);
626 ASSERT(i
== 0 || i
== numbufs
-1 || tocpy
== db
->db_size
);
628 if (tocpy
== db
->db_size
)
629 dmu_buf_will_fill(db
, tx
);
631 dmu_buf_will_dirty(db
, tx
);
633 bcopy(buf
, (char *)db
->db_data
+ bufoff
, tocpy
);
635 if (tocpy
== db
->db_size
)
636 dmu_buf_fill_done(db
, tx
);
640 buf
= (char *)buf
+ tocpy
;
642 dmu_buf_rele_array(dbp
, numbufs
, FTAG
);
646 dmu_prealloc(objset_t
*os
, uint64_t object
, uint64_t offset
, uint64_t size
,
655 VERIFY(0 == dmu_buf_hold_array(os
, object
, offset
, size
,
656 FALSE
, FTAG
, &numbufs
, &dbp
));
658 for (i
= 0; i
< numbufs
; i
++) {
659 dmu_buf_t
*db
= dbp
[i
];
661 dmu_buf_will_not_fill(db
, tx
);
663 dmu_buf_rele_array(dbp
, numbufs
, FTAG
);
668 dmu_read_uio(objset_t
*os
, uint64_t object
, uio_t
*uio
, uint64_t size
)
674 * NB: we could do this block-at-a-time, but it's nice
675 * to be reading in parallel.
677 err
= dmu_buf_hold_array(os
, object
, uio
->uio_loffset
, size
, TRUE
, FTAG
,
682 for (i
= 0; i
< numbufs
; i
++) {
685 dmu_buf_t
*db
= dbp
[i
];
689 bufoff
= uio
->uio_loffset
- db
->db_offset
;
690 tocpy
= (int)MIN(db
->db_size
- bufoff
, size
);
692 err
= uiomove((char *)db
->db_data
+ bufoff
, tocpy
,
699 dmu_buf_rele_array(dbp
, numbufs
, FTAG
);
705 dmu_write_uio(objset_t
*os
, uint64_t object
, uio_t
*uio
, uint64_t size
,
715 err
= dmu_buf_hold_array(os
, object
, uio
->uio_loffset
, size
,
716 FALSE
, FTAG
, &numbufs
, &dbp
);
720 for (i
= 0; i
< numbufs
; i
++) {
723 dmu_buf_t
*db
= dbp
[i
];
727 bufoff
= uio
->uio_loffset
- db
->db_offset
;
728 tocpy
= (int)MIN(db
->db_size
- bufoff
, size
);
730 ASSERT(i
== 0 || i
== numbufs
-1 || tocpy
== db
->db_size
);
732 if (tocpy
== db
->db_size
)
733 dmu_buf_will_fill(db
, tx
);
735 dmu_buf_will_dirty(db
, tx
);
738 * XXX uiomove could block forever (eg. nfs-backed
739 * pages). There needs to be a uiolockdown() function
740 * to lock the pages in memory, so that uiomove won't
743 err
= uiomove((char *)db
->db_data
+ bufoff
, tocpy
,
746 if (tocpy
== db
->db_size
)
747 dmu_buf_fill_done(db
, tx
);
754 dmu_buf_rele_array(dbp
, numbufs
, FTAG
);
759 dmu_write_pages(objset_t
*os
, uint64_t object
, uint64_t offset
, uint64_t size
,
760 page_t
*pp
, dmu_tx_t
*tx
)
769 err
= dmu_buf_hold_array(os
, object
, offset
, size
,
770 FALSE
, FTAG
, &numbufs
, &dbp
);
774 for (i
= 0; i
< numbufs
; i
++) {
775 int tocpy
, copied
, thiscpy
;
777 dmu_buf_t
*db
= dbp
[i
];
781 ASSERT3U(db
->db_size
, >=, PAGESIZE
);
783 bufoff
= offset
- db
->db_offset
;
784 tocpy
= (int)MIN(db
->db_size
- bufoff
, size
);
786 ASSERT(i
== 0 || i
== numbufs
-1 || tocpy
== db
->db_size
);
788 if (tocpy
== db
->db_size
)
789 dmu_buf_will_fill(db
, tx
);
791 dmu_buf_will_dirty(db
, tx
);
793 for (copied
= 0; copied
< tocpy
; copied
+= PAGESIZE
) {
794 ASSERT3U(pp
->p_offset
, ==, db
->db_offset
+ bufoff
);
795 thiscpy
= MIN(PAGESIZE
, tocpy
- copied
);
796 va
= zfs_map_page(pp
, S_READ
);
797 bcopy(va
, (char *)db
->db_data
+ bufoff
, thiscpy
);
798 zfs_unmap_page(pp
, va
);
803 if (tocpy
== db
->db_size
)
804 dmu_buf_fill_done(db
, tx
);
812 dmu_buf_rele_array(dbp
, numbufs
, FTAG
);
818 * Allocate a loaned anonymous arc buffer.
821 dmu_request_arcbuf(dmu_buf_t
*handle
, int size
)
823 dnode_t
*dn
= ((dmu_buf_impl_t
*)handle
)->db_dnode
;
825 return (arc_loan_buf(dn
->dn_objset
->os_spa
, size
));
829 * Free a loaned arc buffer.
832 dmu_return_arcbuf(arc_buf_t
*buf
)
834 arc_return_buf(buf
, FTAG
);
835 VERIFY(arc_buf_remove_ref(buf
, FTAG
) == 1);
839 * When possible directly assign passed loaned arc buffer to a dbuf.
840 * If this is not possible copy the contents of passed arc buf via
844 dmu_assign_arcbuf(dmu_buf_t
*handle
, uint64_t offset
, arc_buf_t
*buf
,
847 dnode_t
*dn
= ((dmu_buf_impl_t
*)handle
)->db_dnode
;
849 uint32_t blksz
= (uint32_t)arc_buf_size(buf
);
852 rw_enter(&dn
->dn_struct_rwlock
, RW_READER
);
853 blkid
= dbuf_whichblock(dn
, offset
);
854 VERIFY((db
= dbuf_hold(dn
, blkid
, FTAG
)) != NULL
);
855 rw_exit(&dn
->dn_struct_rwlock
);
857 if (offset
== db
->db
.db_offset
&& blksz
== db
->db
.db_size
) {
858 dbuf_assign_arcbuf(db
, buf
, tx
);
862 ASSERT(dn
->dn_objset
->os
.os
== dn
->dn_objset
);
863 dmu_write(&dn
->dn_objset
->os
, dn
->dn_object
, offset
, blksz
,
865 dmu_return_arcbuf(buf
);
870 dbuf_dirty_record_t
*dr
;
877 dmu_sync_ready(zio_t
*zio
, arc_buf_t
*buf
, void *varg
)
879 blkptr_t
*bp
= zio
->io_bp
;
881 if (!BP_IS_HOLE(bp
)) {
882 dmu_sync_arg_t
*in
= varg
;
883 dbuf_dirty_record_t
*dr
= in
->dr
;
884 dmu_buf_impl_t
*db
= dr
->dr_dbuf
;
885 ASSERT(BP_GET_TYPE(bp
) == db
->db_dnode
->dn_type
);
886 ASSERT(BP_GET_LEVEL(bp
) == 0);
893 dmu_sync_done(zio_t
*zio
, arc_buf_t
*buf
, void *varg
)
895 dmu_sync_arg_t
*in
= varg
;
896 dbuf_dirty_record_t
*dr
= in
->dr
;
897 dmu_buf_impl_t
*db
= dr
->dr_dbuf
;
898 dmu_sync_cb_t
*done
= in
->done
;
900 mutex_enter(&db
->db_mtx
);
901 ASSERT(dr
->dt
.dl
.dr_override_state
== DR_IN_DMU_SYNC
);
902 dr
->dt
.dl
.dr_overridden_by
= *zio
->io_bp
; /* structure assignment */
903 dr
->dt
.dl
.dr_override_state
= DR_OVERRIDDEN
;
904 cv_broadcast(&db
->db_changed
);
905 mutex_exit(&db
->db_mtx
);
908 done(&(db
->db
), in
->arg
);
910 kmem_free(in
, sizeof (dmu_sync_arg_t
));
914 * Intent log support: sync the block associated with db to disk.
915 * N.B. and XXX: the caller is responsible for making sure that the
916 * data isn't changing while dmu_sync() is writing it.
920 * EEXIST: this txg has already been synced, so there's nothing to to.
921 * The caller should not log the write.
923 * ENOENT: the block was dbuf_free_range()'d, so there's nothing to do.
924 * The caller should not log the write.
926 * EALREADY: this block is already in the process of being synced.
927 * The caller should track its progress (somehow).
929 * EINPROGRESS: the IO has been initiated.
930 * The caller should log this blkptr in the callback.
932 * 0: completed. Sets *bp to the blkptr just written.
933 * The caller should log this blkptr immediately.
936 dmu_sync(zio_t
*pio
, dmu_buf_t
*db_fake
,
937 blkptr_t
*bp
, uint64_t txg
, dmu_sync_cb_t
*done
, void *arg
)
939 dmu_buf_impl_t
*db
= (dmu_buf_impl_t
*)db_fake
;
940 objset_impl_t
*os
= db
->db_objset
;
941 dsl_pool_t
*dp
= os
->os_dsl_dataset
->ds_dir
->dd_pool
;
942 tx_state_t
*tx
= &dp
->dp_tx
;
943 dbuf_dirty_record_t
*dr
;
946 writeprops_t wp
= { 0 };
950 ASSERT(BP_IS_HOLE(bp
));
953 dprintf("dmu_sync txg=%llu, s,o,q %llu %llu %llu\n",
954 txg
, tx
->tx_synced_txg
, tx
->tx_open_txg
, tx
->tx_quiesced_txg
);
957 * XXX - would be nice if we could do this without suspending...
962 * If this txg already synced, there's nothing to do.
964 if (txg
<= tx
->tx_synced_txg
) {
967 * If we're running ziltest, we need the blkptr regardless.
969 if (txg
> spa_freeze_txg(dp
->dp_spa
)) {
970 /* if db_blkptr == NULL, this was an empty write */
972 *bp
= *db
->db_blkptr
; /* structure assignment */
978 mutex_enter(&db
->db_mtx
);
980 if (txg
== tx
->tx_syncing_txg
) {
981 while (db
->db_data_pending
) {
983 * IO is in-progress. Wait for it to finish.
984 * XXX - would be nice to be able to somehow "attach"
985 * this zio to the parent zio passed in.
987 cv_wait(&db
->db_changed
, &db
->db_mtx
);
988 if (!db
->db_data_pending
&&
989 db
->db_blkptr
&& BP_IS_HOLE(db
->db_blkptr
)) {
991 * IO was compressed away
993 *bp
= *db
->db_blkptr
; /* structure assignment */
994 mutex_exit(&db
->db_mtx
);
998 ASSERT(db
->db_data_pending
||
999 (db
->db_blkptr
&& db
->db_blkptr
->blk_birth
== txg
));
1002 if (db
->db_blkptr
&& db
->db_blkptr
->blk_birth
== txg
) {
1004 * IO is already completed.
1006 *bp
= *db
->db_blkptr
; /* structure assignment */
1007 mutex_exit(&db
->db_mtx
);
1013 dr
= db
->db_last_dirty
;
1014 while (dr
&& dr
->dr_txg
> txg
)
1016 if (dr
== NULL
|| dr
->dr_txg
< txg
) {
1018 * This dbuf isn't dirty, must have been free_range'd.
1019 * There's no need to log writes to freed blocks, so we're done.
1021 mutex_exit(&db
->db_mtx
);
1026 ASSERT(dr
->dr_txg
== txg
);
1027 if (dr
->dt
.dl
.dr_override_state
== DR_IN_DMU_SYNC
) {
1029 * We have already issued a sync write for this buffer.
1031 mutex_exit(&db
->db_mtx
);
1034 } else if (dr
->dt
.dl
.dr_override_state
== DR_OVERRIDDEN
) {
1036 * This buffer has already been synced. It could not
1037 * have been dirtied since, or we would have cleared the state.
1039 *bp
= dr
->dt
.dl
.dr_overridden_by
; /* structure assignment */
1040 mutex_exit(&db
->db_mtx
);
1045 dr
->dt
.dl
.dr_override_state
= DR_IN_DMU_SYNC
;
1046 in
= kmem_alloc(sizeof (dmu_sync_arg_t
), KM_SLEEP
);
1050 mutex_exit(&db
->db_mtx
);
1053 zb
.zb_objset
= os
->os_dsl_dataset
->ds_object
;
1054 zb
.zb_object
= db
->db
.db_object
;
1055 zb
.zb_level
= db
->db_level
;
1056 zb
.zb_blkid
= db
->db_blkid
;
1058 wp
.wp_type
= db
->db_dnode
->dn_type
;
1059 wp
.wp_level
= db
->db_level
;
1060 wp
.wp_copies
= os
->os_copies
;
1061 wp
.wp_dnchecksum
= db
->db_dnode
->dn_checksum
;
1062 wp
.wp_oschecksum
= os
->os_checksum
;
1063 wp
.wp_dncompress
= db
->db_dnode
->dn_compress
;
1064 wp
.wp_oscompress
= os
->os_compress
;
1066 ASSERT(BP_IS_HOLE(bp
));
1068 zio
= arc_write(pio
, os
->os_spa
, &wp
, DBUF_IS_L2CACHEABLE(db
),
1069 txg
, bp
, dr
->dt
.dl
.dr_data
, dmu_sync_ready
, dmu_sync_done
, in
,
1070 ZIO_PRIORITY_SYNC_WRITE
, ZIO_FLAG_MUSTSUCCEED
, &zb
);
1075 err
= zio_wait(zio
);
1082 dmu_object_set_blocksize(objset_t
*os
, uint64_t object
, uint64_t size
, int ibs
,
1088 err
= dnode_hold(os
->os
, object
, FTAG
, &dn
);
1091 err
= dnode_set_blksz(dn
, size
, ibs
, tx
);
1092 dnode_rele(dn
, FTAG
);
1097 dmu_object_set_checksum(objset_t
*os
, uint64_t object
, uint8_t checksum
,
1102 /* XXX assumes dnode_hold will not get an i/o error */
1103 (void) dnode_hold(os
->os
, object
, FTAG
, &dn
);
1104 ASSERT(checksum
< ZIO_CHECKSUM_FUNCTIONS
);
1105 dn
->dn_checksum
= checksum
;
1106 dnode_setdirty(dn
, tx
);
1107 dnode_rele(dn
, FTAG
);
1111 dmu_object_set_compress(objset_t
*os
, uint64_t object
, uint8_t compress
,
1116 /* XXX assumes dnode_hold will not get an i/o error */
1117 (void) dnode_hold(os
->os
, object
, FTAG
, &dn
);
1118 ASSERT(compress
< ZIO_COMPRESS_FUNCTIONS
);
1119 dn
->dn_compress
= compress
;
1120 dnode_setdirty(dn
, tx
);
1121 dnode_rele(dn
, FTAG
);
1125 dmu_offset_next(objset_t
*os
, uint64_t object
, boolean_t hole
, uint64_t *off
)
1130 err
= dnode_hold(os
->os
, object
, FTAG
, &dn
);
1134 * Sync any current changes before
1135 * we go trundling through the block pointers.
1137 for (i
= 0; i
< TXG_SIZE
; i
++) {
1138 if (list_link_active(&dn
->dn_dirty_link
[i
]))
1141 if (i
!= TXG_SIZE
) {
1142 dnode_rele(dn
, FTAG
);
1143 txg_wait_synced(dmu_objset_pool(os
), 0);
1144 err
= dnode_hold(os
->os
, object
, FTAG
, &dn
);
1149 err
= dnode_next_offset(dn
, (hole
? DNODE_FIND_HOLE
: 0), off
, 1, 1, 0);
1150 dnode_rele(dn
, FTAG
);
1156 dmu_object_info_from_dnode(dnode_t
*dn
, dmu_object_info_t
*doi
)
1158 rw_enter(&dn
->dn_struct_rwlock
, RW_READER
);
1159 mutex_enter(&dn
->dn_mtx
);
1161 doi
->doi_data_block_size
= dn
->dn_datablksz
;
1162 doi
->doi_metadata_block_size
= dn
->dn_indblkshift
?
1163 1ULL << dn
->dn_indblkshift
: 0;
1164 doi
->doi_indirection
= dn
->dn_nlevels
;
1165 doi
->doi_checksum
= dn
->dn_checksum
;
1166 doi
->doi_compress
= dn
->dn_compress
;
1167 doi
->doi_physical_blks
= (DN_USED_BYTES(dn
->dn_phys
) +
1168 SPA_MINBLOCKSIZE
/2) >> SPA_MINBLOCKSHIFT
;
1169 doi
->doi_max_block_offset
= dn
->dn_phys
->dn_maxblkid
;
1170 doi
->doi_type
= dn
->dn_type
;
1171 doi
->doi_bonus_size
= dn
->dn_bonuslen
;
1172 doi
->doi_bonus_type
= dn
->dn_bonustype
;
1174 mutex_exit(&dn
->dn_mtx
);
1175 rw_exit(&dn
->dn_struct_rwlock
);
1179 * Get information on a DMU object.
1180 * If doi is NULL, just indicates whether the object exists.
1183 dmu_object_info(objset_t
*os
, uint64_t object
, dmu_object_info_t
*doi
)
1186 int err
= dnode_hold(os
->os
, object
, FTAG
, &dn
);
1192 dmu_object_info_from_dnode(dn
, doi
);
1194 dnode_rele(dn
, FTAG
);
1199 * As above, but faster; can be used when you have a held dbuf in hand.
1202 dmu_object_info_from_db(dmu_buf_t
*db
, dmu_object_info_t
*doi
)
1204 dmu_object_info_from_dnode(((dmu_buf_impl_t
*)db
)->db_dnode
, doi
);
1208 * Faster still when you only care about the size.
1209 * This is specifically optimized for zfs_getattr().
1212 dmu_object_size_from_db(dmu_buf_t
*db
, uint32_t *blksize
, u_longlong_t
*nblk512
)
1214 dnode_t
*dn
= ((dmu_buf_impl_t
*)db
)->db_dnode
;
1216 *blksize
= dn
->dn_datablksz
;
1217 /* add 1 for dnode space */
1218 *nblk512
= ((DN_USED_BYTES(dn
->dn_phys
) + SPA_MINBLOCKSIZE
/2) >>
1219 SPA_MINBLOCKSHIFT
) + 1;
1223 byteswap_uint64_array(void *vbuf
, size_t size
)
1225 uint64_t *buf
= vbuf
;
1226 size_t count
= size
>> 3;
1229 ASSERT((size
& 7) == 0);
1231 for (i
= 0; i
< count
; i
++)
1232 buf
[i
] = BSWAP_64(buf
[i
]);
1236 byteswap_uint32_array(void *vbuf
, size_t size
)
1238 uint32_t *buf
= vbuf
;
1239 size_t count
= size
>> 2;
1242 ASSERT((size
& 3) == 0);
1244 for (i
= 0; i
< count
; i
++)
1245 buf
[i
] = BSWAP_32(buf
[i
]);
1249 byteswap_uint16_array(void *vbuf
, size_t size
)
1251 uint16_t *buf
= vbuf
;
1252 size_t count
= size
>> 1;
1255 ASSERT((size
& 1) == 0);
1257 for (i
= 0; i
< count
; i
++)
1258 buf
[i
] = BSWAP_16(buf
[i
]);
1263 byteswap_uint8_array(void *vbuf
, size_t size
)