4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
24 * Copyright (c) 2017 Datto Inc.
27 #include <sys/bpobj.h>
28 #include <sys/zfs_context.h>
29 #include <sys/refcount.h>
30 #include <sys/dsl_pool.h>
31 #include <sys/zfeature.h>
35 * Return an empty bpobj, preferably the empty dummy one (dp_empty_bpobj).
38 bpobj_alloc_empty(objset_t
*os
, int blocksize
, dmu_tx_t
*tx
)
40 spa_t
*spa
= dmu_objset_spa(os
);
41 dsl_pool_t
*dp
= dmu_objset_pool(os
);
43 if (spa_feature_is_enabled(spa
, SPA_FEATURE_EMPTY_BPOBJ
)) {
44 if (!spa_feature_is_active(spa
, SPA_FEATURE_EMPTY_BPOBJ
)) {
45 ASSERT0(dp
->dp_empty_bpobj
);
47 bpobj_alloc(os
, SPA_OLD_MAXBLOCKSIZE
, tx
);
49 DMU_POOL_DIRECTORY_OBJECT
,
50 DMU_POOL_EMPTY_BPOBJ
, sizeof (uint64_t), 1,
51 &dp
->dp_empty_bpobj
, tx
) == 0);
53 spa_feature_incr(spa
, SPA_FEATURE_EMPTY_BPOBJ
, tx
);
54 ASSERT(dp
->dp_empty_bpobj
!= 0);
55 return (dp
->dp_empty_bpobj
);
57 return (bpobj_alloc(os
, blocksize
, tx
));
62 bpobj_decr_empty(objset_t
*os
, dmu_tx_t
*tx
)
64 dsl_pool_t
*dp
= dmu_objset_pool(os
);
66 spa_feature_decr(dmu_objset_spa(os
), SPA_FEATURE_EMPTY_BPOBJ
, tx
);
67 if (!spa_feature_is_active(dmu_objset_spa(os
),
68 SPA_FEATURE_EMPTY_BPOBJ
)) {
69 VERIFY3U(0, ==, zap_remove(dp
->dp_meta_objset
,
70 DMU_POOL_DIRECTORY_OBJECT
,
71 DMU_POOL_EMPTY_BPOBJ
, tx
));
72 VERIFY3U(0, ==, dmu_object_free(os
, dp
->dp_empty_bpobj
, tx
));
73 dp
->dp_empty_bpobj
= 0;
78 bpobj_alloc(objset_t
*os
, int blocksize
, dmu_tx_t
*tx
)
82 if (spa_version(dmu_objset_spa(os
)) < SPA_VERSION_BPOBJ_ACCOUNT
)
84 else if (spa_version(dmu_objset_spa(os
)) < SPA_VERSION_DEADLISTS
)
87 size
= sizeof (bpobj_phys_t
);
89 return (dmu_object_alloc(os
, DMU_OT_BPOBJ
, blocksize
,
90 DMU_OT_BPOBJ_HDR
, size
, tx
));
94 bpobj_free(objset_t
*os
, uint64_t obj
, dmu_tx_t
*tx
)
98 dmu_object_info_t doi
;
100 dmu_buf_t
*dbuf
= NULL
;
102 ASSERT(obj
!= dmu_objset_pool(os
)->dp_empty_bpobj
);
103 VERIFY3U(0, ==, bpobj_open(&bpo
, os
, obj
));
105 mutex_enter(&bpo
.bpo_lock
);
107 if (!bpo
.bpo_havesubobj
|| bpo
.bpo_phys
->bpo_subobjs
== 0)
110 VERIFY3U(0, ==, dmu_object_info(os
, bpo
.bpo_phys
->bpo_subobjs
, &doi
));
111 epb
= doi
.doi_data_block_size
/ sizeof (uint64_t);
113 for (i
= bpo
.bpo_phys
->bpo_num_subobjs
- 1; i
>= 0; i
--) {
115 uint64_t offset
, blkoff
;
117 offset
= i
* sizeof (uint64_t);
118 blkoff
= P2PHASE(i
, epb
);
120 if (dbuf
== NULL
|| dbuf
->db_offset
> offset
) {
122 dmu_buf_rele(dbuf
, FTAG
);
123 VERIFY3U(0, ==, dmu_buf_hold(os
,
124 bpo
.bpo_phys
->bpo_subobjs
, offset
, FTAG
, &dbuf
, 0));
127 ASSERT3U(offset
, >=, dbuf
->db_offset
);
128 ASSERT3U(offset
, <, dbuf
->db_offset
+ dbuf
->db_size
);
130 objarray
= dbuf
->db_data
;
131 bpobj_free(os
, objarray
[blkoff
], tx
);
134 dmu_buf_rele(dbuf
, FTAG
);
137 VERIFY3U(0, ==, dmu_object_free(os
, bpo
.bpo_phys
->bpo_subobjs
, tx
));
140 mutex_exit(&bpo
.bpo_lock
);
143 VERIFY3U(0, ==, dmu_object_free(os
, obj
, tx
));
147 bpobj_open(bpobj_t
*bpo
, objset_t
*os
, uint64_t object
)
149 dmu_object_info_t doi
;
152 err
= dmu_object_info(os
, object
, &doi
);
156 bzero(bpo
, sizeof (*bpo
));
157 mutex_init(&bpo
->bpo_lock
, NULL
, MUTEX_DEFAULT
, NULL
);
159 ASSERT(bpo
->bpo_dbuf
== NULL
);
160 ASSERT(bpo
->bpo_phys
== NULL
);
162 ASSERT3U(doi
.doi_type
, ==, DMU_OT_BPOBJ
);
163 ASSERT3U(doi
.doi_bonus_type
, ==, DMU_OT_BPOBJ_HDR
);
165 err
= dmu_bonus_hold(os
, object
, bpo
, &bpo
->bpo_dbuf
);
170 bpo
->bpo_object
= object
;
171 bpo
->bpo_epb
= doi
.doi_data_block_size
>> SPA_BLKPTRSHIFT
;
172 bpo
->bpo_havecomp
= (doi
.doi_bonus_size
> BPOBJ_SIZE_V0
);
173 bpo
->bpo_havesubobj
= (doi
.doi_bonus_size
> BPOBJ_SIZE_V1
);
174 bpo
->bpo_phys
= bpo
->bpo_dbuf
->db_data
;
179 bpobj_is_open(const bpobj_t
*bpo
)
181 return (bpo
->bpo_object
!= 0);
185 bpobj_close(bpobj_t
*bpo
)
187 /* Lame workaround for closing a bpobj that was never opened. */
188 if (bpo
->bpo_object
== 0)
191 dmu_buf_rele(bpo
->bpo_dbuf
, bpo
);
192 if (bpo
->bpo_cached_dbuf
!= NULL
)
193 dmu_buf_rele(bpo
->bpo_cached_dbuf
, bpo
);
194 bpo
->bpo_dbuf
= NULL
;
195 bpo
->bpo_phys
= NULL
;
196 bpo
->bpo_cached_dbuf
= NULL
;
199 mutex_destroy(&bpo
->bpo_lock
);
203 bpobj_is_empty(bpobj_t
*bpo
)
205 return (bpo
->bpo_phys
->bpo_num_blkptrs
== 0 &&
206 (!bpo
->bpo_havesubobj
|| bpo
->bpo_phys
->bpo_num_subobjs
== 0));
210 bpobj_iterate_impl(bpobj_t
*bpo
, bpobj_itor_t func
, void *arg
, dmu_tx_t
*tx
,
213 dmu_object_info_t doi
;
217 dmu_buf_t
*dbuf
= NULL
;
219 ASSERT(bpobj_is_open(bpo
));
220 mutex_enter(&bpo
->bpo_lock
);
223 dmu_buf_will_dirty(bpo
->bpo_dbuf
, tx
);
225 for (i
= bpo
->bpo_phys
->bpo_num_blkptrs
- 1; i
>= 0; i
--) {
228 uint64_t offset
, blkoff
;
230 offset
= i
* sizeof (blkptr_t
);
231 blkoff
= P2PHASE(i
, bpo
->bpo_epb
);
233 if (dbuf
== NULL
|| dbuf
->db_offset
> offset
) {
235 dmu_buf_rele(dbuf
, FTAG
);
236 err
= dmu_buf_hold(bpo
->bpo_os
, bpo
->bpo_object
, offset
,
242 ASSERT3U(offset
, >=, dbuf
->db_offset
);
243 ASSERT3U(offset
, <, dbuf
->db_offset
+ dbuf
->db_size
);
245 bparray
= dbuf
->db_data
;
246 bp
= &bparray
[blkoff
];
247 err
= func(arg
, bp
, tx
);
251 bpo
->bpo_phys
->bpo_bytes
-=
252 bp_get_dsize_sync(dmu_objset_spa(bpo
->bpo_os
), bp
);
253 ASSERT3S(bpo
->bpo_phys
->bpo_bytes
, >=, 0);
254 if (bpo
->bpo_havecomp
) {
255 bpo
->bpo_phys
->bpo_comp
-= BP_GET_PSIZE(bp
);
256 bpo
->bpo_phys
->bpo_uncomp
-= BP_GET_UCSIZE(bp
);
258 bpo
->bpo_phys
->bpo_num_blkptrs
--;
259 ASSERT3S(bpo
->bpo_phys
->bpo_num_blkptrs
, >=, 0);
263 dmu_buf_rele(dbuf
, FTAG
);
267 VERIFY3U(0, ==, dmu_free_range(bpo
->bpo_os
, bpo
->bpo_object
,
268 (i
+ 1) * sizeof (blkptr_t
), DMU_OBJECT_END
, tx
));
270 if (err
|| !bpo
->bpo_havesubobj
|| bpo
->bpo_phys
->bpo_subobjs
== 0)
273 ASSERT(bpo
->bpo_havecomp
);
274 err
= dmu_object_info(bpo
->bpo_os
, bpo
->bpo_phys
->bpo_subobjs
, &doi
);
276 mutex_exit(&bpo
->bpo_lock
);
279 ASSERT3U(doi
.doi_type
, ==, DMU_OT_BPOBJ_SUBOBJ
);
280 epb
= doi
.doi_data_block_size
/ sizeof (uint64_t);
282 for (i
= bpo
->bpo_phys
->bpo_num_subobjs
- 1; i
>= 0; i
--) {
284 uint64_t offset
, blkoff
;
286 uint64_t used_before
, comp_before
, uncomp_before
;
287 uint64_t used_after
, comp_after
, uncomp_after
;
289 offset
= i
* sizeof (uint64_t);
290 blkoff
= P2PHASE(i
, epb
);
292 if (dbuf
== NULL
|| dbuf
->db_offset
> offset
) {
294 dmu_buf_rele(dbuf
, FTAG
);
295 err
= dmu_buf_hold(bpo
->bpo_os
,
296 bpo
->bpo_phys
->bpo_subobjs
, offset
, FTAG
, &dbuf
, 0);
301 ASSERT3U(offset
, >=, dbuf
->db_offset
);
302 ASSERT3U(offset
, <, dbuf
->db_offset
+ dbuf
->db_size
);
304 objarray
= dbuf
->db_data
;
305 err
= bpobj_open(&sublist
, bpo
->bpo_os
, objarray
[blkoff
]);
309 err
= bpobj_space(&sublist
,
310 &used_before
, &comp_before
, &uncomp_before
);
312 bpobj_close(&sublist
);
316 err
= bpobj_iterate_impl(&sublist
, func
, arg
, tx
, free
);
318 VERIFY3U(0, ==, bpobj_space(&sublist
,
319 &used_after
, &comp_after
, &uncomp_after
));
320 bpo
->bpo_phys
->bpo_bytes
-= used_before
- used_after
;
321 ASSERT3S(bpo
->bpo_phys
->bpo_bytes
, >=, 0);
322 bpo
->bpo_phys
->bpo_comp
-= comp_before
- comp_after
;
323 bpo
->bpo_phys
->bpo_uncomp
-=
324 uncomp_before
- uncomp_after
;
327 bpobj_close(&sublist
);
331 err
= dmu_object_free(bpo
->bpo_os
,
332 objarray
[blkoff
], tx
);
335 bpo
->bpo_phys
->bpo_num_subobjs
--;
336 ASSERT3S(bpo
->bpo_phys
->bpo_num_subobjs
, >=, 0);
340 dmu_buf_rele(dbuf
, FTAG
);
344 VERIFY3U(0, ==, dmu_free_range(bpo
->bpo_os
,
345 bpo
->bpo_phys
->bpo_subobjs
,
346 (i
+ 1) * sizeof (uint64_t), DMU_OBJECT_END
, tx
));
350 /* If there are no entries, there should be no bytes. */
351 if (bpobj_is_empty(bpo
)) {
352 ASSERT0(bpo
->bpo_phys
->bpo_bytes
);
353 ASSERT0(bpo
->bpo_phys
->bpo_comp
);
354 ASSERT0(bpo
->bpo_phys
->bpo_uncomp
);
357 mutex_exit(&bpo
->bpo_lock
);
362 * Iterate and remove the entries. If func returns nonzero, iteration
363 * will stop and that entry will not be removed.
366 bpobj_iterate(bpobj_t
*bpo
, bpobj_itor_t func
, void *arg
, dmu_tx_t
*tx
)
368 return (bpobj_iterate_impl(bpo
, func
, arg
, tx
, B_TRUE
));
372 * Iterate the entries. If func returns nonzero, iteration will stop.
375 bpobj_iterate_nofree(bpobj_t
*bpo
, bpobj_itor_t func
, void *arg
, dmu_tx_t
*tx
)
377 return (bpobj_iterate_impl(bpo
, func
, arg
, tx
, B_FALSE
));
381 * Logically add subobj's contents to the parent bpobj.
383 * In the most general case, this is accomplished in constant time by adding
384 * a reference to subobj. This case is used when enqueuing a large subobj:
385 * +--------------+ +--------------+
386 * | bpobj |----------------------->| subobj list |
387 * +----+----+----+----+----+ +-----+-----+--+--+
388 * | bp | bp | bp | bp | bp | | obj | obj | obj |
389 * +----+----+----+----+----+ +-----+-----+-----+
391 * +--------------+ +--------------+
392 * | sub-bpobj |----------------------> | subsubobj |
393 * +----+----+----+----+---------+----+ +-----+-----+--+--------+-----+
394 * | bp | bp | bp | bp | ... | bp | | obj | obj | ... | obj |
395 * +----+----+----+----+---------+----+ +-----+-----+-----------+-----+
397 * Result: sub-bpobj added to parent's subobj list.
398 * +--------------+ +--------------+
399 * | bpobj |----------------------->| subobj list |
400 * +----+----+----+----+----+ +-----+-----+--+--+-----+
401 * | bp | bp | bp | bp | bp | | obj | obj | obj | OBJ |
402 * +----+----+----+----+----+ +-----+-----+-----+--|--+
404 * /-----------------------------------------------------/
406 * +--------------+ +--------------+
407 * | sub-bpobj |----------------------> | subsubobj |
408 * +----+----+----+----+---------+----+ +-----+-----+--+--------+-----+
409 * | bp | bp | bp | bp | ... | bp | | obj | obj | ... | obj |
410 * +----+----+----+----+---------+----+ +-----+-----+-----------+-----+
413 * In a common case, the subobj is small: its bp's and its list of subobj's
414 * are each stored in a single block. In this case we copy the subobj's
415 * contents to the parent:
416 * +--------------+ +--------------+
417 * | bpobj |----------------------->| subobj list |
418 * +----+----+----+----+----+ +-----+-----+--+--+
419 * | bp | bp | bp | bp | bp | | obj | obj | obj |
420 * +----+----+----+----+----+ +-----+-----+-----+
422 * +--------------+ | +--------------+ |
423 * | sub-bpobj |---------^------------> | subsubobj | ^
424 * +----+----+----+ | +-----+-----+--+ |
425 * | BP | BP |-->-->-->-->-/ | OBJ | OBJ |-->-/
426 * +----+----+ +-----+-----+
428 * Result: subobj destroyed, contents copied to parent:
429 * +--------------+ +--------------+
430 * | bpobj |----------------------->| subobj list |
431 * +----+----+----+----+----+----+----+ +-----+-----+--+--+-----+-----+
432 * | bp | bp | bp | bp | bp | BP | BP | | obj | obj | obj | OBJ | OBJ |
433 * +----+----+----+----+----+----+----+ +-----+-----+-----+-----+-----+
436 * If the subobj has many BP's but few subobj's, we can copy the sub-subobj's
437 * but retain the sub-bpobj:
438 * +--------------+ +--------------+
439 * | bpobj |----------------------->| subobj list |
440 * +----+----+----+----+----+ +-----+-----+--+--+
441 * | bp | bp | bp | bp | bp | | obj | obj | obj |
442 * +----+----+----+----+----+ +-----+-----+-----+
444 * +--------------+ +--------------+ |
445 * | sub-bpobj |----------------------> | subsubobj | ^
446 * +----+----+----+----+---------+----+ +-----+-----+--+ |
447 * | bp | bp | bp | bp | ... | bp | | OBJ | OBJ |-->-/
448 * +----+----+----+----+---------+----+ +-----+-----+
450 * Result: sub-sub-bpobjs and subobj added to parent's subobj list.
451 * +--------------+ +--------------+
452 * | bpobj |-------------------->| subobj list |
453 * +----+----+----+----+----+ +-----+-----+--+--+-----+-----+------+
454 * | bp | bp | bp | bp | bp | | obj | obj | obj | OBJ | OBJ | OBJ* |
455 * +----+----+----+----+----+ +-----+-----+-----+-----+-----+--|---+
457 * /--------------------------------------------------------------/
461 * +----+----+----+----+---------+----+
462 * | bp | bp | bp | bp | ... | bp |
463 * +----+----+----+----+---------+----+
466 bpobj_enqueue_subobj(bpobj_t
*bpo
, uint64_t subobj
, dmu_tx_t
*tx
)
469 uint64_t used
, comp
, uncomp
, subsubobjs
;
470 boolean_t copy_subsub
= B_TRUE
;
471 boolean_t copy_bps
= B_TRUE
;
473 ASSERT(bpobj_is_open(bpo
));
475 ASSERT(bpo
->bpo_havesubobj
);
476 ASSERT(bpo
->bpo_havecomp
);
477 ASSERT(bpo
->bpo_object
!= dmu_objset_pool(bpo
->bpo_os
)->dp_empty_bpobj
);
479 if (subobj
== dmu_objset_pool(bpo
->bpo_os
)->dp_empty_bpobj
) {
480 bpobj_decr_empty(bpo
->bpo_os
, tx
);
484 VERIFY3U(0, ==, bpobj_open(&subbpo
, bpo
->bpo_os
, subobj
));
485 VERIFY3U(0, ==, bpobj_space(&subbpo
, &used
, &comp
, &uncomp
));
487 if (bpobj_is_empty(&subbpo
)) {
488 /* No point in having an empty subobj. */
489 bpobj_close(&subbpo
);
490 bpobj_free(bpo
->bpo_os
, subobj
, tx
);
494 mutex_enter(&bpo
->bpo_lock
);
495 dmu_buf_will_dirty(bpo
->bpo_dbuf
, tx
);
497 dmu_object_info_t doi
;
499 if (bpo
->bpo_phys
->bpo_subobjs
!= 0) {
500 ASSERT0(dmu_object_info(bpo
->bpo_os
, bpo
->bpo_phys
->bpo_subobjs
,
502 ASSERT3U(doi
.doi_type
, ==, DMU_OT_BPOBJ_SUBOBJ
);
506 * If subobj has only one block of subobjs, then move subobj's
507 * subobjs to bpo's subobj list directly. This reduces recursion in
508 * bpobj_iterate due to nested subobjs.
510 subsubobjs
= subbpo
.bpo_phys
->bpo_subobjs
;
511 if (subsubobjs
!= 0) {
512 VERIFY0(dmu_object_info(bpo
->bpo_os
, subsubobjs
, &doi
));
513 if (doi
.doi_max_offset
> doi
.doi_data_block_size
) {
514 copy_subsub
= B_FALSE
;
519 * If, in addition to having only one block of subobj's, subobj has
520 * only one block of bp's, then move subobj's bp's to bpo's bp list
521 * directly. This reduces recursion in bpobj_iterate due to nested
524 VERIFY3U(0, ==, dmu_object_info(bpo
->bpo_os
, subobj
, &doi
));
525 if (doi
.doi_max_offset
> doi
.doi_data_block_size
|| !copy_subsub
) {
529 if (copy_subsub
&& subsubobjs
!= 0) {
531 uint64_t numsubsub
= subbpo
.bpo_phys
->bpo_num_subobjs
;
533 VERIFY0(dmu_buf_hold(bpo
->bpo_os
, subsubobjs
,
534 0, FTAG
, &subdb
, 0));
536 * Make sure that we are not asking dmu_write()
537 * to write more data than we have in our buffer.
539 VERIFY3U(subdb
->db_size
, >=,
540 numsubsub
* sizeof (subobj
));
541 if (bpo
->bpo_phys
->bpo_subobjs
== 0) {
542 bpo
->bpo_phys
->bpo_subobjs
=
543 dmu_object_alloc(bpo
->bpo_os
,
544 DMU_OT_BPOBJ_SUBOBJ
, SPA_OLD_MAXBLOCKSIZE
,
547 dmu_write(bpo
->bpo_os
, bpo
->bpo_phys
->bpo_subobjs
,
548 bpo
->bpo_phys
->bpo_num_subobjs
* sizeof (subobj
),
549 numsubsub
* sizeof (subobj
), subdb
->db_data
, tx
);
550 dmu_buf_rele(subdb
, FTAG
);
551 bpo
->bpo_phys
->bpo_num_subobjs
+= numsubsub
;
553 dmu_buf_will_dirty(subbpo
.bpo_dbuf
, tx
);
554 subbpo
.bpo_phys
->bpo_subobjs
= 0;
555 VERIFY0(dmu_object_free(bpo
->bpo_os
, subsubobjs
, tx
));
560 uint64_t numbps
= subbpo
.bpo_phys
->bpo_num_blkptrs
;
563 VERIFY0(dmu_buf_hold(bpo
->bpo_os
, subobj
,
567 * Make sure that we are not asking dmu_write()
568 * to write more data than we have in our buffer.
570 VERIFY3U(bps
->db_size
, >=, numbps
* sizeof (blkptr_t
));
571 dmu_write(bpo
->bpo_os
, bpo
->bpo_object
,
572 bpo
->bpo_phys
->bpo_num_blkptrs
* sizeof (blkptr_t
),
573 numbps
* sizeof (blkptr_t
),
575 dmu_buf_rele(bps
, FTAG
);
576 bpo
->bpo_phys
->bpo_num_blkptrs
+= numbps
;
578 bpobj_close(&subbpo
);
579 VERIFY0(dmu_object_free(bpo
->bpo_os
, subobj
, tx
));
581 bpobj_close(&subbpo
);
582 if (bpo
->bpo_phys
->bpo_subobjs
== 0) {
583 bpo
->bpo_phys
->bpo_subobjs
=
584 dmu_object_alloc(bpo
->bpo_os
,
585 DMU_OT_BPOBJ_SUBOBJ
, SPA_OLD_MAXBLOCKSIZE
,
589 dmu_write(bpo
->bpo_os
, bpo
->bpo_phys
->bpo_subobjs
,
590 bpo
->bpo_phys
->bpo_num_subobjs
* sizeof (subobj
),
591 sizeof (subobj
), &subobj
, tx
);
592 bpo
->bpo_phys
->bpo_num_subobjs
++;
595 bpo
->bpo_phys
->bpo_bytes
+= used
;
596 bpo
->bpo_phys
->bpo_comp
+= comp
;
597 bpo
->bpo_phys
->bpo_uncomp
+= uncomp
;
598 mutex_exit(&bpo
->bpo_lock
);
603 bpobj_enqueue(bpobj_t
*bpo
, const blkptr_t
*bp
, dmu_tx_t
*tx
)
605 blkptr_t stored_bp
= *bp
;
610 ASSERT(bpobj_is_open(bpo
));
611 ASSERT(!BP_IS_HOLE(bp
));
612 ASSERT(bpo
->bpo_object
!= dmu_objset_pool(bpo
->bpo_os
)->dp_empty_bpobj
);
614 if (BP_IS_EMBEDDED(bp
)) {
616 * The bpobj will compress better without the payload.
618 * Note that we store EMBEDDED bp's because they have an
619 * uncompressed size, which must be accounted for. An
620 * alternative would be to add their size to bpo_uncomp
621 * without storing the bp, but that would create additional
622 * complications: bpo_uncomp would be inconsistent with the
623 * set of BP's stored, and bpobj_iterate() wouldn't visit
624 * all the space accounted for in the bpobj.
626 bzero(&stored_bp
, sizeof (stored_bp
));
627 stored_bp
.blk_prop
= bp
->blk_prop
;
628 stored_bp
.blk_birth
= bp
->blk_birth
;
629 } else if (!BP_GET_DEDUP(bp
)) {
630 /* The bpobj will compress better without the checksum */
631 bzero(&stored_bp
.blk_cksum
, sizeof (stored_bp
.blk_cksum
));
634 /* We never need the fill count. */
635 stored_bp
.blk_fill
= 0;
637 mutex_enter(&bpo
->bpo_lock
);
639 offset
= bpo
->bpo_phys
->bpo_num_blkptrs
* sizeof (stored_bp
);
640 blkoff
= P2PHASE(bpo
->bpo_phys
->bpo_num_blkptrs
, bpo
->bpo_epb
);
642 if (bpo
->bpo_cached_dbuf
== NULL
||
643 offset
< bpo
->bpo_cached_dbuf
->db_offset
||
644 offset
>= bpo
->bpo_cached_dbuf
->db_offset
+
645 bpo
->bpo_cached_dbuf
->db_size
) {
646 if (bpo
->bpo_cached_dbuf
)
647 dmu_buf_rele(bpo
->bpo_cached_dbuf
, bpo
);
648 VERIFY3U(0, ==, dmu_buf_hold(bpo
->bpo_os
, bpo
->bpo_object
,
649 offset
, bpo
, &bpo
->bpo_cached_dbuf
, 0));
652 dmu_buf_will_dirty(bpo
->bpo_cached_dbuf
, tx
);
653 bparray
= bpo
->bpo_cached_dbuf
->db_data
;
654 bparray
[blkoff
] = stored_bp
;
656 dmu_buf_will_dirty(bpo
->bpo_dbuf
, tx
);
657 bpo
->bpo_phys
->bpo_num_blkptrs
++;
658 bpo
->bpo_phys
->bpo_bytes
+=
659 bp_get_dsize_sync(dmu_objset_spa(bpo
->bpo_os
), bp
);
660 if (bpo
->bpo_havecomp
) {
661 bpo
->bpo_phys
->bpo_comp
+= BP_GET_PSIZE(bp
);
662 bpo
->bpo_phys
->bpo_uncomp
+= BP_GET_UCSIZE(bp
);
664 mutex_exit(&bpo
->bpo_lock
);
667 struct space_range_arg
{
678 space_range_cb(void *arg
, const blkptr_t
*bp
, dmu_tx_t
*tx
)
680 struct space_range_arg
*sra
= arg
;
682 if (bp
->blk_birth
> sra
->mintxg
&& bp
->blk_birth
<= sra
->maxtxg
) {
683 if (dsl_pool_sync_context(spa_get_dsl(sra
->spa
)))
684 sra
->used
+= bp_get_dsize_sync(sra
->spa
, bp
);
686 sra
->used
+= bp_get_dsize(sra
->spa
, bp
);
687 sra
->comp
+= BP_GET_PSIZE(bp
);
688 sra
->uncomp
+= BP_GET_UCSIZE(bp
);
694 bpobj_space(bpobj_t
*bpo
, uint64_t *usedp
, uint64_t *compp
, uint64_t *uncompp
)
696 ASSERT(bpobj_is_open(bpo
));
697 mutex_enter(&bpo
->bpo_lock
);
699 *usedp
= bpo
->bpo_phys
->bpo_bytes
;
700 if (bpo
->bpo_havecomp
) {
701 *compp
= bpo
->bpo_phys
->bpo_comp
;
702 *uncompp
= bpo
->bpo_phys
->bpo_uncomp
;
703 mutex_exit(&bpo
->bpo_lock
);
706 mutex_exit(&bpo
->bpo_lock
);
707 return (bpobj_space_range(bpo
, 0, UINT64_MAX
,
708 usedp
, compp
, uncompp
));
713 * Return the amount of space in the bpobj which is:
714 * mintxg < blk_birth <= maxtxg
717 bpobj_space_range(bpobj_t
*bpo
, uint64_t mintxg
, uint64_t maxtxg
,
718 uint64_t *usedp
, uint64_t *compp
, uint64_t *uncompp
)
720 struct space_range_arg sra
= { 0 };
723 ASSERT(bpobj_is_open(bpo
));
726 * As an optimization, if they want the whole txg range, just
727 * get bpo_bytes rather than iterating over the bps.
729 if (mintxg
< TXG_INITIAL
&& maxtxg
== UINT64_MAX
&& bpo
->bpo_havecomp
)
730 return (bpobj_space(bpo
, usedp
, compp
, uncompp
));
732 sra
.spa
= dmu_objset_spa(bpo
->bpo_os
);
736 err
= bpobj_iterate_nofree(bpo
, space_range_cb
, &sra
, NULL
);
739 *uncompp
= sra
.uncomp
;