4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2013, 2017 by Delphix. All rights reserved.
24 * Copyright 2014 HybridCluster. All rights reserved.
28 #include <sys/dmu_objset.h>
29 #include <sys/dmu_tx.h>
30 #include <sys/dnode.h>
32 #include <sys/zfeature.h>
33 #include <sys/dsl_dataset.h>
36 * Each of the concurrent object allocators will grab
37 * 2^dmu_object_alloc_chunk_shift dnode slots at a time. The default is to
38 * grab 128 slots, which is 4 blocks worth. This was experimentally
39 * determined to be the lowest value that eliminates the measurable effect
40 * of lock contention from this code path.
42 int dmu_object_alloc_chunk_shift
= 7;
45 dmu_object_alloc_impl(objset_t
*os
, dmu_object_type_t ot
, int blocksize
,
46 int indirect_blockshift
, dmu_object_type_t bonustype
, int bonuslen
,
47 int dnodesize
, dnode_t
**allocated_dnode
, void *tag
, dmu_tx_t
*tx
)
50 uint64_t L1_dnode_count
= DNODES_PER_BLOCK
<<
51 (DMU_META_DNODE(os
)->dn_indblkshift
- SPA_BLKPTRSHIFT
);
53 int dn_slots
= dnodesize
>> DNODE_SHIFT
;
54 boolean_t restarted
= B_FALSE
;
55 uint64_t *cpuobj
= NULL
;
56 int dnodes_per_chunk
= 1 << dmu_object_alloc_chunk_shift
;
60 cpuobj
= &os
->os_obj_next_percpu
[CPU_SEQID
%
61 os
->os_obj_next_percpu_len
];
65 dn_slots
= DNODE_MIN_SLOTS
;
67 ASSERT3S(dn_slots
, >=, DNODE_MIN_SLOTS
);
68 ASSERT3S(dn_slots
, <=, DNODE_MAX_SLOTS
);
72 * The "chunk" of dnodes that is assigned to a CPU-specific
73 * allocator needs to be at least one block's worth, to avoid
74 * lock contention on the dbuf. It can be at most one L1 block's
75 * worth, so that the "rescan after polishing off a L1's worth"
76 * logic below will be sure to kick in.
78 if (dnodes_per_chunk
< DNODES_PER_BLOCK
)
79 dnodes_per_chunk
= DNODES_PER_BLOCK
;
80 if (dnodes_per_chunk
> L1_dnode_count
)
81 dnodes_per_chunk
= L1_dnode_count
;
84 * The caller requested the dnode be returned as a performance
85 * optimization in order to avoid releasing the hold only to
86 * immediately reacquire it. Since they caller is responsible
87 * for releasing the hold they must provide the tag.
89 if (allocated_dnode
!= NULL
) {
90 ASSERT3P(tag
, !=, NULL
);
92 ASSERT3P(tag
, ==, NULL
);
99 * If we finished a chunk of dnodes, get a new one from
100 * the global allocator.
102 if ((P2PHASE(object
, dnodes_per_chunk
) == 0) ||
103 (P2PHASE(object
+ dn_slots
- 1, dnodes_per_chunk
) <
105 DNODE_STAT_BUMP(dnode_alloc_next_chunk
);
106 mutex_enter(&os
->os_obj_lock
);
107 ASSERT0(P2PHASE(os
->os_obj_next_chunk
,
109 object
= os
->os_obj_next_chunk
;
112 * Each time we polish off a L1 bp worth of dnodes
113 * (2^12 objects), move to another L1 bp that's
114 * still reasonably sparse (at most 1/4 full). Look
115 * from the beginning at most once per txg. If we
116 * still can't allocate from that L1 block, search
117 * for an empty L0 block, which will quickly skip
118 * to the end of the metadnode if no nearby L0
119 * blocks are empty. This fallback avoids a
120 * pathology where full dnode blocks containing
121 * large dnodes appear sparse because they have a
122 * low blk_fill, leading to many failed allocation
123 * attempts. In the long term a better mechanism to
124 * search for sparse metadnode regions, such as
125 * spacemaps, could be implemented.
127 * os_scan_dnodes is set during txg sync if enough
128 * objects have been freed since the previous
129 * rescan to justify backfilling again.
131 * Note that dmu_traverse depends on the behavior
132 * that we use multiple blocks of the dnode object
133 * before going back to reuse objects. Any change
134 * to this algorithm should preserve that property
135 * or find another solution to the issues described
136 * in traverse_visitbp.
138 if (P2PHASE(object
, L1_dnode_count
) == 0) {
142 if (os
->os_rescan_dnodes
) {
144 os
->os_rescan_dnodes
= B_FALSE
;
146 offset
= object
<< DNODE_SHIFT
;
148 blkfill
= restarted
? 1 : DNODES_PER_BLOCK
>> 2;
149 minlvl
= restarted
? 1 : 2;
151 error
= dnode_next_offset(DMU_META_DNODE(os
),
152 DNODE_FIND_HOLE
, &offset
, minlvl
,
155 object
= offset
>> DNODE_SHIFT
;
159 * Note: if "restarted", we may find a L0 that
160 * is not suitably aligned.
162 os
->os_obj_next_chunk
=
163 P2ALIGN(object
, dnodes_per_chunk
) +
165 (void) atomic_swap_64(cpuobj
, object
);
166 mutex_exit(&os
->os_obj_lock
);
170 * The value of (*cpuobj) before adding dn_slots is the object
171 * ID assigned to us. The value afterwards is the object ID
172 * assigned to whoever wants to do an allocation next.
174 object
= atomic_add_64_nv(cpuobj
, dn_slots
) - dn_slots
;
177 * XXX We should check for an i/o error here and return
178 * up to our caller. Actually we should pre-read it in
179 * dmu_tx_assign(), but there is currently no mechanism
182 error
= dnode_hold_impl(os
, object
, DNODE_MUST_BE_FREE
,
185 rw_enter(&dn
->dn_struct_rwlock
, RW_WRITER
);
187 * Another thread could have allocated it; check
188 * again now that we have the struct lock.
190 if (dn
->dn_type
== DMU_OT_NONE
) {
191 dnode_allocate(dn
, ot
, blocksize
,
192 indirect_blockshift
, bonustype
,
193 bonuslen
, dn_slots
, tx
);
194 rw_exit(&dn
->dn_struct_rwlock
);
195 dmu_tx_add_new_object(tx
, dn
);
198 * Caller requested the allocated dnode be
199 * returned and is responsible for the hold.
201 if (allocated_dnode
!= NULL
)
202 *allocated_dnode
= dn
;
208 rw_exit(&dn
->dn_struct_rwlock
);
210 DNODE_STAT_BUMP(dnode_alloc_race
);
214 * Skip to next known valid starting point on error. This
215 * is the start of the next block of dnodes.
217 if (dmu_object_next(os
, &object
, B_TRUE
, 0) != 0) {
218 object
= P2ROUNDUP(object
+ 1, DNODES_PER_BLOCK
);
219 DNODE_STAT_BUMP(dnode_alloc_next_block
);
221 (void) atomic_swap_64(cpuobj
, object
);
226 dmu_object_alloc(objset_t
*os
, dmu_object_type_t ot
, int blocksize
,
227 dmu_object_type_t bonustype
, int bonuslen
, dmu_tx_t
*tx
)
229 return dmu_object_alloc_impl(os
, ot
, blocksize
, 0, bonustype
,
230 bonuslen
, 0, NULL
, NULL
, tx
);
234 dmu_object_alloc_ibs(objset_t
*os
, dmu_object_type_t ot
, int blocksize
,
235 int indirect_blockshift
, dmu_object_type_t bonustype
, int bonuslen
,
238 return dmu_object_alloc_impl(os
, ot
, blocksize
, indirect_blockshift
,
239 bonustype
, bonuslen
, 0, NULL
, NULL
, tx
);
243 dmu_object_alloc_dnsize(objset_t
*os
, dmu_object_type_t ot
, int blocksize
,
244 dmu_object_type_t bonustype
, int bonuslen
, int dnodesize
, dmu_tx_t
*tx
)
246 return (dmu_object_alloc_impl(os
, ot
, blocksize
, 0, bonustype
,
247 bonuslen
, dnodesize
, NULL
, NULL
, tx
));
251 * Allocate a new object and return a pointer to the newly allocated dnode
252 * via the allocated_dnode argument. The returned dnode will be held and
253 * the caller is responsible for releasing the hold by calling dnode_rele().
256 dmu_object_alloc_hold(objset_t
*os
, dmu_object_type_t ot
, int blocksize
,
257 int indirect_blockshift
, dmu_object_type_t bonustype
, int bonuslen
,
258 int dnodesize
, dnode_t
**allocated_dnode
, void *tag
, dmu_tx_t
*tx
)
260 return (dmu_object_alloc_impl(os
, ot
, blocksize
, indirect_blockshift
,
261 bonustype
, bonuslen
, dnodesize
, allocated_dnode
, tag
, tx
));
265 dmu_object_claim(objset_t
*os
, uint64_t object
, dmu_object_type_t ot
,
266 int blocksize
, dmu_object_type_t bonustype
, int bonuslen
, dmu_tx_t
*tx
)
268 return (dmu_object_claim_dnsize(os
, object
, ot
, blocksize
, bonustype
,
273 dmu_object_claim_dnsize(objset_t
*os
, uint64_t object
, dmu_object_type_t ot
,
274 int blocksize
, dmu_object_type_t bonustype
, int bonuslen
,
275 int dnodesize
, dmu_tx_t
*tx
)
278 int dn_slots
= dnodesize
>> DNODE_SHIFT
;
282 dn_slots
= DNODE_MIN_SLOTS
;
283 ASSERT3S(dn_slots
, >=, DNODE_MIN_SLOTS
);
284 ASSERT3S(dn_slots
, <=, DNODE_MAX_SLOTS
);
286 if (object
== DMU_META_DNODE_OBJECT
&& !dmu_tx_private_ok(tx
))
287 return (SET_ERROR(EBADF
));
289 err
= dnode_hold_impl(os
, object
, DNODE_MUST_BE_FREE
, dn_slots
,
294 dnode_allocate(dn
, ot
, blocksize
, 0, bonustype
, bonuslen
, dn_slots
, tx
);
295 dmu_tx_add_new_object(tx
, dn
);
297 dnode_rele(dn
, FTAG
);
303 dmu_object_reclaim(objset_t
*os
, uint64_t object
, dmu_object_type_t ot
,
304 int blocksize
, dmu_object_type_t bonustype
, int bonuslen
, dmu_tx_t
*tx
)
306 return (dmu_object_reclaim_dnsize(os
, object
, ot
, blocksize
, bonustype
,
307 bonuslen
, DNODE_MIN_SIZE
, tx
));
311 dmu_object_reclaim_dnsize(objset_t
*os
, uint64_t object
, dmu_object_type_t ot
,
312 int blocksize
, dmu_object_type_t bonustype
, int bonuslen
, int dnodesize
,
316 int dn_slots
= dnodesize
>> DNODE_SHIFT
;
320 dn_slots
= DNODE_MIN_SLOTS
;
322 if (object
== DMU_META_DNODE_OBJECT
)
323 return (SET_ERROR(EBADF
));
325 err
= dnode_hold_impl(os
, object
, DNODE_MUST_BE_ALLOCATED
, 0,
330 dnode_reallocate(dn
, ot
, blocksize
, bonustype
, bonuslen
, dn_slots
, tx
);
332 dnode_rele(dn
, FTAG
);
337 dmu_object_free(objset_t
*os
, uint64_t object
, dmu_tx_t
*tx
)
342 ASSERT(object
!= DMU_META_DNODE_OBJECT
|| dmu_tx_private_ok(tx
));
344 err
= dnode_hold_impl(os
, object
, DNODE_MUST_BE_ALLOCATED
, 0,
349 ASSERT(dn
->dn_type
!= DMU_OT_NONE
);
351 * If we don't create this free range, we'll leak indirect blocks when
352 * we get to freeing the dnode in syncing context.
354 dnode_free_range(dn
, 0, DMU_OBJECT_END
, tx
);
356 dnode_rele(dn
, FTAG
);
362 * Return (in *objectp) the next object which is allocated (or a hole)
363 * after *object, taking into account only objects that may have been modified
364 * after the specified txg.
367 dmu_object_next(objset_t
*os
, uint64_t *objectp
, boolean_t hole
, uint64_t txg
)
371 struct dsl_dataset
*ds
= os
->os_dsl_dataset
;
376 } else if (ds
&& dsl_dataset_feature_is_active(ds
,
377 SPA_FEATURE_LARGE_DNODE
)) {
378 uint64_t i
= *objectp
+ 1;
379 uint64_t last_obj
= *objectp
| (DNODES_PER_BLOCK
- 1);
380 dmu_object_info_t doi
;
383 * Scan through the remaining meta dnode block. The contents
384 * of each slot in the block are known so it can be quickly
385 * checked. If the block is exhausted without a match then
386 * hand off to dnode_next_offset() for further scanning.
388 while (i
<= last_obj
) {
389 error
= dmu_object_info(os
, i
, &doi
);
390 if (error
== ENOENT
) {
397 } else if (error
== EEXIST
) {
399 } else if (error
== 0) {
401 i
+= doi
.doi_dnodesize
>> DNODE_SHIFT
;
413 start_obj
= *objectp
+ 1;
416 offset
= start_obj
<< DNODE_SHIFT
;
418 error
= dnode_next_offset(DMU_META_DNODE(os
),
419 (hole
? DNODE_FIND_HOLE
: 0), &offset
, 0, DNODES_PER_BLOCK
, txg
);
421 *objectp
= offset
>> DNODE_SHIFT
;
427 * Turn this object from old_type into DMU_OTN_ZAP_METADATA, and bump the
428 * refcount on SPA_FEATURE_EXTENSIBLE_DATASET.
430 * Only for use from syncing context, on MOS objects.
433 dmu_object_zapify(objset_t
*mos
, uint64_t object
, dmu_object_type_t old_type
,
438 ASSERT(dmu_tx_is_syncing(tx
));
440 VERIFY0(dnode_hold(mos
, object
, FTAG
, &dn
));
441 if (dn
->dn_type
== DMU_OTN_ZAP_METADATA
) {
442 dnode_rele(dn
, FTAG
);
445 ASSERT3U(dn
->dn_type
, ==, old_type
);
446 ASSERT0(dn
->dn_maxblkid
);
449 * We must initialize the ZAP data before changing the type,
450 * so that concurrent calls to *_is_zapified() can determine if
451 * the object has been completely zapified by checking the type.
453 mzap_create_impl(dn
, 0, 0, tx
);
455 dn
->dn_next_type
[tx
->tx_txg
& TXG_MASK
] = dn
->dn_type
=
456 DMU_OTN_ZAP_METADATA
;
457 dnode_setdirty(dn
, tx
);
458 dnode_rele(dn
, FTAG
);
460 spa_feature_incr(dmu_objset_spa(mos
),
461 SPA_FEATURE_EXTENSIBLE_DATASET
, tx
);
465 dmu_object_free_zapified(objset_t
*mos
, uint64_t object
, dmu_tx_t
*tx
)
470 ASSERT(dmu_tx_is_syncing(tx
));
472 VERIFY0(dnode_hold(mos
, object
, FTAG
, &dn
));
474 dnode_rele(dn
, FTAG
);
476 if (t
== DMU_OTN_ZAP_METADATA
) {
477 spa_feature_decr(dmu_objset_spa(mos
),
478 SPA_FEATURE_EXTENSIBLE_DATASET
, tx
);
480 VERIFY0(dmu_object_free(mos
, object
, tx
));
484 EXPORT_SYMBOL(dmu_object_alloc
);
485 EXPORT_SYMBOL(dmu_object_alloc_ibs
);
486 EXPORT_SYMBOL(dmu_object_alloc_dnsize
);
487 EXPORT_SYMBOL(dmu_object_alloc_hold
);
488 EXPORT_SYMBOL(dmu_object_claim
);
489 EXPORT_SYMBOL(dmu_object_claim_dnsize
);
490 EXPORT_SYMBOL(dmu_object_reclaim
);
491 EXPORT_SYMBOL(dmu_object_reclaim_dnsize
);
492 EXPORT_SYMBOL(dmu_object_free
);
493 EXPORT_SYMBOL(dmu_object_next
);
494 EXPORT_SYMBOL(dmu_object_zapify
);
495 EXPORT_SYMBOL(dmu_object_free_zapified
);
498 module_param(dmu_object_alloc_chunk_shift
, int, 0644);
499 MODULE_PARM_DESC(dmu_object_alloc_chunk_shift
,
500 "CPU-specific allocator grabs 2^N objects at once");