]>
Commit | Line | Data |
---|---|---|
34dc7c2f BB |
1 | /* |
2 | * CDDL HEADER START | |
3 | * | |
4 | * The contents of this file are subject to the terms of the | |
5 | * Common Development and Distribution License (the "License"). | |
6 | * You may not use this file except in compliance with the License. | |
7 | * | |
8 | * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE | |
1d3ba0bf | 9 | * or https://opensource.org/licenses/CDDL-1.0. |
34dc7c2f BB |
10 | * See the License for the specific language governing permissions |
11 | * and limitations under the License. | |
12 | * | |
13 | * When distributing Covered Code, include this CDDL HEADER in each | |
14 | * file and include the License file at usr/src/OPENSOLARIS.LICENSE. | |
15 | * If applicable, add the following below this CDDL HEADER, with the | |
16 | * fields enclosed by brackets "[]" replaced with your own identifying | |
17 | * information: Portions Copyright [yyyy] [name of copyright owner] | |
18 | * | |
19 | * CDDL HEADER END | |
20 | */ | |
21 | /* | |
428870ff | 22 | * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. |
dd0b5c85 | 23 | * Copyright (c) 2012, 2021 by Delphix. All rights reserved. |
34dc7c2f BB |
24 | */ |
25 | ||
34dc7c2f | 26 | #include <sys/zfs_context.h> |
27d96d22 | 27 | #include <sys/zfs_refcount.h> |
34dc7c2f | 28 | |
18168da7 | 29 | #ifdef ZFS_DEBUG |
791e480c BB |
30 | /* |
31 | * Reference count tracking is disabled by default. It's memory requirements | |
32 | * are reasonable, however as implemented it consumes a significant amount of | |
33 | * cpu time. Until its performance is improved it should be manually enabled. | |
34 | */ | |
18168da7 | 35 | int reference_tracking_enable = B_FALSE; |
fdc2d303 | 36 | static uint_t reference_history = 3; /* tunable */ |
34dc7c2f BB |
37 | |
38 | static kmem_cache_t *reference_cache; | |
39 | static kmem_cache_t *reference_history_cache; | |
40 | ||
41 | void | |
424fd7c3 | 42 | zfs_refcount_init(void) |
34dc7c2f BB |
43 | { |
44 | reference_cache = kmem_cache_create("reference_cache", | |
45 | sizeof (reference_t), 0, NULL, NULL, NULL, NULL, NULL, 0); | |
46 | ||
47 | reference_history_cache = kmem_cache_create("reference_history_cache", | |
48 | sizeof (uint64_t), 0, NULL, NULL, NULL, NULL, NULL, 0); | |
49 | } | |
50 | ||
51 | void | |
424fd7c3 | 52 | zfs_refcount_fini(void) |
34dc7c2f BB |
53 | { |
54 | kmem_cache_destroy(reference_cache); | |
55 | kmem_cache_destroy(reference_history_cache); | |
56 | } | |
57 | ||
58 | void | |
424fd7c3 | 59 | zfs_refcount_create(zfs_refcount_t *rc) |
34dc7c2f BB |
60 | { |
61 | mutex_init(&rc->rc_mtx, NULL, MUTEX_DEFAULT, NULL); | |
62 | list_create(&rc->rc_list, sizeof (reference_t), | |
63 | offsetof(reference_t, ref_link)); | |
64 | list_create(&rc->rc_removed, sizeof (reference_t), | |
65 | offsetof(reference_t, ref_link)); | |
66 | rc->rc_count = 0; | |
67 | rc->rc_removed_count = 0; | |
13fe0198 MA |
68 | rc->rc_tracked = reference_tracking_enable; |
69 | } | |
70 | ||
3dfb57a3 | 71 | void |
424fd7c3 | 72 | zfs_refcount_create_tracked(zfs_refcount_t *rc) |
3dfb57a3 | 73 | { |
424fd7c3 | 74 | zfs_refcount_create(rc); |
3dfb57a3 DB |
75 | rc->rc_tracked = B_TRUE; |
76 | } | |
77 | ||
13fe0198 | 78 | void |
424fd7c3 | 79 | zfs_refcount_create_untracked(zfs_refcount_t *rc) |
13fe0198 | 80 | { |
424fd7c3 | 81 | zfs_refcount_create(rc); |
13fe0198 | 82 | rc->rc_tracked = B_FALSE; |
34dc7c2f BB |
83 | } |
84 | ||
85 | void | |
424fd7c3 | 86 | zfs_refcount_destroy_many(zfs_refcount_t *rc, uint64_t number) |
34dc7c2f BB |
87 | { |
88 | reference_t *ref; | |
89 | ||
30af21b0 | 90 | ASSERT3U(rc->rc_count, ==, number); |
c65aa5b2 | 91 | while ((ref = list_head(&rc->rc_list))) { |
34dc7c2f BB |
92 | list_remove(&rc->rc_list, ref); |
93 | kmem_cache_free(reference_cache, ref); | |
94 | } | |
95 | list_destroy(&rc->rc_list); | |
96 | ||
c65aa5b2 | 97 | while ((ref = list_head(&rc->rc_removed))) { |
34dc7c2f BB |
98 | list_remove(&rc->rc_removed, ref); |
99 | kmem_cache_free(reference_history_cache, ref->ref_removed); | |
100 | kmem_cache_free(reference_cache, ref); | |
101 | } | |
102 | list_destroy(&rc->rc_removed); | |
103 | mutex_destroy(&rc->rc_mtx); | |
104 | } | |
105 | ||
106 | void | |
424fd7c3 | 107 | zfs_refcount_destroy(zfs_refcount_t *rc) |
34dc7c2f | 108 | { |
424fd7c3 | 109 | zfs_refcount_destroy_many(rc, 0); |
34dc7c2f BB |
110 | } |
111 | ||
112 | int | |
424fd7c3 | 113 | zfs_refcount_is_zero(zfs_refcount_t *rc) |
34dc7c2f | 114 | { |
e829a865 | 115 | return (zfs_refcount_count(rc) == 0); |
34dc7c2f BB |
116 | } |
117 | ||
118 | int64_t | |
424fd7c3 | 119 | zfs_refcount_count(zfs_refcount_t *rc) |
34dc7c2f | 120 | { |
e829a865 | 121 | return (atomic_load_64(&rc->rc_count)); |
34dc7c2f BB |
122 | } |
123 | ||
124 | int64_t | |
dc04a8c7 | 125 | zfs_refcount_add_many(zfs_refcount_t *rc, uint64_t number, const void *holder) |
34dc7c2f | 126 | { |
d4ed6673 | 127 | reference_t *ref = NULL; |
34dc7c2f BB |
128 | int64_t count; |
129 | ||
e829a865 AM |
130 | if (!rc->rc_tracked) { |
131 | count = atomic_add_64_nv(&(rc)->rc_count, number); | |
132 | ASSERT3U(count, >=, number); | |
133 | return (count); | |
34dc7c2f | 134 | } |
e829a865 AM |
135 | |
136 | ref = kmem_cache_alloc(reference_cache, KM_SLEEP); | |
137 | ref->ref_holder = holder; | |
138 | ref->ref_number = number; | |
34dc7c2f | 139 | mutex_enter(&rc->rc_mtx); |
e829a865 | 140 | list_insert_head(&rc->rc_list, ref); |
34dc7c2f BB |
141 | rc->rc_count += number; |
142 | count = rc->rc_count; | |
143 | mutex_exit(&rc->rc_mtx); | |
144 | ||
145 | return (count); | |
146 | } | |
147 | ||
148 | int64_t | |
dc04a8c7 | 149 | zfs_refcount_add(zfs_refcount_t *rc, const void *holder) |
34dc7c2f | 150 | { |
424fd7c3 | 151 | return (zfs_refcount_add_many(rc, 1, holder)); |
34dc7c2f BB |
152 | } |
153 | ||
154 | int64_t | |
dc04a8c7 PD |
155 | zfs_refcount_remove_many(zfs_refcount_t *rc, uint64_t number, |
156 | const void *holder) | |
34dc7c2f BB |
157 | { |
158 | reference_t *ref; | |
159 | int64_t count; | |
160 | ||
13fe0198 | 161 | if (!rc->rc_tracked) { |
e829a865 AM |
162 | count = atomic_add_64_nv(&(rc)->rc_count, -number); |
163 | ASSERT3S(count, >=, 0); | |
34dc7c2f BB |
164 | return (count); |
165 | } | |
166 | ||
e829a865 AM |
167 | mutex_enter(&rc->rc_mtx); |
168 | ASSERT3U(rc->rc_count, >=, number); | |
34dc7c2f BB |
169 | for (ref = list_head(&rc->rc_list); ref; |
170 | ref = list_next(&rc->rc_list, ref)) { | |
171 | if (ref->ref_holder == holder && ref->ref_number == number) { | |
172 | list_remove(&rc->rc_list, ref); | |
173 | if (reference_history > 0) { | |
174 | ref->ref_removed = | |
175 | kmem_cache_alloc(reference_history_cache, | |
79c76d5b | 176 | KM_SLEEP); |
34dc7c2f BB |
177 | list_insert_head(&rc->rc_removed, ref); |
178 | rc->rc_removed_count++; | |
13fe0198 | 179 | if (rc->rc_removed_count > reference_history) { |
34dc7c2f BB |
180 | ref = list_tail(&rc->rc_removed); |
181 | list_remove(&rc->rc_removed, ref); | |
182 | kmem_cache_free(reference_history_cache, | |
183 | ref->ref_removed); | |
184 | kmem_cache_free(reference_cache, ref); | |
185 | rc->rc_removed_count--; | |
186 | } | |
187 | } else { | |
188 | kmem_cache_free(reference_cache, ref); | |
189 | } | |
190 | rc->rc_count -= number; | |
191 | count = rc->rc_count; | |
192 | mutex_exit(&rc->rc_mtx); | |
193 | return (count); | |
194 | } | |
195 | } | |
196 | panic("No such hold %p on refcount %llx", holder, | |
197 | (u_longlong_t)(uintptr_t)rc); | |
198 | return (-1); | |
199 | } | |
200 | ||
201 | int64_t | |
dc04a8c7 | 202 | zfs_refcount_remove(zfs_refcount_t *rc, const void *holder) |
34dc7c2f | 203 | { |
424fd7c3 | 204 | return (zfs_refcount_remove_many(rc, 1, holder)); |
34dc7c2f BB |
205 | } |
206 | ||
572e2857 | 207 | void |
424fd7c3 | 208 | zfs_refcount_transfer(zfs_refcount_t *dst, zfs_refcount_t *src) |
572e2857 BB |
209 | { |
210 | int64_t count, removed_count; | |
211 | list_t list, removed; | |
212 | ||
213 | list_create(&list, sizeof (reference_t), | |
214 | offsetof(reference_t, ref_link)); | |
215 | list_create(&removed, sizeof (reference_t), | |
216 | offsetof(reference_t, ref_link)); | |
217 | ||
218 | mutex_enter(&src->rc_mtx); | |
219 | count = src->rc_count; | |
220 | removed_count = src->rc_removed_count; | |
221 | src->rc_count = 0; | |
222 | src->rc_removed_count = 0; | |
223 | list_move_tail(&list, &src->rc_list); | |
224 | list_move_tail(&removed, &src->rc_removed); | |
225 | mutex_exit(&src->rc_mtx); | |
226 | ||
227 | mutex_enter(&dst->rc_mtx); | |
228 | dst->rc_count += count; | |
229 | dst->rc_removed_count += removed_count; | |
230 | list_move_tail(&dst->rc_list, &list); | |
231 | list_move_tail(&dst->rc_removed, &removed); | |
232 | mutex_exit(&dst->rc_mtx); | |
233 | ||
234 | list_destroy(&list); | |
235 | list_destroy(&removed); | |
236 | } | |
237 | ||
d3c2ae1c | 238 | void |
d7e4b30a | 239 | zfs_refcount_transfer_ownership_many(zfs_refcount_t *rc, uint64_t number, |
dc04a8c7 | 240 | const void *current_holder, const void *new_holder) |
d3c2ae1c GW |
241 | { |
242 | reference_t *ref; | |
243 | boolean_t found = B_FALSE; | |
244 | ||
e829a865 | 245 | if (!rc->rc_tracked) |
d3c2ae1c | 246 | return; |
d3c2ae1c | 247 | |
e829a865 | 248 | mutex_enter(&rc->rc_mtx); |
d3c2ae1c GW |
249 | for (ref = list_head(&rc->rc_list); ref; |
250 | ref = list_next(&rc->rc_list, ref)) { | |
d7e4b30a BB |
251 | if (ref->ref_holder == current_holder && |
252 | ref->ref_number == number) { | |
d3c2ae1c GW |
253 | ref->ref_holder = new_holder; |
254 | found = B_TRUE; | |
255 | break; | |
256 | } | |
257 | } | |
258 | ASSERT(found); | |
259 | mutex_exit(&rc->rc_mtx); | |
260 | } | |
3dfb57a3 | 261 | |
d7e4b30a | 262 | void |
dc04a8c7 PD |
263 | zfs_refcount_transfer_ownership(zfs_refcount_t *rc, const void *current_holder, |
264 | const void *new_holder) | |
d7e4b30a BB |
265 | { |
266 | return (zfs_refcount_transfer_ownership_many(rc, 1, current_holder, | |
267 | new_holder)); | |
268 | } | |
269 | ||
3dfb57a3 DB |
270 | /* |
271 | * If tracking is enabled, return true if a reference exists that matches | |
272 | * the "holder" tag. If tracking is disabled, then return true if a reference | |
273 | * might be held. | |
274 | */ | |
275 | boolean_t | |
dc04a8c7 | 276 | zfs_refcount_held(zfs_refcount_t *rc, const void *holder) |
3dfb57a3 DB |
277 | { |
278 | reference_t *ref; | |
279 | ||
e829a865 AM |
280 | if (!rc->rc_tracked) |
281 | return (zfs_refcount_count(rc) > 0); | |
3dfb57a3 | 282 | |
e829a865 | 283 | mutex_enter(&rc->rc_mtx); |
3dfb57a3 DB |
284 | for (ref = list_head(&rc->rc_list); ref; |
285 | ref = list_next(&rc->rc_list, ref)) { | |
286 | if (ref->ref_holder == holder) { | |
287 | mutex_exit(&rc->rc_mtx); | |
288 | return (B_TRUE); | |
289 | } | |
290 | } | |
291 | mutex_exit(&rc->rc_mtx); | |
292 | return (B_FALSE); | |
293 | } | |
294 | ||
295 | /* | |
296 | * If tracking is enabled, return true if a reference does not exist that | |
297 | * matches the "holder" tag. If tracking is disabled, always return true | |
298 | * since the reference might not be held. | |
299 | */ | |
300 | boolean_t | |
dc04a8c7 | 301 | zfs_refcount_not_held(zfs_refcount_t *rc, const void *holder) |
3dfb57a3 DB |
302 | { |
303 | reference_t *ref; | |
304 | ||
e829a865 | 305 | if (!rc->rc_tracked) |
3dfb57a3 | 306 | return (B_TRUE); |
3dfb57a3 | 307 | |
e829a865 | 308 | mutex_enter(&rc->rc_mtx); |
3dfb57a3 DB |
309 | for (ref = list_head(&rc->rc_list); ref; |
310 | ref = list_next(&rc->rc_list, ref)) { | |
311 | if (ref->ref_holder == holder) { | |
312 | mutex_exit(&rc->rc_mtx); | |
313 | return (B_FALSE); | |
314 | } | |
315 | } | |
316 | mutex_exit(&rc->rc_mtx); | |
317 | return (B_TRUE); | |
318 | } | |
dd0b5c85 | 319 | |
280d0f0c BB |
320 | EXPORT_SYMBOL(zfs_refcount_create); |
321 | EXPORT_SYMBOL(zfs_refcount_destroy); | |
322 | EXPORT_SYMBOL(zfs_refcount_is_zero); | |
323 | EXPORT_SYMBOL(zfs_refcount_count); | |
324 | EXPORT_SYMBOL(zfs_refcount_add); | |
325 | EXPORT_SYMBOL(zfs_refcount_remove); | |
326 | EXPORT_SYMBOL(zfs_refcount_held); | |
327 | ||
dd0b5c85 | 328 | /* BEGIN CSTYLED */ |
18168da7 | 329 | ZFS_MODULE_PARAM(zfs, , reference_tracking_enable, INT, ZMOD_RW, |
dd0b5c85 DB |
330 | "Track reference holders to refcount_t objects"); |
331 | ||
fdc2d303 | 332 | ZFS_MODULE_PARAM(zfs, , reference_history, UINT, ZMOD_RW, |
dd0b5c85 DB |
333 | "Maximum reference holders being tracked"); |
334 | /* END CSTYLED */ | |
572e2857 | 335 | #endif /* ZFS_DEBUG */ |