module/zfs/refcount.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2012, 2021 by Delphix. All rights reserved.
  24  */
  25
  26 #include <sys/zfs_context.h>
  27 #include <sys/zfs_refcount.h>
  28
  29 #ifdef  ZFS_DEBUG
  30 /*
  31  * Reference count tracking is disabled by default.  It's memory requirements
  32  * are reasonable, however as implemented it consumes a significant amount of
  33  * cpu time.  Until its performance is improved it should be manually enabled.
  34  */
  35 int reference_tracking_enable = B_FALSE;
  36 static uint_t reference_history = 3; /* tunable */
  37
  38 static kmem_cache_t *reference_cache;
  39 static kmem_cache_t *reference_history_cache;
  40
  41 void
  42 zfs_refcount_init(void)
  43 {
  44         reference_cache = kmem_cache_create("reference_cache",
  45             sizeof (reference_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
  46
  47         reference_history_cache = kmem_cache_create("reference_history_cache",
  48             sizeof (uint64_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
  49 }
  50
  51 void
  52 zfs_refcount_fini(void)
  53 {
  54         kmem_cache_destroy(reference_cache);
  55         kmem_cache_destroy(reference_history_cache);
  56 }
  57
  58 void
  59 zfs_refcount_create(zfs_refcount_t *rc)
  60 {
  61         mutex_init(&rc->rc_mtx, NULL, MUTEX_DEFAULT, NULL);
  62         list_create(&rc->rc_list, sizeof (reference_t),
  63             offsetof(reference_t, ref_link));
  64         list_create(&rc->rc_removed, sizeof (reference_t),
  65             offsetof(reference_t, ref_link));
  66         rc->rc_count = 0;
  67         rc->rc_removed_count = 0;
  68         rc->rc_tracked = reference_tracking_enable;
  69 }
  70
  71 void
  72 zfs_refcount_create_tracked(zfs_refcount_t *rc)
  73 {
  74         zfs_refcount_create(rc);
  75         rc->rc_tracked = B_TRUE;
  76 }
  77
  78 void
  79 zfs_refcount_create_untracked(zfs_refcount_t *rc)
  80 {
  81         zfs_refcount_create(rc);
  82         rc->rc_tracked = B_FALSE;
  83 }
  84
  85 void
  86 zfs_refcount_destroy_many(zfs_refcount_t *rc, uint64_t number)
  87 {
  88         reference_t *ref;
  89
  90         ASSERT3U(rc->rc_count, ==, number);
  91         while ((ref = list_head(&rc->rc_list))) {
  92                 list_remove(&rc->rc_list, ref);
  93                 kmem_cache_free(reference_cache, ref);
  94         }
  95         list_destroy(&rc->rc_list);
  96
  97         while ((ref = list_head(&rc->rc_removed))) {
  98                 list_remove(&rc->rc_removed, ref);
  99                 kmem_cache_free(reference_history_cache, ref->ref_removed);
 100                 kmem_cache_free(reference_cache, ref);
 101         }
 102         list_destroy(&rc->rc_removed);
 103         mutex_destroy(&rc->rc_mtx);
 104 }
 105
 106 void
 107 zfs_refcount_destroy(zfs_refcount_t *rc)
 108 {
 109         zfs_refcount_destroy_many(rc, 0);
 110 }
 111
 112 int
 113 zfs_refcount_is_zero(zfs_refcount_t *rc)
 114 {
 115         return (zfs_refcount_count(rc) == 0);
 116 }
 117
 118 int64_t
 119 zfs_refcount_count(zfs_refcount_t *rc)
 120 {
 121         return (atomic_load_64(&rc->rc_count));
 122 }
 123
 124 int64_t
 125 zfs_refcount_add_many(zfs_refcount_t *rc, uint64_t number, const void *holder)
 126 {
 127         reference_t *ref = NULL;
 128         int64_t count;
 129
 130         if (!rc->rc_tracked) {
 131                 count = atomic_add_64_nv(&(rc)->rc_count, number);
 132                 ASSERT3U(count, >=, number);
 133                 return (count);
 134         }
 135
 136         ref = kmem_cache_alloc(reference_cache, KM_SLEEP);
 137         ref->ref_holder = holder;
 138         ref->ref_number = number;
 139         mutex_enter(&rc->rc_mtx);
 140         list_insert_head(&rc->rc_list, ref);
 141         rc->rc_count += number;
 142         count = rc->rc_count;
 143         mutex_exit(&rc->rc_mtx);
 144
 145         return (count);
 146 }
 147
 148 int64_t
 149 zfs_refcount_add(zfs_refcount_t *rc, const void *holder)
 150 {
 151         return (zfs_refcount_add_many(rc, 1, holder));
 152 }
 153
 154 void
 155 zfs_refcount_add_few(zfs_refcount_t *rc, uint64_t number, const void *holder)
 156 {
 157         if (!rc->rc_tracked)
 158                 (void) zfs_refcount_add_many(rc, number, holder);
 159         else for (; number > 0; number--)
 160                 (void) zfs_refcount_add(rc, holder);
 161 }
 162
 163 int64_t
 164 zfs_refcount_remove_many(zfs_refcount_t *rc, uint64_t number,
 165     const void *holder)
 166 {
 167         reference_t *ref;
 168         int64_t count;
 169
 170         if (!rc->rc_tracked) {
 171                 count = atomic_add_64_nv(&(rc)->rc_count, -number);
 172                 ASSERT3S(count, >=, 0);
 173                 return (count);
 174         }
 175
 176         mutex_enter(&rc->rc_mtx);
 177         ASSERT3U(rc->rc_count, >=, number);
 178         for (ref = list_head(&rc->rc_list); ref;
 179             ref = list_next(&rc->rc_list, ref)) {
 180                 if (ref->ref_holder == holder && ref->ref_number == number) {
 181                         list_remove(&rc->rc_list, ref);
 182                         if (reference_history > 0) {
 183                                 ref->ref_removed =
 184                                     kmem_cache_alloc(reference_history_cache,
 185                                     KM_SLEEP);
 186                                 list_insert_head(&rc->rc_removed, ref);
 187                                 rc->rc_removed_count++;
 188                                 if (rc->rc_removed_count > reference_history) {
 189                                         ref = list_tail(&rc->rc_removed);
 190                                         list_remove(&rc->rc_removed, ref);
 191                                         kmem_cache_free(reference_history_cache,
 192                                             ref->ref_removed);
 193                                         kmem_cache_free(reference_cache, ref);
 194                                         rc->rc_removed_count--;
 195                                 }
 196                         } else {
 197                                 kmem_cache_free(reference_cache, ref);
 198                         }
 199                         rc->rc_count -= number;
 200                         count = rc->rc_count;
 201                         mutex_exit(&rc->rc_mtx);
 202                         return (count);
 203                 }
 204         }
 205         panic("No such hold %p on refcount %llx", holder,
 206             (u_longlong_t)(uintptr_t)rc);
 207         return (-1);
 208 }
 209
 210 int64_t
 211 zfs_refcount_remove(zfs_refcount_t *rc, const void *holder)
 212 {
 213         return (zfs_refcount_remove_many(rc, 1, holder));
 214 }
 215
 216 void
 217 zfs_refcount_remove_few(zfs_refcount_t *rc, uint64_t number, const void *holder)
 218 {
 219         if (!rc->rc_tracked)
 220                 (void) zfs_refcount_remove_many(rc, number, holder);
 221         else for (; number > 0; number--)
 222                 (void) zfs_refcount_remove(rc, holder);
 223 }
 224
 225 void
 226 zfs_refcount_transfer(zfs_refcount_t *dst, zfs_refcount_t *src)
 227 {
 228         int64_t count, removed_count;
 229         list_t list, removed;
 230
 231         list_create(&list, sizeof (reference_t),
 232             offsetof(reference_t, ref_link));
 233         list_create(&removed, sizeof (reference_t),
 234             offsetof(reference_t, ref_link));
 235
 236         mutex_enter(&src->rc_mtx);
 237         count = src->rc_count;
 238         removed_count = src->rc_removed_count;
 239         src->rc_count = 0;
 240         src->rc_removed_count = 0;
 241         list_move_tail(&list, &src->rc_list);
 242         list_move_tail(&removed, &src->rc_removed);
 243         mutex_exit(&src->rc_mtx);
 244
 245         mutex_enter(&dst->rc_mtx);
 246         dst->rc_count += count;
 247         dst->rc_removed_count += removed_count;
 248         list_move_tail(&dst->rc_list, &list);
 249         list_move_tail(&dst->rc_removed, &removed);
 250         mutex_exit(&dst->rc_mtx);
 251
 252         list_destroy(&list);
 253         list_destroy(&removed);
 254 }
 255
 256 void
 257 zfs_refcount_transfer_ownership_many(zfs_refcount_t *rc, uint64_t number,
 258     const void *current_holder, const void *new_holder)
 259 {
 260         reference_t *ref;
 261         boolean_t found = B_FALSE;
 262
 263         if (!rc->rc_tracked)
 264                 return;
 265
 266         mutex_enter(&rc->rc_mtx);
 267         for (ref = list_head(&rc->rc_list); ref;
 268             ref = list_next(&rc->rc_list, ref)) {
 269                 if (ref->ref_holder == current_holder &&
 270                     ref->ref_number == number) {
 271                         ref->ref_holder = new_holder;
 272                         found = B_TRUE;
 273                         break;
 274                 }
 275         }
 276         ASSERT(found);
 277         mutex_exit(&rc->rc_mtx);
 278 }
 279
 280 void
 281 zfs_refcount_transfer_ownership(zfs_refcount_t *rc, const void *current_holder,
 282     const void *new_holder)
 283 {
 284         return (zfs_refcount_transfer_ownership_many(rc, 1, current_holder,
 285             new_holder));
 286 }
 287
 288 /*
 289  * If tracking is enabled, return true if a reference exists that matches
 290  * the "holder" tag. If tracking is disabled, then return true if a reference
 291  * might be held.
 292  */
 293 boolean_t
 294 zfs_refcount_held(zfs_refcount_t *rc, const void *holder)
 295 {
 296         reference_t *ref;
 297
 298         if (!rc->rc_tracked)
 299                 return (zfs_refcount_count(rc) > 0);
 300
 301         mutex_enter(&rc->rc_mtx);
 302         for (ref = list_head(&rc->rc_list); ref;
 303             ref = list_next(&rc->rc_list, ref)) {
 304                 if (ref->ref_holder == holder) {
 305                         mutex_exit(&rc->rc_mtx);
 306                         return (B_TRUE);
 307                 }
 308         }
 309         mutex_exit(&rc->rc_mtx);
 310         return (B_FALSE);
 311 }
 312
 313 /*
 314  * If tracking is enabled, return true if a reference does not exist that
 315  * matches the "holder" tag. If tracking is disabled, always return true
 316  * since the reference might not be held.
 317  */
 318 boolean_t
 319 zfs_refcount_not_held(zfs_refcount_t *rc, const void *holder)
 320 {
 321         reference_t *ref;
 322
 323         if (!rc->rc_tracked)
 324                 return (B_TRUE);
 325
 326         mutex_enter(&rc->rc_mtx);
 327         for (ref = list_head(&rc->rc_list); ref;
 328             ref = list_next(&rc->rc_list, ref)) {
 329                 if (ref->ref_holder == holder) {
 330                         mutex_exit(&rc->rc_mtx);
 331                         return (B_FALSE);
 332                 }
 333         }
 334         mutex_exit(&rc->rc_mtx);
 335         return (B_TRUE);
 336 }
 337
 338 EXPORT_SYMBOL(zfs_refcount_create);
 339 EXPORT_SYMBOL(zfs_refcount_destroy);
 340 EXPORT_SYMBOL(zfs_refcount_is_zero);
 341 EXPORT_SYMBOL(zfs_refcount_count);
 342 EXPORT_SYMBOL(zfs_refcount_add);
 343 EXPORT_SYMBOL(zfs_refcount_remove);
 344 EXPORT_SYMBOL(zfs_refcount_held);
 345
 346 /* BEGIN CSTYLED */
 347 ZFS_MODULE_PARAM(zfs, , reference_tracking_enable, INT, ZMOD_RW,
 348         "Track reference holders to refcount_t objects");
 349
 350 ZFS_MODULE_PARAM(zfs, , reference_history, UINT, ZMOD_RW,
 351         "Maximum reference holders being tracked");
 352 /* END CSTYLED */
 353 #endif  /* ZFS_DEBUG */