module/os/freebsd/zfs/arc_os.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 #include <sys/spa.h>
  23 #include <sys/zio.h>
  24 #include <sys/spa_impl.h>
  25 #include <sys/counter.h>
  26 #include <sys/zio_compress.h>
  27 #include <sys/zio_checksum.h>
  28 #include <sys/zfs_context.h>
  29 #include <sys/arc.h>
  30 #include <sys/arc_os.h>
  31 #include <sys/zfs_refcount.h>
  32 #include <sys/vdev.h>
  33 #include <sys/vdev_trim.h>
  34 #include <sys/vdev_impl.h>
  35 #include <sys/dsl_pool.h>
  36 #include <sys/zio_checksum.h>
  37 #include <sys/multilist.h>
  38 #include <sys/abd.h>
  39 #include <sys/zil.h>
  40 #include <sys/fm/fs/zfs.h>
  41 #include <sys/eventhandler.h>
  42 #include <sys/callb.h>
  43 #include <sys/kstat.h>
  44 #include <sys/zthr.h>
  45 #include <zfs_fletcher.h>
  46 #include <sys/arc_impl.h>
  47 #include <sys/sdt.h>
  48 #include <sys/aggsum.h>
  49 #include <sys/vnode.h>
  50 #include <cityhash.h>
  51 #include <machine/vmparam.h>
  52 #include <sys/vm.h>
  53 #include <sys/vmmeter.h>
  54
  55 #if __FreeBSD_version >= 1300139
  56 static struct sx arc_vnlru_lock;
  57 static struct vnode *arc_vnlru_marker;
  58 #endif
  59
  60 extern struct vfsops zfs_vfsops;
  61
  62 uint_t zfs_arc_free_target = 0;
  63
  64 static void
  65 arc_free_target_init(void *unused __unused)
  66 {
  67         zfs_arc_free_target = vm_cnt.v_free_target;
  68 }
  69 SYSINIT(arc_free_target_init, SI_SUB_KTHREAD_PAGE, SI_ORDER_ANY,
  70     arc_free_target_init, NULL);
  71
  72 /*
  73  * We don't have a tunable for arc_free_target due to the dependency on
  74  * pagedaemon initialisation.
  75  */
  76 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, free_target,
  77     param_set_arc_free_target, 0, CTLFLAG_RW,
  78         "Desired number of free pages below which ARC triggers reclaim");
  79 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, no_grow_shift,
  80     param_set_arc_no_grow_shift, 0, ZMOD_RW,
  81         "log2(fraction of ARC which must be free to allow growing)");
  82
  83 int64_t
  84 arc_available_memory(void)
  85 {
  86         int64_t lowest = INT64_MAX;
  87         int64_t n __unused;
  88
  89         /*
  90          * Cooperate with pagedaemon when it's time for it to scan
  91          * and reclaim some pages.
  92          */
  93         n = PAGESIZE * ((int64_t)freemem - zfs_arc_free_target);
  94         if (n < lowest) {
  95                 lowest = n;
  96         }
  97 #if defined(__i386) || !defined(UMA_MD_SMALL_ALLOC)
  98         /*
  99          * If we're on an i386 platform, it's possible that we'll exhaust the
 100          * kernel heap space before we ever run out of available physical
 101          * memory.  Most checks of the size of the heap_area compare against
 102          * tune.t_minarmem, which is the minimum available real memory that we
 103          * can have in the system.  However, this is generally fixed at 25 pages
 104          * which is so low that it's useless.  In this comparison, we seek to
 105          * calculate the total heap-size, and reclaim if more than 3/4ths of the
 106          * heap is allocated.  (Or, in the calculation, if less than 1/4th is
 107          * free)
 108          */
 109         n = uma_avail() - (long)(uma_limit() / 4);
 110         if (n < lowest) {
 111                 lowest = n;
 112         }
 113 #endif
 114
 115         DTRACE_PROBE1(arc__available_memory, int64_t, lowest);
 116         return (lowest);
 117 }
 118
 119 /*
 120  * Return a default max arc size based on the amount of physical memory.
 121  */
 122 uint64_t
 123 arc_default_max(uint64_t min, uint64_t allmem)
 124 {
 125         uint64_t size;
 126
 127         if (allmem >= 1 << 30)
 128                 size = allmem - (1 << 30);
 129         else
 130                 size = min;
 131         return (MAX(allmem * 5 / 8, size));
 132 }
 133
 134 /*
 135  * Helper function for arc_prune_async() it is responsible for safely
 136  * handling the execution of a registered arc_prune_func_t.
 137  */
 138 static void
 139 arc_prune_task(void *arg)
 140 {
 141         uint64_t nr_scan = (uintptr_t)arg;
 142
 143         arc_reduce_target_size(ptob(nr_scan));
 144
 145 #ifndef __ILP32__
 146         if (nr_scan > INT_MAX)
 147                 nr_scan = INT_MAX;
 148 #endif
 149
 150 #if __FreeBSD_version >= 1300139
 151         sx_xlock(&arc_vnlru_lock);
 152         vnlru_free_vfsops(nr_scan, &zfs_vfsops, arc_vnlru_marker);
 153         sx_xunlock(&arc_vnlru_lock);
 154 #else
 155         vnlru_free(nr_scan, &zfs_vfsops);
 156 #endif
 157 }
 158
 159 /*
 160  * Notify registered consumers they must drop holds on a portion of the ARC
 161  * buffered they reference.  This provides a mechanism to ensure the ARC can
 162  * honor the arc_meta_limit and reclaim otherwise pinned ARC buffers.  This
 163  * is analogous to dnlc_reduce_cache() but more generic.
 164  *
 165  * This operation is performed asynchronously so it may be safely called
 166  * in the context of the arc_reclaim_thread().  A reference is taken here
 167  * for each registered arc_prune_t and the arc_prune_task() is responsible
 168  * for releasing it once the registered arc_prune_func_t has completed.
 169  */
 170 void
 171 arc_prune_async(uint64_t adjust)
 172 {
 173
 174 #ifndef __LP64__
 175         if (adjust > UINTPTR_MAX)
 176                 adjust = UINTPTR_MAX;
 177 #endif
 178         taskq_dispatch(arc_prune_taskq, arc_prune_task,
 179             (void *)(intptr_t)adjust, TQ_SLEEP);
 180         ARCSTAT_BUMP(arcstat_prune);
 181 }
 182
 183 uint64_t
 184 arc_all_memory(void)
 185 {
 186         return (ptob(physmem));
 187 }
 188
 189 int
 190 arc_memory_throttle(spa_t *spa, uint64_t reserve, uint64_t txg)
 191 {
 192         return (0);
 193 }
 194
 195 uint64_t
 196 arc_free_memory(void)
 197 {
 198         return (ptob(freemem));
 199 }
 200
 201 static eventhandler_tag arc_event_lowmem = NULL;
 202
 203 static void
 204 arc_lowmem(void *arg __unused, int howto __unused)
 205 {
 206         int64_t free_memory, to_free;
 207
 208         arc_no_grow = B_TRUE;
 209         arc_warm = B_TRUE;
 210         arc_growtime = gethrtime() + SEC2NSEC(arc_grow_retry);
 211         free_memory = arc_available_memory();
 212         int64_t can_free = arc_c - arc_c_min;
 213         if (can_free <= 0)
 214                 return;
 215         to_free = (can_free >> arc_shrink_shift) - MIN(free_memory, 0);
 216         DTRACE_PROBE2(arc__needfree, int64_t, free_memory, int64_t, to_free);
 217         arc_reduce_target_size(to_free);
 218
 219         /*
 220          * It is unsafe to block here in arbitrary threads, because we can come
 221          * here from ARC itself and may hold ARC locks and thus risk a deadlock
 222          * with ARC reclaim thread.
 223          */
 224         if (curproc == pageproc)
 225                 arc_wait_for_eviction(to_free, B_FALSE);
 226 }
 227
 228 void
 229 arc_lowmem_init(void)
 230 {
 231         arc_event_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, arc_lowmem, NULL,
 232             EVENTHANDLER_PRI_FIRST);
 233 #if __FreeBSD_version >= 1300139
 234         arc_vnlru_marker = vnlru_alloc_marker();
 235         sx_init(&arc_vnlru_lock, "arc vnlru lock");
 236 #endif
 237 }
 238
 239 void
 240 arc_lowmem_fini(void)
 241 {
 242         if (arc_event_lowmem != NULL)
 243                 EVENTHANDLER_DEREGISTER(vm_lowmem, arc_event_lowmem);
 244 #if __FreeBSD_version >= 1300139
 245         if (arc_vnlru_marker != NULL) {
 246                 vnlru_free_marker(arc_vnlru_marker);
 247                 sx_destroy(&arc_vnlru_lock);
 248         }
 249 #endif
 250 }
 251
 252 void
 253 arc_register_hotplug(void)
 254 {
 255 }
 256
 257 void
 258 arc_unregister_hotplug(void)
 259 {
 260 }