4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or https://opensource.org/licenses/CDDL-1.0.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2018, Joyent, Inc.
24 * Copyright (c) 2011, 2019 by Delphix. All rights reserved.
25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
26 * Copyright 2017 Nexenta Systems, Inc. All rights reserved.
31 #include <sys/spa_impl.h>
32 #include <sys/zio_compress.h>
33 #include <sys/zio_checksum.h>
34 #include <sys/zfs_context.h>
36 #include <sys/zfs_refcount.h>
38 #include <sys/vdev_trim.h>
39 #include <sys/vdev_impl.h>
40 #include <sys/dsl_pool.h>
41 #include <sys/multilist.h>
44 #include <sys/fm/fs/zfs.h>
46 #include <sys/shrinker.h>
47 #include <sys/vmsystm.h>
49 #include <linux/page_compat.h>
50 #include <linux/notifier.h>
51 #include <linux/memory.h>
53 #include <sys/callb.h>
54 #include <sys/kstat.h>
56 #include <zfs_fletcher.h>
57 #include <sys/arc_impl.h>
58 #include <sys/trace_zfs.h>
59 #include <sys/aggsum.h>
62 * This is a limit on how many pages the ARC shrinker makes available for
63 * eviction in response to one page allocation attempt. Note that in
64 * practice, the kernel's shrinker can ask us to evict up to about 4x this
65 * for one allocation attempt.
67 * The default limit of 10,000 (in practice, 160MB per allocation attempt
68 * with 4K pages) limits the amount of time spent attempting to reclaim ARC
69 * memory to less than 100ms per allocation attempt, even with a small
70 * average compressed block size of ~8KB.
72 * See also the comment in arc_shrinker_count().
73 * Set to 0 to disable limit.
75 int zfs_arc_shrinker_limit
= 10000;
77 #ifdef CONFIG_MEMORY_HOTPLUG
78 static struct notifier_block arc_hotplug_callback_mem_nb
;
82 * Return a default max arc size based on the amount of physical memory.
83 * This may be overridden by tuning the zfs_arc_max module parameter.
86 arc_default_max(uint64_t min
, uint64_t allmem
)
90 if (allmem
>= 1 << 30)
91 size
= allmem
- (1 << 30);
94 return (MAX(allmem
* 5 / 8, size
));
99 * Return maximum amount of memory that we could possibly use. Reduced
100 * to half of all memory in user space which is primarily used for testing.
105 #ifdef CONFIG_HIGHMEM
106 return (ptob(zfs_totalram_pages
- zfs_totalhigh_pages
));
108 return (ptob(zfs_totalram_pages
));
109 #endif /* CONFIG_HIGHMEM */
113 * Return the amount of memory that is considered free. In user space
114 * which is primarily used for testing we pretend that free memory ranges
115 * from 0-20% of all memory.
118 arc_free_memory(void)
120 #ifdef CONFIG_HIGHMEM
123 return (ptob(si
.freeram
- si
.freehigh
));
125 return (ptob(nr_free_pages() +
126 nr_inactive_file_pages()));
127 #endif /* CONFIG_HIGHMEM */
131 * Return the amount of memory that can be consumed before reclaim will be
132 * needed. Positive if there is sufficient free memory, negative indicates
133 * the amount of memory that needs to be freed up.
136 arc_available_memory(void)
138 return (arc_free_memory() - arc_sys_free
);
142 arc_evictable_memory(void)
144 int64_t asize
= aggsum_value(&arc_sums
.arcstat_size
);
146 zfs_refcount_count(&arc_mru
->arcs_esize
[ARC_BUFC_DATA
]) +
147 zfs_refcount_count(&arc_mru
->arcs_esize
[ARC_BUFC_METADATA
]) +
148 zfs_refcount_count(&arc_mfu
->arcs_esize
[ARC_BUFC_DATA
]) +
149 zfs_refcount_count(&arc_mfu
->arcs_esize
[ARC_BUFC_METADATA
]);
150 uint64_t arc_dirty
= MAX((int64_t)asize
- (int64_t)arc_clean
, 0);
153 * Scale reported evictable memory in proportion to page cache, cap
154 * at specified min/max.
156 uint64_t min
= (ptob(nr_file_pages()) / 100) * zfs_arc_pc_percent
;
157 min
= MAX(arc_c_min
, MIN(arc_c_max
, min
));
159 if (arc_dirty
>= min
)
162 return (MAX((int64_t)asize
- (int64_t)min
, 0));
166 * The _count() function returns the number of free-able objects.
167 * The _scan() function returns the number of objects that were freed.
170 arc_shrinker_count(struct shrinker
*shrink
, struct shrink_control
*sc
)
173 * __GFP_FS won't be set if we are called from ZFS code (see
174 * kmem_flags_convert(), which removes it). To avoid a deadlock, we
175 * don't allow evicting in this case. We return 0 rather than
176 * SHRINK_STOP so that the shrinker logic doesn't accumulate a
177 * deficit against us.
179 if (!(sc
->gfp_mask
& __GFP_FS
)) {
184 * This code is reached in the "direct reclaim" case, where the
185 * kernel (outside ZFS) is trying to allocate a page, and the system
188 * The kernel's shrinker code doesn't understand how many pages the
189 * ARC's callback actually frees, so it may ask the ARC to shrink a
190 * lot for one page allocation. This is problematic because it may
191 * take a long time, thus delaying the page allocation, and because
192 * it may force the ARC to unnecessarily shrink very small.
194 * Therefore, we limit the amount of data that we say is evictable,
195 * which limits the amount that the shrinker will ask us to evict for
196 * one page allocation attempt.
198 * In practice, we may be asked to shrink 4x the limit to satisfy one
199 * page allocation, before the kernel's shrinker code gives up on us.
200 * When that happens, we rely on the kernel code to find the pages
201 * that we freed before invoking the OOM killer. This happens in
202 * __alloc_pages_slowpath(), which retries and finds the pages we
203 * freed when it calls get_page_from_freelist().
205 * See also the comment above zfs_arc_shrinker_limit.
207 int64_t limit
= zfs_arc_shrinker_limit
!= 0 ?
208 zfs_arc_shrinker_limit
: INT64_MAX
;
209 return (MIN(limit
, btop((int64_t)arc_evictable_memory())));
213 arc_shrinker_scan(struct shrinker
*shrink
, struct shrink_control
*sc
)
215 ASSERT((sc
->gfp_mask
& __GFP_FS
) != 0);
217 /* The arc is considered warm once reclaim has occurred */
218 if (unlikely(arc_warm
== B_FALSE
))
222 * Evict the requested number of pages by reducing arc_c and waiting
223 * for the requested amount of data to be evicted.
225 arc_reduce_target_size(ptob(sc
->nr_to_scan
));
226 arc_wait_for_eviction(ptob(sc
->nr_to_scan
), B_FALSE
);
227 if (current
->reclaim_state
!= NULL
)
228 #ifdef HAVE_RECLAIM_STATE_RECLAIMED
229 current
->reclaim_state
->reclaimed
+= sc
->nr_to_scan
;
231 current
->reclaim_state
->reclaimed_slab
+= sc
->nr_to_scan
;
235 * We are experiencing memory pressure which the arc_evict_zthr was
236 * unable to keep up with. Set arc_no_grow to briefly pause arc
237 * growth to avoid compounding the memory pressure.
239 arc_no_grow
= B_TRUE
;
242 * When direct reclaim is observed it usually indicates a rapid
243 * increase in memory pressure. This occurs because the kswapd
244 * threads were unable to asynchronously keep enough free memory
247 if (current_is_kswapd()) {
248 ARCSTAT_BUMP(arcstat_memory_indirect_count
);
250 ARCSTAT_BUMP(arcstat_memory_direct_count
);
253 return (sc
->nr_to_scan
);
256 SPL_SHRINKER_DECLARE(arc_shrinker
,
257 arc_shrinker_count
, arc_shrinker_scan
, DEFAULT_SEEKS
);
260 arc_memory_throttle(spa_t
*spa
, uint64_t reserve
, uint64_t txg
)
262 uint64_t free_memory
= arc_free_memory();
264 if (free_memory
> arc_all_memory() * arc_lotsfree_percent
/ 100)
267 if (txg
> spa
->spa_lowmem_last_txg
) {
268 spa
->spa_lowmem_last_txg
= txg
;
269 spa
->spa_lowmem_page_load
= 0;
272 * If we are in pageout, we know that memory is already tight,
273 * the arc is already going to be evicting, so we just want to
274 * continue to let page writes occur as quickly as possible.
276 if (current_is_kswapd()) {
277 if (spa
->spa_lowmem_page_load
>
278 MAX(arc_sys_free
/ 4, free_memory
) / 4) {
279 DMU_TX_STAT_BUMP(dmu_tx_memory_reclaim
);
280 return (SET_ERROR(ERESTART
));
282 /* Note: reserve is inflated, so we deflate */
283 atomic_add_64(&spa
->spa_lowmem_page_load
, reserve
/ 8);
285 } else if (spa
->spa_lowmem_page_load
> 0 && arc_reclaim_needed()) {
286 /* memory is low, delay before restarting */
287 ARCSTAT_INCR(arcstat_memory_throttle_count
, 1);
288 DMU_TX_STAT_BUMP(dmu_tx_memory_reclaim
);
289 return (SET_ERROR(EAGAIN
));
291 spa
->spa_lowmem_page_load
= 0;
296 arc_set_sys_free(uint64_t allmem
)
299 * The ARC tries to keep at least this much memory available for the
300 * system. This gives the ARC time to shrink in response to memory
301 * pressure, before running completely out of memory and invoking the
302 * direct-reclaim ARC shrinker.
304 * This should be more than twice high_wmark_pages(), so that
305 * arc_wait_for_eviction() will wait until at least the
306 * high_wmark_pages() are free (see arc_evict_state_impl()).
308 * Note: Even when the system is very low on memory, the kernel's
309 * shrinker code may only ask for one "batch" of pages (512KB) to be
310 * evicted. If concurrent allocations consume these pages, there may
311 * still be insufficient free pages, and the OOM killer takes action.
313 * By setting arc_sys_free large enough, and having
314 * arc_wait_for_eviction() wait until there is at least arc_sys_free/2
315 * free memory, it is much less likely that concurrent allocations can
316 * consume all the memory that was evicted before checking for
319 * It's hard to iterate the zones from a linux kernel module, which
320 * makes it difficult to determine the watermark dynamically. Instead
321 * we compute the maximum high watermark for this system, based
322 * on the amount of memory, assuming default parameters on Linux kernel
327 * Base wmark_low is 4 * the square root of Kbytes of RAM.
329 long wmark
= 4 * int_sqrt(allmem
/1024) * 1024;
332 * Clamp to between 128K and 64MB.
334 wmark
= MAX(wmark
, 128 * 1024);
335 wmark
= MIN(wmark
, 64 * 1024 * 1024);
338 * watermark_boost can increase the wmark by up to 150%.
340 wmark
+= wmark
* 150 / 100;
343 * arc_sys_free needs to be more than 2x the watermark, because
344 * arc_wait_for_eviction() waits for half of arc_sys_free. Bump this up
345 * to 3x to ensure we're above it.
347 arc_sys_free
= wmark
* 3 + allmem
/ 32;
351 arc_lowmem_init(void)
353 uint64_t allmem
= arc_all_memory();
356 * Register a shrinker to support synchronous (direct) memory
357 * reclaim from the arc. This is done to prevent kswapd from
358 * swapping out pages when it is preferable to shrink the arc.
360 spl_register_shrinker(&arc_shrinker
);
361 arc_set_sys_free(allmem
);
365 arc_lowmem_fini(void)
367 spl_unregister_shrinker(&arc_shrinker
);
371 param_set_arc_u64(const char *buf
, zfs_kernel_param_t
*kp
)
375 error
= spl_param_set_u64(buf
, kp
);
377 return (SET_ERROR(error
));
379 arc_tuning_update(B_TRUE
);
385 param_set_arc_min(const char *buf
, zfs_kernel_param_t
*kp
)
387 return (param_set_arc_u64(buf
, kp
));
391 param_set_arc_max(const char *buf
, zfs_kernel_param_t
*kp
)
393 return (param_set_arc_u64(buf
, kp
));
397 param_set_arc_int(const char *buf
, zfs_kernel_param_t
*kp
)
401 error
= param_set_int(buf
, kp
);
403 return (SET_ERROR(error
));
405 arc_tuning_update(B_TRUE
);
410 #ifdef CONFIG_MEMORY_HOTPLUG
412 arc_hotplug_callback(struct notifier_block
*self
, unsigned long action
,
415 (void) self
, (void) arg
;
416 uint64_t allmem
= arc_all_memory();
417 if (action
!= MEM_ONLINE
)
420 arc_set_limits(allmem
);
423 if (zfs_dirty_data_max_max
== 0)
424 zfs_dirty_data_max_max
= MIN(4ULL * 1024 * 1024 * 1024,
425 allmem
* zfs_dirty_data_max_max_percent
/ 100);
427 if (zfs_dirty_data_max_max
== 0)
428 zfs_dirty_data_max_max
= MIN(1ULL * 1024 * 1024 * 1024,
429 allmem
* zfs_dirty_data_max_max_percent
/ 100);
432 arc_set_sys_free(allmem
);
438 arc_register_hotplug(void)
440 #ifdef CONFIG_MEMORY_HOTPLUG
441 arc_hotplug_callback_mem_nb
.notifier_call
= arc_hotplug_callback
;
442 /* There is no significance to the value 100 */
443 arc_hotplug_callback_mem_nb
.priority
= 100;
444 register_memory_notifier(&arc_hotplug_callback_mem_nb
);
449 arc_unregister_hotplug(void)
451 #ifdef CONFIG_MEMORY_HOTPLUG
452 unregister_memory_notifier(&arc_hotplug_callback_mem_nb
);
457 arc_available_memory(void)
459 int64_t lowest
= INT64_MAX
;
461 /* Every 100 calls, free a small amount */
462 if (random_in_range(100) == 0)
469 arc_memory_throttle(spa_t
*spa
, uint64_t reserve
, uint64_t txg
)
471 (void) spa
, (void) reserve
, (void) txg
;
478 return (ptob(physmem
) / 2);
482 arc_free_memory(void)
484 return (random_in_range(arc_all_memory() * 20 / 100));
488 arc_register_hotplug(void)
493 arc_unregister_hotplug(void)
498 ZFS_MODULE_PARAM(zfs_arc
, zfs_arc_
, shrinker_limit
, INT
, ZMOD_RW
,
499 "Limit on number of pages that ARC shrinker can reclaim at once");