#define KMC_REAP_CHUNK INT_MAX
#define KMC_DEFAULT_SEEKS 1
-#define KMC_EXPIRE_AGE 0x1 /* Due to age */
-#define KMC_EXPIRE_MEM 0x2 /* Due to low memory */
-
#define KMC_RECLAIM_ONCE 0x1 /* Force a single shrinker pass */
-extern unsigned int spl_kmem_cache_expire;
extern struct list_head spl_kmem_cache_list;
extern struct rw_semaphore spl_kmem_cache_sem;
#define SKS_MAGIC 0x22222222
#define SKC_MAGIC 0x2c2c2c2c
-#define SPL_KMEM_CACHE_DELAY 15 /* Minimum slab release age */
#define SPL_KMEM_CACHE_OBJ_PER_SLAB 8 /* Target objects per slab */
-#define SPL_KMEM_CACHE_OBJ_PER_SLAB_MIN 1 /* Minimum objects per slab */
#define SPL_KMEM_CACHE_ALIGN 8 /* Default object alignment */
#ifdef _LP64
#define SPL_KMEM_CACHE_MAX_SIZE 32 /* Max slab size in MB */
uint32_t skm_size; /* Magazine size */
uint32_t skm_refill; /* Batch refill size */
struct spl_kmem_cache *skm_cache; /* Owned by cache */
- unsigned long skm_age; /* Last cache access */
unsigned int skm_cpu; /* Owned by cpu */
void *skm_objs[0]; /* Object pointers */
} spl_kmem_magazine_t;
uint32_t skc_obj_align; /* Object alignment */
uint32_t skc_slab_objs; /* Objects per slab */
uint32_t skc_slab_size; /* Slab size */
- uint32_t skc_delay; /* Slab reclaim interval */
atomic_t skc_ref; /* Ref count callers */
taskqid_t skc_taskqid; /* Slab reclaim task */
struct list_head skc_list; /* List of caches linkage */
#define smp_mb__after_atomic(x) smp_mb__after_clear_bit(x)
#endif
-/*
- * Cache expiration was implemented because it was part of the default Solaris
- * kmem_cache behavior. The idea is that per-cpu objects which haven't been
- * accessed in several seconds should be returned to the cache. On the other
- * hand Linux slabs never move objects back to the slabs unless there is
- * memory pressure on the system. By default the Linux method is enabled
- * because it has been shown to improve responsiveness on low memory systems.
- * This policy may be changed by setting KMC_EXPIRE_AGE or KMC_EXPIRE_MEM.
- */
/* BEGIN CSTYLED */
-unsigned int spl_kmem_cache_expire = KMC_EXPIRE_MEM;
-EXPORT_SYMBOL(spl_kmem_cache_expire);
-module_param(spl_kmem_cache_expire, uint, 0644);
-MODULE_PARM_DESC(spl_kmem_cache_expire, "By age (0x1) or low memory (0x2)");
/*
* Cache magazines are an optimization designed to minimize the cost of
module_param(spl_kmem_cache_obj_per_slab, uint, 0644);
MODULE_PARM_DESC(spl_kmem_cache_obj_per_slab, "Number of objects per slab");
-unsigned int spl_kmem_cache_obj_per_slab_min = SPL_KMEM_CACHE_OBJ_PER_SLAB_MIN;
-module_param(spl_kmem_cache_obj_per_slab_min, uint, 0644);
-MODULE_PARM_DESC(spl_kmem_cache_obj_per_slab_min,
- "Minimal number of objects per slab");
-
unsigned int spl_kmem_cache_max_size = SPL_KMEM_CACHE_MAX_SIZE;
module_param(spl_kmem_cache_max_size, uint, 0644);
MODULE_PARM_DESC(spl_kmem_cache_max_size, "Maximum size of slab in MB");
* argument contains the max number of entries to remove from the magazine.
*/
static void
-__spl_cache_flush(spl_kmem_cache_t *skc, spl_kmem_magazine_t *skm, int flush)
+spl_cache_flush(spl_kmem_cache_t *skc, spl_kmem_magazine_t *skm, int flush)
{
- int i, count = MIN(flush, skm->skm_avail);
+ spin_lock(&skc->skc_lock);
ASSERT(skc->skc_magic == SKC_MAGIC);
ASSERT(skm->skm_magic == SKM_MAGIC);
- for (i = 0; i < count; i++)
+ int count = MIN(flush, skm->skm_avail);
+ for (int i = 0; i < count; i++)
spl_cache_shrink(skc, skm->skm_objs[i]);
skm->skm_avail -= count;
memmove(skm->skm_objs, &(skm->skm_objs[count]),
sizeof (void *) * skm->skm_avail);
-}
-
-static void
-spl_cache_flush(spl_kmem_cache_t *skc, spl_kmem_magazine_t *skm, int flush)
-{
- spin_lock(&skc->skc_lock);
- __spl_cache_flush(skc, skm, flush);
- spin_unlock(&skc->skc_lock);
-}
-
-static void
-spl_magazine_age(void *data)
-{
- spl_kmem_cache_t *skc = (spl_kmem_cache_t *)data;
- spl_kmem_magazine_t *skm = skc->skc_mag[smp_processor_id()];
-
- ASSERT(skm->skm_magic == SKM_MAGIC);
- ASSERT(skm->skm_cpu == smp_processor_id());
- ASSERT(irqs_disabled());
-
- /* There are no available objects or they are too young to age out */
- if ((skm->skm_avail == 0) ||
- time_before(jiffies, skm->skm_age + skc->skc_delay * HZ))
- return;
-
- /*
- * Because we're executing in interrupt context we may have
- * interrupted the holder of this lock. To avoid a potential
- * deadlock return if the lock is contended.
- */
- if (!spin_trylock(&skc->skc_lock))
- return;
-
- __spl_cache_flush(skc, skm, skm->skm_refill);
- spin_unlock(&skc->skc_lock);
-}
-/*
- * Called regularly to keep a downward pressure on the cache.
- *
- * Objects older than skc->skc_delay seconds in the per-cpu magazines will
- * be returned to the caches. This is done to prevent idle magazines from
- * holding memory which could be better used elsewhere. The delay is
- * present to prevent thrashing the magazine.
- *
- * The newly released objects may result in empty partial slabs. Those
- * slabs should be released to the system. Otherwise moving the objects
- * out of the magazines is just wasted work.
- */
-static void
-spl_cache_age(void *data)
-{
- spl_kmem_cache_t *skc = (spl_kmem_cache_t *)data;
- taskqid_t id = 0;
-
- ASSERT(skc->skc_magic == SKC_MAGIC);
-
- /* Dynamically disabled at run time */
- if (!(spl_kmem_cache_expire & KMC_EXPIRE_AGE))
- return;
-
- atomic_inc(&skc->skc_ref);
-
- if (!(skc->skc_flags & KMC_NOMAGAZINE))
- on_each_cpu(spl_magazine_age, skc, 1);
-
- spl_slab_reclaim(skc);
-
- while (!test_bit(KMC_BIT_DESTROY, &skc->skc_flags) && !id) {
- id = taskq_dispatch_delay(
- spl_kmem_cache_taskq, spl_cache_age, skc, TQ_SLEEP,
- ddi_get_lbolt() + skc->skc_delay / 3 * HZ);
-
- /* Destroy issued after dispatch immediately cancel it */
- if (test_bit(KMC_BIT_DESTROY, &skc->skc_flags) && id)
- taskq_cancel_id(spl_kmem_cache_taskq, id);
- }
-
- spin_lock(&skc->skc_lock);
- skc->skc_taskqid = id;
spin_unlock(&skc->skc_lock);
-
- atomic_dec(&skc->skc_ref);
}
/*
skm->skm_size = skc->skc_mag_size;
skm->skm_refill = skc->skc_mag_refill;
skm->skm_cache = skc;
- skm->skm_age = jiffies;
skm->skm_cpu = cpu;
}
skc->skc_flags = flags;
skc->skc_obj_size = size;
skc->skc_obj_align = SPL_KMEM_CACHE_ALIGN;
- skc->skc_delay = SPL_KMEM_CACHE_DELAY;
atomic_set(&skc->skc_ref, 0);
INIT_LIST_HEAD(&skc->skc_list);
skc->skc_flags |= KMC_NOMAGAZINE;
}
- if (spl_kmem_cache_expire & KMC_EXPIRE_AGE) {
- skc->skc_taskqid = taskq_dispatch_delay(spl_kmem_cache_taskq,
- spl_cache_age, skc, TQ_SLEEP,
- ddi_get_lbolt() + skc->skc_delay / 3 * HZ);
- }
-
down_write(&spl_kmem_cache_sem);
list_add_tail(&skc->skc_list, &spl_kmem_cache_list);
up_write(&spl_kmem_cache_sem);
if (likely(skm->skm_avail)) {
/* Object available in CPU cache, use it */
obj = skm->skm_objs[--skm->skm_avail];
- skm->skm_age = jiffies;
} else {
obj = spl_cache_refill(skc, skm, flags);
if ((obj == NULL) && !(flags & KM_NOSLEEP))
goto out;
/* Reclaim from the magazine and free all now empty slabs. */
- if (spl_kmem_cache_expire & KMC_EXPIRE_MEM) {
- spl_kmem_magazine_t *skm;
- unsigned long irq_flags;
-
- local_irq_save(irq_flags);
- skm = skc->skc_mag[smp_processor_id()];
- spl_cache_flush(skc, skm, skm->skm_avail);
- local_irq_restore(irq_flags);
- }
+ unsigned long irq_flags;
+ local_irq_save(irq_flags);
+ spl_kmem_magazine_t *skm = skc->skc_mag[smp_processor_id()];
+ spl_cache_flush(skc, skm, skm->skm_avail);
+ local_irq_restore(irq_flags);
spl_slab_reclaim(skc);
clear_bit_unlock(KMC_BIT_REAPING, &skc->skc_flags);
projects / mirror_zfs.git / commitdiff