modules/spl/spl-kmem.c

   1 #include <sys/kmem.h>
   2
   3 /*
   4  * Memory allocation interfaces
   5  */
   6 #ifdef DEBUG_KMEM
   7 /* Shim layer memory accounting */
   8 atomic64_t kmem_alloc_used;
   9 unsigned long kmem_alloc_max = 0;
  10 atomic64_t vmem_alloc_used;
  11 unsigned long vmem_alloc_max = 0;
  12 int kmem_warning_flag = 1;
  13
  14 EXPORT_SYMBOL(kmem_alloc_used);
  15 EXPORT_SYMBOL(kmem_alloc_max);
  16 EXPORT_SYMBOL(vmem_alloc_used);
  17 EXPORT_SYMBOL(vmem_alloc_max);
  18 EXPORT_SYMBOL(kmem_warning_flag);
  19
  20 int kmem_set_warning(int flag) { return (kmem_warning_flag = !!flag); }
  21 #else
  22 int kmem_set_warning(int flag) { return 0; }
  23 #endif
  24 EXPORT_SYMBOL(kmem_set_warning);
  25
  26 /*
  27  * Slab allocation interfaces
  28  *
  29  * While the linux slab implementation was inspired by solaris they
  30  * have made some changes to the API which complicates this shim
  31  * layer.  For one thing the same symbol names are used with different
  32  * arguments for the prototypes.  To deal with this we must use the
  33  * preprocessor to re-order arguments.  Happily for us standard C says,
  34  * "Macro's appearing in their own expansion are not reexpanded" so
  35  * this does not result in an infinite recursion.  Additionally the
  36  * function pointers registered by solarias differ from those used
  37  * by linux so a lookup and mapping from linux style callback to a
  38  * solaris style callback is needed.  There is some overhead in this
  39  * operation which isn't horibile but it needs to be kept in mind.
  40  */
  41 typedef struct kmem_cache_cb {
  42         struct list_head    kcc_list;
  43         kmem_cache_t *      kcc_cache;
  44         kmem_constructor_t  kcc_constructor;
  45         kmem_destructor_t   kcc_destructor;
  46         kmem_reclaim_t      kcc_reclaim;
  47         void *              kcc_private;
  48         void *              kcc_vmp;
  49 } kmem_cache_cb_t;
  50
  51
  52 static spinlock_t kmem_cache_cb_lock = SPIN_LOCK_UNLOCKED;
  53 static LIST_HEAD(kmem_cache_cb_list);
  54 static struct shrinker *kmem_cache_shrinker;
  55
  56 /* Function must be called while holding the kmem_cache_cb_lock
  57  * Because kmem_cache_t is an opaque datatype we're forced to
  58  * match pointers to identify specific cache entires.
  59  */
  60 static kmem_cache_cb_t *
  61 kmem_cache_find_cache_cb(kmem_cache_t *cache)
  62 {
  63         kmem_cache_cb_t *kcc;
  64
  65         list_for_each_entry(kcc, &kmem_cache_cb_list, kcc_list)
  66                 if (cache == kcc->kcc_cache)
  67                         return kcc;
  68
  69         return NULL;
  70 }
  71
  72 static kmem_cache_cb_t *
  73 kmem_cache_add_cache_cb(kmem_cache_t *cache,
  74                         kmem_constructor_t constructor,
  75                         kmem_destructor_t destructor,
  76                         kmem_reclaim_t reclaim,
  77                         void *priv, void *vmp)
  78 {
  79         kmem_cache_cb_t *kcc;
  80         unsigned long flags;
  81
  82         kcc = (kmem_cache_cb_t *)kmalloc(sizeof(*kcc), GFP_KERNEL);
  83         if (kcc) {
  84                 kcc->kcc_cache = cache;
  85                 kcc->kcc_constructor = constructor;
  86                 kcc->kcc_destructor = destructor;
  87                 kcc->kcc_reclaim = reclaim;
  88                 kcc->kcc_private = priv;
  89                 kcc->kcc_vmp = vmp;
  90                 spin_lock_irqsave(&kmem_cache_cb_lock, flags);
  91                 list_add(&kcc->kcc_list, &kmem_cache_cb_list);
  92                 spin_unlock_irqrestore(&kmem_cache_cb_lock, flags);
  93         }
  94
  95         return kcc;
  96 }
  97
  98 static void
  99 kmem_cache_remove_cache_cb(kmem_cache_cb_t *kcc)
 100 {
 101         unsigned long flags;
 102
 103         spin_lock_irqsave(&kmem_cache_cb_lock, flags);
 104         list_del(&kcc->kcc_list);
 105         spin_unlock_irqrestore(&kmem_cache_cb_lock, flags);
 106
 107        if (kcc)
 108               kfree(kcc);
 109 }
 110
 111 static void
 112 kmem_cache_generic_constructor(void *ptr, kmem_cache_t *cache, unsigned long flags)
 113 {
 114         kmem_cache_cb_t *kcc;
 115         kmem_constructor_t constructor;
 116         unsigned long irqflags;
 117         void *private;
 118
 119         spin_lock_irqsave(&kmem_cache_cb_lock, irqflags);
 120
 121         /* Callback list must be in sync with linux slab caches */
 122         kcc = kmem_cache_find_cache_cb(cache);
 123         BUG_ON(!kcc);
 124         constructor = kcc->kcc_constructor;
 125         private = kcc->kcc_private;
 126
 127         spin_unlock_irqrestore(&kmem_cache_cb_lock, irqflags);
 128
 129         if (constructor)
 130                 constructor(ptr, private, (int)flags);
 131
 132         /* Linux constructor has no return code, silently eat it */
 133 }
 134
 135 static void
 136 kmem_cache_generic_destructor(void *ptr, kmem_cache_t *cache, unsigned long flags)
 137 {
 138         kmem_cache_cb_t *kcc;
 139         kmem_destructor_t destructor;
 140         unsigned long irqflags;
 141         void *private;
 142
 143         spin_lock_irqsave(&kmem_cache_cb_lock, irqflags);
 144
 145         /* Callback list must be in sync with linux slab caches */
 146         kcc = kmem_cache_find_cache_cb(cache);
 147         BUG_ON(!kcc);
 148         destructor = kcc->kcc_destructor;
 149         private = kcc->kcc_private;
 150
 151         spin_unlock_irqrestore(&kmem_cache_cb_lock, irqflags);
 152
 153         /* Solaris destructor takes no flags, silently eat them */
 154         if (destructor)
 155                 destructor(ptr, private);
 156 }
 157
 158 /* XXX - Arguments are ignored */
 159 static int
 160 kmem_cache_generic_shrinker(int nr_to_scan, unsigned int gfp_mask)
 161 {
 162         kmem_cache_cb_t *kcc;
 163         unsigned long flags;
 164         int total = 0;
 165
 166         /* Under linux a shrinker is not tightly coupled with a slab
 167          * cache.  In fact linux always systematically trys calling all
 168          * registered shrinker callbacks until its target reclamation level
 169          * is reached.  Because of this we only register one shrinker
 170          * function in the shim layer for all slab caches.  And we always
 171          * attempt to shrink all caches when this generic shrinker is called.
 172          */
 173         spin_lock_irqsave(&kmem_cache_cb_lock, flags);
 174
 175         list_for_each_entry(kcc, &kmem_cache_cb_list, kcc_list) {
 176                 /* Under linux the desired number and gfp type of objects
 177                  * is passed to the reclaiming function as a sugested reclaim
 178                  * target.  I do not pass these args on because reclaim
 179                  * policy is entirely up to the owner under solaris.  We only
 180                  * pass on the pre-registered private data.
 181                  */
 182                 if (kcc->kcc_reclaim)
 183                         kcc->kcc_reclaim(kcc->kcc_private);
 184
 185                 total += 1;
 186         }
 187
 188         /* Under linux we should return the remaining number of entires in
 189          * the cache.  Unfortunately, I don't see an easy way to safely
 190          * emulate this behavior so I'm returning one entry per cache which
 191          * was registered with the generic shrinker.  This should fake out
 192          * the linux VM when it attempts to shrink caches.
 193          */
 194         spin_unlock_irqrestore(&kmem_cache_cb_lock, flags);
 195         return total;
 196 }
 197
 198 /* Ensure the __kmem_cache_create/__kmem_cache_destroy macros are
 199  * removed here to prevent a recursive substitution, we want to call
 200  * the native linux version.
 201  */
 202 #undef kmem_cache_create
 203 #undef kmem_cache_destroy
 204
 205 kmem_cache_t *
 206 __kmem_cache_create(char *name, size_t size, size_t align,
 207         kmem_constructor_t constructor,
 208         kmem_destructor_t destructor,
 209         kmem_reclaim_t reclaim,
 210         void *priv, void *vmp, int flags)
 211 {
 212         kmem_cache_t *cache;
 213         kmem_cache_cb_t *kcc;
 214         int shrinker_flag = 0;
 215         char *cache_name;
 216
 217         /* FIXME: - Option currently unsupported by shim layer */
 218         BUG_ON(vmp);
 219
 220         cache_name = kzalloc(strlen(name) + 1, GFP_KERNEL);
 221         if (cache_name == NULL)
 222                 return NULL;
 223
 224         strcpy(cache_name, name);
 225         cache = kmem_cache_create(cache_name, size, align, flags,
 226                                   kmem_cache_generic_constructor,
 227                                   kmem_cache_generic_destructor);
 228         if (cache == NULL)
 229                 return NULL;
 230
 231         /* Register shared shrinker function on initial cache create */
 232         spin_lock(&kmem_cache_cb_lock);
 233         if (list_empty(&kmem_cache_cb_list)) {
 234                 kmem_cache_shrinker = set_shrinker(KMC_DEFAULT_SEEKS,
 235                                                  kmem_cache_generic_shrinker);
 236                 if (kmem_cache_shrinker == NULL) {
 237                         kmem_cache_destroy(cache);
 238                         spin_unlock(&kmem_cache_cb_lock);
 239                         return NULL;
 240                 }
 241
 242         }
 243         spin_unlock(&kmem_cache_cb_lock);
 244
 245         kcc = kmem_cache_add_cache_cb(cache, constructor, destructor,
 246                                       reclaim, priv, vmp);
 247         if (kcc == NULL) {
 248                 if (shrinker_flag) /* New shrinker registered must be removed */
 249                         remove_shrinker(kmem_cache_shrinker);
 250
 251                 kmem_cache_destroy(cache);
 252                 return NULL;
 253         }
 254
 255         return cache;
 256 }
 257 EXPORT_SYMBOL(__kmem_cache_create);
 258
 259 /* Return code provided despite Solaris's void return.  There should be no
 260  * harm here since the Solaris versions will ignore it anyway. */
 261 int
 262 __kmem_cache_destroy(kmem_cache_t *cache)
 263 {
 264         kmem_cache_cb_t *kcc;
 265         char *name;
 266         unsigned long flags;
 267         int rc;
 268
 269         spin_lock_irqsave(&kmem_cache_cb_lock, flags);
 270         kcc = kmem_cache_find_cache_cb(cache);
 271         spin_unlock_irqrestore(&kmem_cache_cb_lock, flags);
 272         if (kcc == NULL)
 273                 return -EINVAL;
 274
 275         name = (char *)kmem_cache_name(cache);
 276         rc = kmem_cache_destroy(cache);
 277         kmem_cache_remove_cache_cb(kcc);
 278         kfree(name);
 279
 280         /* Unregister generic shrinker on removal of all caches */
 281         spin_lock_irqsave(&kmem_cache_cb_lock, flags);
 282         if (list_empty(&kmem_cache_cb_list))
 283                 remove_shrinker(kmem_cache_shrinker);
 284
 285         spin_unlock_irqrestore(&kmem_cache_cb_lock, flags);
 286         return rc;
 287 }
 288 EXPORT_SYMBOL(__kmem_cache_destroy);
 289
 290 void
 291 __kmem_reap(void) {
 292         /* Since there's no easy hook in to linux to force all the registered
 293          * shrinkers to run we just run the ones registered for this shim */
 294         kmem_cache_generic_shrinker(KMC_REAP_CHUNK, GFP_KERNEL);
 295 }
 296 EXPORT_SYMBOL(__kmem_reap);
 297
 298 int
 299 kmem_init(void)
 300 {
 301 #ifdef DEBUG_KMEM
 302         atomic64_set(&kmem_alloc_used, 0);
 303         atomic64_set(&vmem_alloc_used, 0);
 304 #endif
 305         return 0;
 306 }
 307
 308 void
 309 kmem_fini(void)
 310 {
 311 #ifdef DEBUG_KMEM
 312         if (atomic64_read(&kmem_alloc_used) != 0)
 313                 printk("spl: Warning kmem leaked %ld/%ld bytes\n",
 314                        atomic_read(&kmem_alloc_used), kmem_alloc_max);
 315
 316         if (atomic64_read(&vmem_alloc_used) != 0)
 317                 printk("spl: Warning vmem leaked %ld/%ld bytes\n",
 318                        atomic_read(&vmem_alloc_used), vmem_alloc_max);
 319 #endif
 320 }