include/sys/kmem.h

   1 /*
   2  *  This file is part of the SPL: Solaris Porting Layer.
   3  *
   4  *  Copyright (c) 2008 Lawrence Livermore National Security, LLC.
   5  *  Produced at Lawrence Livermore National Laboratory
   6  *  Written by:
   7  *          Brian Behlendorf <behlendorf1@llnl.gov>,
   8  *          Herb Wartens <wartens2@llnl.gov>,
   9  *          Jim Garlick <garlick@llnl.gov>
  10  *  UCRL-CODE-235197
  11  *
  12  *  This is free software; you can redistribute it and/or modify it
  13  *  under the terms of the GNU General Public License as published by
  14  *  the Free Software Foundation; either version 2 of the License, or
  15  *  (at your option) any later version.
  16  *
  17  *  This is distributed in the hope that it will be useful, but WITHOUT
  18  *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  19  *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  20  *  for more details.
  21  *
  22  *  You should have received a copy of the GNU General Public License along
  23  *  with this program; if not, write to the Free Software Foundation, Inc.,
  24  *  51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  25  */
  26
  27 #ifndef _SPL_KMEM_H
  28 #define _SPL_KMEM_H
  29
  30 #ifdef  __cplusplus
  31 extern "C" {
  32 #endif
  33
  34 #undef DEBUG_KMEM_UNIMPLEMENTED
  35
  36 #include <linux/module.h>
  37 #include <linux/slab.h>
  38 #include <linux/vmalloc.h>
  39 #include <linux/mm.h>
  40 #include <linux/spinlock.h>
  41 #include <linux/rwsem.h>
  42 #include <linux/hash.h>
  43 #include <linux/ctype.h>
  44 #include <sys/types.h>
  45 #include <sys/debug.h>
  46 /*
  47  * Memory allocation interfaces
  48  */
  49 #define KM_SLEEP                        GFP_KERNEL
  50 #define KM_NOSLEEP                      GFP_ATOMIC
  51 #undef  KM_PANIC                        /* No linux analog */
  52 #define KM_PUSHPAGE                     (GFP_KERNEL | __GFP_HIGH)
  53 #define KM_VMFLAGS                      GFP_LEVEL_MASK
  54 #define KM_FLAGS                        __GFP_BITS_MASK
  55
  56 #ifdef DEBUG_KMEM
  57 extern atomic64_t kmem_alloc_used;
  58 extern unsigned long kmem_alloc_max;
  59 extern atomic64_t vmem_alloc_used;
  60 extern unsigned long vmem_alloc_max;
  61
  62 extern int kmem_warning_flag;
  63 extern atomic64_t kmem_cache_alloc_failed;
  64
  65 /* XXX - Not to surprisingly with debugging enabled the xmem_locks are very
  66  * highly contended particularly on xfree().  If we want to run with this
  67  * detailed debugging enabled for anything other than debugging  we need to
  68  * minimize the contention by moving to a lock per xmem_table entry model.
  69  */
  70 #define KMEM_HASH_BITS          10
  71 #define KMEM_TABLE_SIZE         (1 << KMEM_HASH_BITS)
  72
  73 extern struct hlist_head kmem_table[KMEM_TABLE_SIZE];
  74 extern struct list_head kmem_list;
  75 extern spinlock_t kmem_lock;
  76
  77 #define VMEM_HASH_BITS          10
  78 #define VMEM_TABLE_SIZE         (1 << VMEM_HASH_BITS)
  79
  80 extern struct hlist_head vmem_table[VMEM_TABLE_SIZE];
  81 extern struct list_head vmem_list;
  82 extern spinlock_t vmem_lock;
  83
  84 typedef struct kmem_debug {
  85         struct hlist_node kd_hlist;     /* Hash node linkage */
  86         struct list_head kd_list;       /* List of all allocations */
  87         void *kd_addr;                  /* Allocation pointer */
  88         size_t kd_size;                 /* Allocation size */
  89         const char *kd_func;            /* Allocation function */
  90         int kd_line;                    /* Allocation line */
  91 } kmem_debug_t;
  92
  93 static __inline__ kmem_debug_t *
  94 __kmem_del_init(spinlock_t *lock,struct hlist_head *table,int bits,void *addr)
  95 {
  96         struct hlist_head *head;
  97         struct hlist_node *node;
  98         struct kmem_debug *p;
  99         unsigned long flags;
 100
 101         spin_lock_irqsave(lock, flags);
 102         head = &table[hash_ptr(addr, bits)];
 103         hlist_for_each_entry_rcu(p, node, head, kd_hlist) {
 104                 if (p->kd_addr == addr) {
 105                         hlist_del_init(&p->kd_hlist);
 106                         list_del_init(&p->kd_list);
 107                         spin_unlock_irqrestore(lock, flags);
 108                         return p;
 109                 }
 110         }
 111
 112         spin_unlock_irqrestore(lock, flags);
 113         return NULL;
 114 }
 115
 116 #define __kmem_alloc(size, flags, allocator)                                  \
 117 ({      void *_ptr_ = NULL;                                                   \
 118         kmem_debug_t *_dptr_;                                                 \
 119         unsigned long _flags_;                                                \
 120                                                                               \
 121         _dptr_ = (kmem_debug_t *)kmalloc(sizeof(kmem_debug_t), (flags));      \
 122         if (_dptr_ == NULL) {                                                 \
 123                 __CDEBUG_LIMIT(S_KMEM, D_WARNING, "Warning "                  \
 124                                "kmem_alloc(%d, 0x%x) debug failed\n",         \
 125                                sizeof(kmem_debug_t), (int)(flags));           \
 126         } else {                                                              \
 127                 /* Marked unlikely because we should never be doing this, */  \
 128                 /* we tolerate to up 2 pages but a single page is best.   */  \
 129                 if (unlikely((size) > (PAGE_SIZE * 2)) && kmem_warning_flag)  \
 130                         __CDEBUG_LIMIT(S_KMEM, D_WARNING, "Warning large "    \
 131                                        "kmem_alloc(%d, 0x%x) (%ld/%ld)\n",    \
 132                                        (int)(size), (int)(flags),             \
 133                                        atomic64_read(&kmem_alloc_used),       \
 134                                        kmem_alloc_max);                       \
 135                                                                               \
 136                 _ptr_ = (void *)allocator((size), (flags));                   \
 137                 if (_ptr_ == NULL) {                                          \
 138                         kfree(_dptr_);                                        \
 139                         __CDEBUG_LIMIT(S_KMEM, D_WARNING, "Warning "          \
 140                                        "kmem_alloc(%d, 0x%x) failed (%ld/"    \
 141                                        "%ld)\n", (int)(size), (int)(flags),   \
 142                                        atomic64_read(&kmem_alloc_used),       \
 143                                        kmem_alloc_max);                       \
 144                 } else {                                                      \
 145                         atomic64_add((size), &kmem_alloc_used);               \
 146                         if (unlikely(atomic64_read(&kmem_alloc_used) >        \
 147                             kmem_alloc_max))                                  \
 148                                 kmem_alloc_max =                              \
 149                                         atomic64_read(&kmem_alloc_used);      \
 150                                                                               \
 151                         INIT_HLIST_NODE(&_dptr_->kd_hlist);                   \
 152                         INIT_LIST_HEAD(&_dptr_->kd_list);                     \
 153                         _dptr_->kd_addr = _ptr_;                              \
 154                         _dptr_->kd_size = (size);                             \
 155                         _dptr_->kd_func = __FUNCTION__;                       \
 156                         _dptr_->kd_line = __LINE__;                           \
 157                         spin_lock_irqsave(&kmem_lock, _flags_);               \
 158                         hlist_add_head_rcu(&_dptr_->kd_hlist,                 \
 159                                 &kmem_table[hash_ptr(_ptr_, KMEM_HASH_BITS)]);\
 160                         list_add_tail(&_dptr_->kd_list, &kmem_list);          \
 161                         spin_unlock_irqrestore(&kmem_lock, _flags_);          \
 162                                                                               \
 163                         __CDEBUG_LIMIT(S_KMEM, D_INFO, "kmem_alloc("          \
 164                                        "%d, 0x%x) = %p (%ld/%ld)\n",          \
 165                                        (int)(size), (int)(flags), _ptr_,      \
 166                                        atomic64_read(&kmem_alloc_used),       \
 167                                        kmem_alloc_max);                       \
 168                 }                                                             \
 169         }                                                                     \
 170                                                                               \
 171         _ptr_;                                                                \
 172 })
 173
 174 #define kmem_alloc(size, flags)         __kmem_alloc((size), (flags), kmalloc)
 175 #define kmem_zalloc(size, flags)        __kmem_alloc((size), (flags), kzalloc)
 176
 177 #define kmem_free(ptr, size)                                                  \
 178 ({                                                                            \
 179         kmem_debug_t *_dptr_;                                                 \
 180         ASSERT((ptr) || (size > 0));                                          \
 181                                                                               \
 182         _dptr_ = __kmem_del_init(&kmem_lock, kmem_table, KMEM_HASH_BITS, ptr);\
 183         ASSERT(_dptr_); /* Must exist in hash due to kmem_alloc() */          \
 184         ASSERTF(_dptr_->kd_size == (size), "kd_size (%d) != size (%d), "      \
 185                 "kd_func = %s, kd_line = %d\n", _dptr_->kd_size, (size),      \
 186                 _dptr_->kd_func, _dptr_->kd_line); /* Size must match */      \
 187         atomic64_sub((size), &kmem_alloc_used);                               \
 188         __CDEBUG_LIMIT(S_KMEM, D_INFO, "kmem_free(%p, %d) (%ld/%ld)\n",       \
 189                        (ptr), (int)(size), atomic64_read(&kmem_alloc_used),   \
 190                        kmem_alloc_max);                                       \
 191                                                                               \
 192         memset(_dptr_, 0x5a, sizeof(kmem_debug_t));                           \
 193         kfree(_dptr_);                                                        \
 194                                                                               \
 195         memset(ptr, 0x5a, (size));                                            \
 196         kfree(ptr);                                                           \
 197 })
 198
 199 #define __vmem_alloc(size, flags)                                             \
 200 ({      void *_ptr_ = NULL;                                                   \
 201         kmem_debug_t *_dptr_;                                                 \
 202         unsigned long _flags_;                                                \
 203                                                                               \
 204         ASSERT((flags) & KM_SLEEP);                                           \
 205                                                                               \
 206         _dptr_ = (kmem_debug_t *)kmalloc(sizeof(kmem_debug_t), (flags));      \
 207         if (_dptr_ == NULL) {                                                 \
 208                 __CDEBUG_LIMIT(S_KMEM, D_WARNING, "Warning "                  \
 209                                "vmem_alloc(%d, 0x%x) debug failed\n",         \
 210                                sizeof(kmem_debug_t), (int)(flags));           \
 211         } else {                                                              \
 212                 _ptr_ = (void *)__vmalloc((size), (((flags) |                 \
 213                                           __GFP_HIGHMEM) & ~__GFP_ZERO),      \
 214                                           PAGE_KERNEL);                       \
 215                 if (_ptr_ == NULL) {                                          \
 216                         kfree(_dptr_);                                        \
 217                         __CDEBUG_LIMIT(S_KMEM, D_WARNING, "Warning "          \
 218                                        "vmem_alloc(%d, 0x%x) failed (%ld/"    \
 219                                        "%ld)\n", (int)(size), (int)(flags),   \
 220                                       atomic64_read(&vmem_alloc_used),        \
 221                                       vmem_alloc_max);                        \
 222                 } else {                                                      \
 223                         if (flags & __GFP_ZERO)                               \
 224                                 memset(_ptr_, 0, (size));                     \
 225                                                                               \
 226                         atomic64_add((size), &vmem_alloc_used);               \
 227                         if (unlikely(atomic64_read(&vmem_alloc_used) >        \
 228                             vmem_alloc_max))                                  \
 229                                 vmem_alloc_max =                              \
 230                                         atomic64_read(&vmem_alloc_used);      \
 231                                                                               \
 232                         INIT_HLIST_NODE(&_dptr_->kd_hlist);                   \
 233                         INIT_LIST_HEAD(&_dptr_->kd_list);                     \
 234                         _dptr_->kd_addr = _ptr_;                              \
 235                         _dptr_->kd_size = (size);                             \
 236                         _dptr_->kd_func = __FUNCTION__;                       \
 237                         _dptr_->kd_line = __LINE__;                           \
 238                         spin_lock_irqsave(&vmem_lock, _flags_);               \
 239                         hlist_add_head_rcu(&_dptr_->kd_hlist,                 \
 240                                 &vmem_table[hash_ptr(_ptr_, VMEM_HASH_BITS)]);\
 241                         list_add_tail(&_dptr_->kd_list, &vmem_list);          \
 242                         spin_unlock_irqrestore(&vmem_lock, _flags_);          \
 243                                                                               \
 244                         __CDEBUG_LIMIT(S_KMEM, D_INFO, "vmem_alloc("          \
 245                                        "%d, 0x%x) = %p (%ld/%ld)\n",          \
 246                                        (int)(size), (int)(flags), _ptr_,      \
 247                                        atomic64_read(&vmem_alloc_used),       \
 248                                        vmem_alloc_max);                       \
 249                 }                                                             \
 250         }                                                                     \
 251                                                                               \
 252         _ptr_;                                                                \
 253 })
 254
 255 #define vmem_alloc(size, flags)         __vmem_alloc((size), (flags))
 256 #define vmem_zalloc(size, flags)        __vmem_alloc((size), ((flags) |       \
 257                                                      __GFP_ZERO))
 258
 259 #define vmem_free(ptr, size)                                                  \
 260 ({                                                                            \
 261         kmem_debug_t *_dptr_;                                                 \
 262         ASSERT((ptr) || (size > 0));                                          \
 263                                                                               \
 264         _dptr_ = __kmem_del_init(&vmem_lock, vmem_table, VMEM_HASH_BITS, ptr);\
 265         ASSERT(_dptr_); /* Must exist in hash due to vmem_alloc() */          \
 266         ASSERTF(_dptr_->kd_size == (size), "kd_size (%d) != size (%d), "      \
 267                 "kd_func = %s, kd_line = %d\n", _dptr_->kd_size, (size),      \
 268                 _dptr_->kd_func, _dptr_->kd_line); /* Size must match */      \
 269         atomic64_sub((size), &vmem_alloc_used);                               \
 270         __CDEBUG_LIMIT(S_KMEM, D_INFO, "vmem_free(%p, %d) (%ld/%ld)\n",       \
 271                        (ptr), (int)(size), atomic64_read(&vmem_alloc_used),   \
 272                        vmem_alloc_max);                                       \
 273                                                                               \
 274         memset(_dptr_, 0x5a, sizeof(kmem_debug_t));                           \
 275         kfree(_dptr_);                                                        \
 276                                                                               \
 277         memset(ptr, 0x5a, (size));                                            \
 278         vfree(ptr);                                                           \
 279 })
 280
 281 #else /* DEBUG_KMEM */
 282
 283 #define kmem_alloc(size, flags)         kmalloc((size), (flags))
 284 #define kmem_zalloc(size, flags)        kzalloc((size), (flags))
 285 #define kmem_free(ptr, size)            kfree(ptr)
 286
 287 #define vmem_alloc(size, flags)         __vmalloc((size), ((flags) |          \
 288                                         __GFP_HIGHMEM), PAGE_KERNEL)
 289 #define vmem_zalloc(size, flags)                                              \
 290 ({                                                                            \
 291         void *_ptr_ = __vmalloc((size),((flags)|__GFP_HIGHMEM),PAGE_KERNEL);  \
 292         if (_ptr_)                                                            \
 293                 memset(_ptr_, 0, (size));                                     \
 294         _ptr_;                                                                \
 295 })
 296 #define vmem_free(ptr, size)            vfree(ptr)
 297
 298 #endif /* DEBUG_KMEM */
 299
 300 #ifdef DEBUG_KMEM_UNIMPLEMENTED
 301 static __inline__ void *
 302 kmem_alloc_tryhard(size_t size, size_t *alloc_size, int kmflags)
 303 {
 304 #error "kmem_alloc_tryhard() not implemented"
 305 }
 306 #endif /* DEBUG_KMEM_UNIMPLEMENTED */
 307
 308 /*
 309  * Slab allocation interfaces
 310  */
 311 #undef  KMC_NOTOUCH                     /* XXX: Unsupported */
 312 #define KMC_NODEBUG                     0x00000000 /* Default behavior */
 313 #define KMC_NOMAGAZINE                  /* XXX: Unsupported */
 314 #define KMC_NOHASH                      /* XXX: Unsupported */
 315 #define KMC_QCACHE                      /* XXX: Unsupported */
 316
 317 #define KMC_REAP_CHUNK                  256
 318 #define KMC_DEFAULT_SEEKS               DEFAULT_SEEKS
 319
 320 #ifdef DEBUG_KMEM_UNIMPLEMENTED
 321 static __inline__ void kmem_init(void) {
 322 #error "kmem_init() not implemented"
 323 }
 324
 325 static __inline__ void kmem_thread_init(void) {
 326 #error "kmem_thread_init() not implemented"
 327 }
 328
 329 static __inline__ void kmem_mp_init(void) {
 330 #error "kmem_mp_init() not implemented"
 331 }
 332
 333 static __inline__ void kmem_reap_idspace(void) {
 334 #error "kmem_reap_idspace() not implemented"
 335 }
 336
 337 static __inline__ size_t kmem_avail(void) {
 338 #error "kmem_avail() not implemented"
 339 }
 340
 341 static __inline__ size_t kmem_maxavail(void) {
 342 #error "kmem_maxavail() not implemented"
 343 }
 344
 345 static __inline__ uint64_t kmem_cache_stat(spl_kmem_cache_t *cache) {
 346 #error "kmem_cache_stat() not implemented"
 347 }
 348 #endif /* DEBUG_KMEM_UNIMPLEMENTED */
 349
 350 /* XXX - Used by arc.c to adjust its memory footprint. We may want
 351  *       to use this hook in the future to adjust behavior based on
 352  *       debug levels.  For now it's safe to always return 0.
 353  */
 354 static __inline__ int
 355 kmem_debugging(void)
 356 {
 357         return 0;
 358 }
 359
 360 extern int kmem_set_warning(int flag);
 361
 362
 363 #define SKO_MAGIC                       0x20202020
 364 #define SKS_MAGIC                       0x22222222
 365 #define SKC_MAGIC                       0x2c2c2c2c
 366
 367 #define SPL_KMEM_CACHE_HASH_BITS        12
 368 #define SPL_KMEM_CACHE_HASH_ELTS        (1 << SPL_KMEM_CACHE_HASH_BITS)
 369 #define SPL_KMEM_CACHE_HASH_SIZE        (sizeof(struct hlist_head) * \
 370                                          SPL_KMEM_CACHE_HASH_ELTS)
 371
 372 #define SPL_KMEM_CACHE_DELAY            5
 373 #define SPL_KMEM_CACHE_OBJ_PER_SLAB     32
 374
 375 typedef int (*spl_kmem_ctor_t)(void *, void *, int);
 376 typedef void (*spl_kmem_dtor_t)(void *, void *);
 377 typedef void (*spl_kmem_reclaim_t)(void *);
 378
 379 typedef struct spl_kmem_obj {
 380         uint32_t                sko_magic;      /* Sanity magic */
 381         uint32_t                sko_flags;      /* Per object flags */
 382         void                    *sko_addr;      /* Buffer address */
 383         struct spl_kmem_slab    *sko_slab;      /* Owned by slab */
 384         struct list_head        sko_list;       /* Free object list linkage */
 385         struct hlist_node       sko_hlist;      /* Used object hash linkage */
 386 } spl_kmem_obj_t;
 387
 388 typedef struct spl_kmem_slab {
 389         uint32_t                sks_magic;      /* Sanity magic */
 390         uint32_t                sks_objs;       /* Objects per slab */
 391         struct spl_kmem_cache   *sks_cache;     /* Owned by cache */
 392         struct list_head        sks_list;       /* Slab list linkage */
 393         struct list_head        sks_free_list;  /* Free object list */
 394         unsigned long           sks_age;        /* Last modify jiffie */
 395         atomic_t                sks_ref;        /* Ref count used objects */
 396 } spl_kmem_slab_t;
 397
 398 typedef struct spl_kmem_cache {
 399         uint32_t                skc_magic;      /* Sanity magic */
 400         uint32_t                skc_name_size;  /* Name length */
 401         char                    *skc_name;      /* Name string */
 402         spl_kmem_ctor_t         skc_ctor;       /* Constructor */
 403         spl_kmem_dtor_t         skc_dtor;       /* Destructor */
 404         spl_kmem_reclaim_t      skc_reclaim;    /* Reclaimator */
 405         void                    *skc_private;   /* Private data */
 406         void                    *skc_vmp;       /* Unused */
 407         uint32_t                skc_flags;      /* Flags */
 408         uint32_t                skc_obj_size;   /* Object size */
 409         uint32_t                skc_chunk_size; /* sizeof(*obj) + alignment */
 410         uint32_t                skc_slab_size;  /* slab size */
 411         uint32_t                skc_max_chunks; /* max chunks per slab */
 412         uint32_t                skc_delay;      /* slab reclaim interval */
 413         uint32_t                skc_hash_bits;  /* Hash table bits */
 414         uint32_t                skc_hash_size;  /* Hash table size */
 415         uint32_t                skc_hash_elts;  /* Hash table elements */
 416         struct hlist_head       *skc_hash;      /* Hash table address */
 417         struct list_head        skc_list;       /* List of caches linkage */
 418         struct list_head        skc_complete_list;/* Completely alloc'ed */
 419         struct list_head        skc_partial_list; /* Partially alloc'ed */
 420         spinlock_t              skc_lock;       /* Cache lock */
 421         uint64_t                skc_slab_fail;  /* Slab alloc failures */
 422         uint64_t                skc_slab_create;/* Slab creates */
 423         uint64_t                skc_slab_destroy;/* Slab destroys */
 424         uint64_t                skc_slab_total; /* Slab total current */
 425         uint64_t                skc_slab_alloc; /* Slab alloc current */
 426         uint64_t                skc_slab_max;   /* Slab max historic  */
 427         uint64_t                skc_obj_total;  /* Obj total current */
 428         uint64_t                skc_obj_alloc;  /* Obj alloc current */
 429         uint64_t                skc_obj_max;    /* Obj max historic */
 430         uint64_t                skc_hash_depth; /* Hash depth */
 431         uint64_t                skc_hash_max;   /* Hash depth max */
 432 } spl_kmem_cache_t;
 433
 434 extern spl_kmem_cache_t *
 435 spl_kmem_cache_create(char *name, size_t size, size_t align,
 436         spl_kmem_ctor_t ctor, spl_kmem_dtor_t dtor, spl_kmem_reclaim_t reclaim,
 437         void *priv, void *vmp, int flags);
 438
 439 extern void spl_kmem_cache_destroy(spl_kmem_cache_t *skc);
 440 extern void *spl_kmem_cache_alloc(spl_kmem_cache_t *skc, int flags);
 441 extern void spl_kmem_cache_free(spl_kmem_cache_t *skc, void *obj);
 442 extern void spl_kmem_cache_reap_now(spl_kmem_cache_t *skc);
 443 extern void spl_kmem_reap(void);
 444
 445 int spl_kmem_init(void);
 446 void spl_kmem_fini(void);
 447
 448 #define kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags) \
 449         spl_kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags)
 450 #define kmem_cache_destroy(skc)         spl_kmem_cache_destroy(skc)
 451 #define kmem_cache_alloc(skc, flags)    spl_kmem_cache_alloc(skc, flags)
 452 #define kmem_cache_free(skc, obj)       spl_kmem_cache_free(skc, obj)
 453 #define kmem_cache_reap_now(skc)        spl_kmem_cache_reap_now(skc)
 454 #define kmem_reap()                     spl_kmem_reap()
 455
 456 #ifdef HAVE_KMEM_CACHE_CREATE_DTOR
 457 #define __kmem_cache_create(name, size, align, flags, ctor, dtor) \
 458         kmem_cache_create(name, size, align, flags, ctor, dtor)
 459 #else
 460 #define __kmem_cache_create(name, size, align, flags, ctor, dtor) \
 461         kmem_cache_create(name, size, align, flags, ctor)
 462 #endif /* HAVE_KMEM_CACHE_CREATE_DTOR */
 463
 464 #ifdef  __cplusplus
 465 }
 466 #endif
 467
 468 #endif  /* _SPL_KMEM_H */