include/sys/kmem.h

   1 /*
   2  *  This file is part of the SPL: Solaris Porting Layer.
   3  *
   4  *  Copyright (c) 2008 Lawrence Livermore National Security, LLC.
   5  *  Produced at Lawrence Livermore National Laboratory
   6  *  Written by:
   7  *          Brian Behlendorf <behlendorf1@llnl.gov>,
   8  *          Herb Wartens <wartens2@llnl.gov>,
   9  *          Jim Garlick <garlick@llnl.gov>
  10  *  UCRL-CODE-235197
  11  *
  12  *  This is free software; you can redistribute it and/or modify it
  13  *  under the terms of the GNU General Public License as published by
  14  *  the Free Software Foundation; either version 2 of the License, or
  15  *  (at your option) any later version.
  16  *
  17  *  This is distributed in the hope that it will be useful, but WITHOUT
  18  *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  19  *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  20  *  for more details.
  21  *
  22  *  You should have received a copy of the GNU General Public License along
  23  *  with this program; if not, write to the Free Software Foundation, Inc.,
  24  *  51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA.
  25  */
  26
  27 #ifndef _SPL_KMEM_H
  28 #define _SPL_KMEM_H
  29
  30 #ifdef  __cplusplus
  31 extern "C" {
  32 #endif
  33
  34 #undef DEBUG_KMEM_UNIMPLEMENTED
  35 #undef DEBUG_KMEM_TRACKING /* Per-allocation memory tracking */
  36
  37 #include <linux/module.h>
  38 #include <linux/slab.h>
  39 #include <linux/vmalloc.h>
  40 #include <linux/mm.h>
  41 #include <linux/spinlock.h>
  42 #include <linux/rwsem.h>
  43 #include <linux/hash.h>
  44 #include <linux/ctype.h>
  45 #include <asm/atomic_compat.h>
  46 #include <sys/types.h>
  47 #include <sys/debug.h>
  48 #include <sys/vmsystm.h>
  49
  50 /*
  51  * Memory allocation interfaces
  52  */
  53 #define KM_SLEEP                        (GFP_KERNEL | __GFP_NOFAIL)
  54 #define KM_NOSLEEP                      GFP_ATOMIC
  55 #undef  KM_PANIC                        /* No linux analog */
  56 #define KM_PUSHPAGE                     (KM_SLEEP | __GFP_HIGH)
  57 #define KM_VMFLAGS                      GFP_LEVEL_MASK
  58 #define KM_FLAGS                        __GFP_BITS_MASK
  59
  60 /*
  61  * Used internally, the kernel does not need to support this flag
  62  */
  63 #ifndef __GFP_ZERO
  64 # define __GFP_ZERO                     0x8000
  65 #endif
  66
  67 #ifdef DEBUG_KMEM
  68
  69 extern atomic64_t kmem_alloc_used;
  70 extern unsigned long long kmem_alloc_max;
  71 extern atomic64_t vmem_alloc_used;
  72 extern unsigned long long vmem_alloc_max;
  73
  74 # define kmem_alloc(size, flags)             __kmem_alloc((size), (flags), 0, 0)
  75 # define kmem_zalloc(size, flags)            __kmem_alloc((size), ((flags) |  \
  76                                                  __GFP_ZERO), 0, 0)
  77
  78 /* The node alloc functions are only used by the SPL code itself */
  79 # ifdef HAVE_KMALLOC_NODE
  80 #  define kmem_alloc_node(size, flags, node) __kmem_alloc((size), (flags), 1, \
  81                                                  node)
  82 # else
  83 #  define kmem_alloc_node(size, flags, node) __kmem_alloc((size), (flags), 0, 0)
  84 # endif
  85
  86 # define vmem_zalloc(size, flags)            vmem_alloc((size), ((flags) |    \
  87                                                  __GFP_ZERO))
  88
  89 # ifdef DEBUG_KMEM_TRACKING
  90
  91 extern void *kmem_alloc_track(size_t size, int flags, const char *func,
  92     int line, int node_alloc, int node);
  93 extern void kmem_free_track(void *ptr, size_t size);
  94 extern void *vmem_alloc_track(size_t size, int flags, const char *func,
  95     int line);
  96 extern void vmem_free_track(void *ptr, size_t size);
  97
  98 #  define __kmem_alloc(size, flags, na, node) kmem_alloc_track((size),        \
  99                                                   (flags), __FUNCTION__,      \
 100                                                   __LINE__, (na), (node))
 101 #  define kmem_free(ptr, size)                kmem_free_track((ptr), (size))
 102 #  define vmem_alloc(size, flags)             vmem_alloc_track((size),        \
 103                                                   (flags),__FUNCTION__,       \
 104                                                   __LINE__)
 105 #  define vmem_free(ptr, size)                vmem_free_track((ptr), (size))
 106
 107 # else /* DEBUG_KMEM_TRACKING */
 108
 109 extern void *kmem_alloc_debug(size_t size, int flags, const char *func,
 110     int line, int node_alloc, int node);
 111 extern void kmem_free_debug(void *ptr, size_t size);
 112 extern void *vmem_alloc_debug(size_t size, int flags, const char *func,
 113     int line);
 114 extern void vmem_free_debug(void *ptr, size_t size);
 115
 116 #  define __kmem_alloc(size, flags, na, node) kmem_alloc_debug((size),        \
 117                                                   (flags), __FUNCTION__,      \
 118                                                   __LINE__, (na), (node))
 119 #  define kmem_free(ptr, size)                kmem_free_debug((ptr), (size))
 120 #  define vmem_alloc(size, flags)             vmem_alloc_debug((size),        \
 121                                                   (flags), __FUNCTION__,      \
 122                                                   __LINE__)
 123 #  define vmem_free(ptr, size)                vmem_free_debug((ptr), (size))
 124
 125 # endif /* DEBUG_KMEM_TRACKING */
 126
 127 #else /* DEBUG_KMEM */
 128
 129 # define kmem_alloc(size, flags)              kmalloc((size), (flags))
 130 # define kmem_zalloc(size, flags)             kzalloc((size), (flags))
 131 # define kmem_free(ptr, size)                 ((void)(size), kfree(ptr))
 132
 133 # ifdef HAVE_KMALLOC_NODE
 134 #  define kmem_alloc_node(size, flags, node)                                  \
 135           kmalloc_node((size), (flags), (node))
 136 # else
 137 #  define kmem_alloc_node(size, flags, node)                                  \
 138           kmalloc((size), (flags))
 139 # endif
 140
 141 # define vmem_alloc(size, flags)              __vmalloc((size), ((flags) |    \
 142                                                   __GFP_HIGHMEM), PAGE_KERNEL)
 143 # define vmem_zalloc(size, flags)                                             \
 144 ({                                                                            \
 145         void *_ptr_ = __vmalloc((size),((flags)|__GFP_HIGHMEM),PAGE_KERNEL);  \
 146         if (_ptr_)                                                            \
 147                 memset(_ptr_, 0, (size));                                     \
 148         _ptr_;                                                                \
 149 })
 150 # define vmem_free(ptr, size)           ((void)(size), vfree(ptr))
 151
 152 #endif /* DEBUG_KMEM */
 153
 154 #ifdef DEBUG_KMEM_UNIMPLEMENTED
 155 static __inline__ void *
 156 kmem_alloc_tryhard(size_t size, size_t *alloc_size, int kmflags)
 157 {
 158 #error "kmem_alloc_tryhard() not implemented"
 159 }
 160 #endif /* DEBUG_KMEM_UNIMPLEMENTED */
 161
 162 /*
 163  * Slab allocation interfaces
 164  */
 165 enum {
 166         KMC_BIT_NOTOUCH         = 0,    /* Don't update ages */
 167         KMC_BIT_NODEBUG         = 1,    /* Default behavior */
 168         KMC_BIT_NOMAGAZINE      = 2,    /* XXX: Unsupported */
 169         KMC_BIT_NOHASH          = 3,    /* XXX: Unsupported */
 170         KMC_BIT_QCACHE          = 4,    /* XXX: Unsupported */
 171         KMC_BIT_KMEM            = 5,    /* Use kmem cache */
 172         KMC_BIT_VMEM            = 6,    /* Use vmem cache */
 173         KMC_BIT_OFFSLAB         = 7,    /* Objects not on slab */
 174         KMC_BIT_REAPING         = 16,   /* Reaping in progress */
 175         KMC_BIT_DESTROY         = 17,   /* Destroy in progress */
 176 };
 177
 178 #define KMC_NOTOUCH             (1 << KMC_BIT_NOTOUCH)
 179 #define KMC_NODEBUG             (1 << KMC_BIT_NODEBUG)
 180 #define KMC_NOMAGAZINE          (1 << KMC_BIT_NOMAGAZINE)
 181 #define KMC_NOHASH              (1 << KMC_BIT_NOHASH)
 182 #define KMC_QCACHE              (1 << KMC_BIT_QCACHE)
 183 #define KMC_KMEM                (1 << KMC_BIT_KMEM)
 184 #define KMC_VMEM                (1 << KMC_BIT_VMEM)
 185 #define KMC_OFFSLAB             (1 << KMC_BIT_OFFSLAB)
 186 #define KMC_REAPING             (1 << KMC_BIT_REAPING)
 187 #define KMC_DESTROY             (1 << KMC_BIT_DESTROY)
 188
 189 #define KMC_REAP_CHUNK                  INT_MAX
 190 #define KMC_DEFAULT_SEEKS               1
 191
 192 #ifdef DEBUG_KMEM_UNIMPLEMENTED
 193 static __inline__ void kmem_init(void) {
 194 #error "kmem_init() not implemented"
 195 }
 196
 197 static __inline__ void kmem_thread_init(void) {
 198 #error "kmem_thread_init() not implemented"
 199 }
 200
 201 static __inline__ void kmem_mp_init(void) {
 202 #error "kmem_mp_init() not implemented"
 203 }
 204
 205 static __inline__ void kmem_reap_idspace(void) {
 206 #error "kmem_reap_idspace() not implemented"
 207 }
 208
 209 static __inline__ size_t kmem_avail(void) {
 210 #error "kmem_avail() not implemented"
 211 }
 212
 213 static __inline__ size_t kmem_maxavail(void) {
 214 #error "kmem_maxavail() not implemented"
 215 }
 216
 217 static __inline__ uint64_t kmem_cache_stat(spl_kmem_cache_t *cache) {
 218 #error "kmem_cache_stat() not implemented"
 219 }
 220 #endif /* DEBUG_KMEM_UNIMPLEMENTED */
 221
 222 /* XXX - Used by arc.c to adjust its memory footprint. We may want
 223  *       to use this hook in the future to adjust behavior based on
 224  *       debug levels.  For now it's safe to always return 0.
 225  */
 226 static __inline__ int
 227 kmem_debugging(void)
 228 {
 229         return 0;
 230 }
 231
 232 extern int kmem_set_warning(int flag);
 233
 234 extern struct list_head spl_kmem_cache_list;
 235 extern struct rw_semaphore spl_kmem_cache_sem;
 236
 237 #define SKM_MAGIC                       0x2e2e2e2e
 238 #define SKO_MAGIC                       0x20202020
 239 #define SKS_MAGIC                       0x22222222
 240 #define SKC_MAGIC                       0x2c2c2c2c
 241
 242 #define SPL_KMEM_CACHE_DELAY            15      /* Minimum slab release age */
 243 #define SPL_KMEM_CACHE_REAP             0       /* Default reap everything */
 244 #define SPL_KMEM_CACHE_OBJ_PER_SLAB     32      /* Target objects per slab */
 245 #define SPL_KMEM_CACHE_OBJ_PER_SLAB_MIN 8       /* Minimum objects per slab */
 246 #define SPL_KMEM_CACHE_ALIGN            8       /* Default object alignment */
 247
 248 typedef int (*spl_kmem_ctor_t)(void *, void *, int);
 249 typedef void (*spl_kmem_dtor_t)(void *, void *);
 250 typedef void (*spl_kmem_reclaim_t)(void *);
 251
 252 typedef struct spl_kmem_magazine {
 253         uint32_t                skm_magic;      /* Sanity magic */
 254         uint32_t                skm_avail;      /* Available objects */
 255         uint32_t                skm_size;       /* Magazine size */
 256         uint32_t                skm_refill;     /* Batch refill size */
 257         struct spl_kmem_cache   *skm_cache;     /* Owned by cache */
 258         struct delayed_work     skm_work;       /* Magazine reclaim work */
 259         unsigned long           skm_age;        /* Last cache access */
 260         void                    *skm_objs[0];   /* Object pointers */
 261 } spl_kmem_magazine_t;
 262
 263 typedef struct spl_kmem_obj {
 264         uint32_t                sko_magic;      /* Sanity magic */
 265         void                    *sko_addr;      /* Buffer address */
 266         struct spl_kmem_slab    *sko_slab;      /* Owned by slab */
 267         struct list_head        sko_list;       /* Free object list linkage */
 268 } spl_kmem_obj_t;
 269
 270 typedef struct spl_kmem_slab {
 271         uint32_t                sks_magic;      /* Sanity magic */
 272         uint32_t                sks_objs;       /* Objects per slab */
 273         struct spl_kmem_cache   *sks_cache;     /* Owned by cache */
 274         struct list_head        sks_list;       /* Slab list linkage */
 275         struct list_head        sks_free_list;  /* Free object list */
 276         unsigned long           sks_age;        /* Last modify jiffie */
 277         uint32_t                sks_ref;        /* Ref count used objects */
 278 } spl_kmem_slab_t;
 279
 280 typedef struct spl_kmem_cache {
 281         uint32_t                skc_magic;      /* Sanity magic */
 282         uint32_t                skc_name_size;  /* Name length */
 283         char                    *skc_name;      /* Name string */
 284         spl_kmem_magazine_t     *skc_mag[NR_CPUS]; /* Per-CPU warm cache */
 285         uint32_t                skc_mag_size;   /* Magazine size */
 286         uint32_t                skc_mag_refill; /* Magazine refill count */
 287         spl_kmem_ctor_t         skc_ctor;       /* Constructor */
 288         spl_kmem_dtor_t         skc_dtor;       /* Destructor */
 289         spl_kmem_reclaim_t      skc_reclaim;    /* Reclaimator */
 290         void                    *skc_private;   /* Private data */
 291         void                    *skc_vmp;       /* Unused */
 292         unsigned long           skc_flags;      /* Flags */
 293         uint32_t                skc_obj_size;   /* Object size */
 294         uint32_t                skc_obj_align;  /* Object alignment */
 295         uint32_t                skc_slab_objs;  /* Objects per slab */
 296         uint32_t                skc_slab_size;  /* Slab size */
 297         uint32_t                skc_delay;      /* Slab reclaim interval */
 298         uint32_t                skc_reap;       /* Slab reclaim count */
 299         atomic_t                skc_ref;        /* Ref count callers */
 300         struct delayed_work     skc_work;       /* Slab reclaim work */
 301         struct list_head        skc_list;       /* List of caches linkage */
 302         struct list_head        skc_complete_list;/* Completely alloc'ed */
 303         struct list_head        skc_partial_list; /* Partially alloc'ed */
 304         spinlock_t              skc_lock;       /* Cache lock */
 305         uint64_t                skc_slab_fail;  /* Slab alloc failures */
 306         uint64_t                skc_slab_create;/* Slab creates */
 307         uint64_t                skc_slab_destroy;/* Slab destroys */
 308         uint64_t                skc_slab_total; /* Slab total current */
 309         uint64_t                skc_slab_alloc; /* Slab alloc current */
 310         uint64_t                skc_slab_max;   /* Slab max historic  */
 311         uint64_t                skc_obj_total;  /* Obj total current */
 312         uint64_t                skc_obj_alloc;  /* Obj alloc current */
 313         uint64_t                skc_obj_max;    /* Obj max historic */
 314 } spl_kmem_cache_t;
 315 #define kmem_cache_t            spl_kmem_cache_t
 316
 317 extern spl_kmem_cache_t *
 318 spl_kmem_cache_create(char *name, size_t size, size_t align,
 319         spl_kmem_ctor_t ctor, spl_kmem_dtor_t dtor, spl_kmem_reclaim_t reclaim,
 320         void *priv, void *vmp, int flags);
 321
 322 extern void spl_kmem_cache_destroy(spl_kmem_cache_t *skc);
 323 extern void *spl_kmem_cache_alloc(spl_kmem_cache_t *skc, int flags);
 324 extern void spl_kmem_cache_free(spl_kmem_cache_t *skc, void *obj);
 325 extern void spl_kmem_cache_reap_now(spl_kmem_cache_t *skc);
 326 extern void spl_kmem_reap(void);
 327
 328 int spl_kmem_init_kallsyms_lookup(void);
 329 int spl_kmem_init(void);
 330 void spl_kmem_fini(void);
 331
 332 #define kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags) \
 333         spl_kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags)
 334 #define kmem_cache_destroy(skc)         spl_kmem_cache_destroy(skc)
 335 #define kmem_cache_alloc(skc, flags)    spl_kmem_cache_alloc(skc, flags)
 336 #define kmem_cache_free(skc, obj)       spl_kmem_cache_free(skc, obj)
 337 #define kmem_cache_reap_now(skc)        spl_kmem_cache_reap_now(skc)
 338 #define kmem_reap()                     spl_kmem_reap()
 339 #define kmem_virt(ptr)                  (((ptr) >= (void *)VMALLOC_START) && \
 340                                          ((ptr) <  (void *)VMALLOC_END))
 341
 342 #ifdef  __cplusplus
 343 }
 344 #endif
 345
 346 #endif  /* _SPL_KMEM_H */