/*
- * This file is part of the SPL: Solaris Porting Layer.
- *
- * Copyright (c) 2008 Lawrence Livermore National Security, LLC.
- * Produced at Lawrence Livermore National Laboratory
- * Written by:
- * Brian Behlendorf <behlendorf1@llnl.gov>,
- * Herb Wartens <wartens2@llnl.gov>,
- * Jim Garlick <garlick@llnl.gov>
+ * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
+ * Copyright (C) 2007 The Regents of the University of California.
+ * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
+ * Written by Brian Behlendorf <behlendorf1@llnl.gov>.
* UCRL-CODE-235197
*
- * This is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
+ * This file is part of the SPL, Solaris Porting Layer.
+ * For details, see <http://zfsonlinux.org/>.
+ *
+ * The SPL is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
*
- * This is distributed in the hope that it will be useful, but WITHOUT
+ * The SPL is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*
* You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ * with the SPL. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _SPL_KMEM_H
#define _SPL_KMEM_H
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#undef DEBUG_KMEM_UNIMPLEMENTED
-
-#include <linux/module.h>
#include <linux/slab.h>
-#include <linux/vmalloc.h>
-#include <linux/mm.h>
-#include <linux/spinlock.h>
-#include <linux/rwsem.h>
-#include <linux/hash.h>
-#include <linux/ctype.h>
-#include <asm/atomic_compat.h>
-#include <sys/types.h>
-#include <sys/debug.h>
-#include <sys/vmsystm.h>
+
+extern int kmem_debugging(void);
+extern char *kmem_vasprintf(const char *fmt, va_list ap);
+extern char *kmem_asprintf(const char *fmt, ...);
+extern char *strdup(const char *str);
+extern void strfree(char *str);
/*
* Memory allocation interfaces
*/
-#define KM_SLEEP (GFP_KERNEL | __GFP_NOFAIL)
-#define KM_NOSLEEP GFP_ATOMIC
-#undef KM_PANIC /* No linux analog */
-#define KM_PUSHPAGE (KM_SLEEP | __GFP_HIGH)
-#define KM_VMFLAGS GFP_LEVEL_MASK
-#define KM_FLAGS __GFP_BITS_MASK
+#define KM_SLEEP GFP_KERNEL /* Can sleep, never fails */
+#define KM_NOSLEEP GFP_ATOMIC /* Can not sleep, may fail */
+#define KM_PUSHPAGE (GFP_NOIO | __GFP_HIGH) /* Use reserved memory */
+#define KM_NODEBUG __GFP_NOWARN /* Suppress warnings */
+#define KM_FLAGS __GFP_BITS_MASK
+#define KM_VMFLAGS GFP_LEVEL_MASK
/*
* Used internally, the kernel does not need to support this flag
*/
#ifndef __GFP_ZERO
-# define __GFP_ZERO 0x8000
+#define __GFP_ZERO 0x8000
#endif
-#ifdef DEBUG_KMEM
-
-extern atomic64_t kmem_alloc_used;
-extern unsigned long long kmem_alloc_max;
-extern atomic64_t vmem_alloc_used;
-extern unsigned long long vmem_alloc_max;
-
-# define kmem_alloc(size, flags) __kmem_alloc((size), (flags), 0, 0)
-# define kmem_zalloc(size, flags) __kmem_alloc((size), ((flags) | \
- __GFP_ZERO), 0, 0)
-
-/* The node alloc functions are only used by the SPL code itself */
-# ifdef HAVE_KMALLOC_NODE
-# define kmem_alloc_node(size, flags, node) __kmem_alloc((size), (flags), 1, \
- node)
-# else
-# define kmem_alloc_node(size, flags, node) __kmem_alloc((size), (flags), 0, 0)
-# endif
-
-# define vmem_zalloc(size, flags) vmem_alloc((size), ((flags) | \
- __GFP_ZERO))
-
-# ifdef DEBUG_KMEM_TRACKING
-
-extern void *kmem_alloc_track(size_t size, int flags, const char *func,
- int line, int node_alloc, int node);
-extern void kmem_free_track(void *ptr, size_t size);
-extern void *vmem_alloc_track(size_t size, int flags, const char *func,
- int line);
-extern void vmem_free_track(void *ptr, size_t size);
-
-# define __kmem_alloc(size, flags, na, node) kmem_alloc_track((size), \
- (flags), __FUNCTION__, \
- __LINE__, (na), (node))
-# define kmem_free(ptr, size) kmem_free_track((ptr), (size))
-# define vmem_alloc(size, flags) vmem_alloc_track((size), \
- (flags),__FUNCTION__, \
- __LINE__)
-# define vmem_free(ptr, size) vmem_free_track((ptr), (size))
-
-# else /* DEBUG_KMEM_TRACKING */
-
-extern void *kmem_alloc_debug(size_t size, int flags, const char *func,
- int line, int node_alloc, int node);
-extern void kmem_free_debug(void *ptr, size_t size);
-extern void *vmem_alloc_debug(size_t size, int flags, const char *func,
- int line);
-extern void vmem_free_debug(void *ptr, size_t size);
-
-# define __kmem_alloc(size, flags, na, node) kmem_alloc_debug((size), \
- (flags), __FUNCTION__, \
- __LINE__, (na), (node))
-# define kmem_free(ptr, size) kmem_free_debug((ptr), (size))
-# define vmem_alloc(size, flags) vmem_alloc_debug((size), \
- (flags), __FUNCTION__, \
- __LINE__)
-# define vmem_free(ptr, size) vmem_free_debug((ptr), (size))
-
-# endif /* DEBUG_KMEM_TRACKING */
-
-#else /* DEBUG_KMEM */
-
-# define kmem_alloc(size, flags) kmalloc((size), (flags))
-# define kmem_zalloc(size, flags) kzalloc((size), (flags))
-# define kmem_free(ptr, size) ((void)(size), kfree(ptr))
-
-# ifdef HAVE_KMALLOC_NODE
-# define kmem_alloc_node(size, flags, node) \
- kmalloc_node((size), (flags), (node))
-# else
-# define kmem_alloc_node(size, flags, node) \
- kmalloc((size), (flags))
-# endif
-
-# define vmem_alloc(size, flags) __vmalloc((size), ((flags) | \
- __GFP_HIGHMEM), PAGE_KERNEL)
-# define vmem_zalloc(size, flags) \
-({ \
- void *_ptr_ = __vmalloc((size),((flags)|__GFP_HIGHMEM),PAGE_KERNEL); \
- if (_ptr_) \
- memset(_ptr_, 0, (size)); \
- _ptr_; \
-})
-# define vmem_free(ptr, size) ((void)(size), vfree(ptr))
-
-#endif /* DEBUG_KMEM */
-
-#ifdef DEBUG_KMEM_UNIMPLEMENTED
-static __inline__ void *
-kmem_alloc_tryhard(size_t size, size_t *alloc_size, int kmflags)
-{
-#error "kmem_alloc_tryhard() not implemented"
-}
-#endif /* DEBUG_KMEM_UNIMPLEMENTED */
-
/*
- * Slab allocation interfaces
+ * __GFP_NOFAIL looks like it will be removed from the kernel perhaps as
+ * early as 2.6.32. To avoid this issue when it occurs in upstream kernels
+ * we retry the allocation here as long as it is not __GFP_WAIT (GFP_ATOMIC).
+ * I would prefer the caller handle the failure case cleanly but we are
+ * trying to emulate Solaris and those are not the Solaris semantics.
*/
-enum {
- KMC_BIT_NOTOUCH = 0, /* Don't update ages */
- KMC_BIT_NODEBUG = 1, /* Default behavior */
- KMC_BIT_NOMAGAZINE = 2, /* XXX: Unsupported */
- KMC_BIT_NOHASH = 3, /* XXX: Unsupported */
- KMC_BIT_QCACHE = 4, /* XXX: Unsupported */
- KMC_BIT_KMEM = 5, /* Use kmem cache */
- KMC_BIT_VMEM = 6, /* Use vmem cache */
- KMC_BIT_OFFSLAB = 7, /* Objects not on slab */
- KMC_BIT_REAPING = 16, /* Reaping in progress */
- KMC_BIT_DESTROY = 17, /* Destroy in progress */
-};
-
-#define KMC_NOTOUCH (1 << KMC_BIT_NOTOUCH)
-#define KMC_NODEBUG (1 << KMC_BIT_NODEBUG)
-#define KMC_NOMAGAZINE (1 << KMC_BIT_NOMAGAZINE)
-#define KMC_NOHASH (1 << KMC_BIT_NOHASH)
-#define KMC_QCACHE (1 << KMC_BIT_QCACHE)
-#define KMC_KMEM (1 << KMC_BIT_KMEM)
-#define KMC_VMEM (1 << KMC_BIT_VMEM)
-#define KMC_OFFSLAB (1 << KMC_BIT_OFFSLAB)
-#define KMC_REAPING (1 << KMC_BIT_REAPING)
-#define KMC_DESTROY (1 << KMC_BIT_DESTROY)
-
-#define KMC_REAP_CHUNK INT_MAX
-#define KMC_DEFAULT_SEEKS 1
-
-#ifdef DEBUG_KMEM_UNIMPLEMENTED
-static __inline__ void kmem_init(void) {
-#error "kmem_init() not implemented"
-}
+static inline void *
+kmalloc_nofail(size_t size, gfp_t flags)
+{
+ void *ptr;
-static __inline__ void kmem_thread_init(void) {
-#error "kmem_thread_init() not implemented"
-}
+ do {
+ ptr = kmalloc(size, flags);
+ } while (ptr == NULL && (flags & __GFP_WAIT));
-static __inline__ void kmem_mp_init(void) {
-#error "kmem_mp_init() not implemented"
+ return (ptr);
}
-static __inline__ void kmem_reap_idspace(void) {
-#error "kmem_reap_idspace() not implemented"
-}
-
-static __inline__ size_t kmem_avail(void) {
-#error "kmem_avail() not implemented"
-}
+static inline void *
+kzalloc_nofail(size_t size, gfp_t flags)
+{
+ void *ptr;
-static __inline__ size_t kmem_maxavail(void) {
-#error "kmem_maxavail() not implemented"
-}
+ do {
+ ptr = kzalloc(size, flags);
+ } while (ptr == NULL && (flags & __GFP_WAIT));
-static __inline__ uint64_t kmem_cache_stat(spl_kmem_cache_t *cache) {
-#error "kmem_cache_stat() not implemented"
+ return (ptr);
}
-#endif /* DEBUG_KMEM_UNIMPLEMENTED */
-/* XXX - Used by arc.c to adjust its memory footprint. We may want
- * to use this hook in the future to adjust behavior based on
- * debug levels. For now it's safe to always return 0.
- */
-static __inline__ int
-kmem_debugging(void)
+static inline void *
+kmalloc_node_nofail(size_t size, gfp_t flags, int node)
{
- return 0;
-}
+ void *ptr;
-extern int kmem_set_warning(int flag);
+ do {
+ ptr = kmalloc_node(size, flags, node);
+ } while (ptr == NULL && (flags & __GFP_WAIT));
-extern struct list_head spl_kmem_cache_list;
-extern struct rw_semaphore spl_kmem_cache_sem;
-
-#define SKM_MAGIC 0x2e2e2e2e
-#define SKO_MAGIC 0x20202020
-#define SKS_MAGIC 0x22222222
-#define SKC_MAGIC 0x2c2c2c2c
-
-#define SPL_KMEM_CACHE_DELAY 15 /* Minimum slab release age */
-#define SPL_KMEM_CACHE_REAP 0 /* Default reap everything */
-#define SPL_KMEM_CACHE_OBJ_PER_SLAB 32 /* Target objects per slab */
-#define SPL_KMEM_CACHE_OBJ_PER_SLAB_MIN 8 /* Minimum objects per slab */
-#define SPL_KMEM_CACHE_ALIGN 8 /* Default object alignment */
-
-typedef int (*spl_kmem_ctor_t)(void *, void *, int);
-typedef void (*spl_kmem_dtor_t)(void *, void *);
-typedef void (*spl_kmem_reclaim_t)(void *);
-
-typedef struct spl_kmem_magazine {
- uint32_t skm_magic; /* Sanity magic */
- uint32_t skm_avail; /* Available objects */
- uint32_t skm_size; /* Magazine size */
- uint32_t skm_refill; /* Batch refill size */
- struct spl_kmem_cache *skm_cache; /* Owned by cache */
- struct delayed_work skm_work; /* Magazine reclaim work */
- unsigned long skm_age; /* Last cache access */
- void *skm_objs[0]; /* Object pointers */
-} spl_kmem_magazine_t;
-
-typedef struct spl_kmem_obj {
- uint32_t sko_magic; /* Sanity magic */
- void *sko_addr; /* Buffer address */
- struct spl_kmem_slab *sko_slab; /* Owned by slab */
- struct list_head sko_list; /* Free object list linkage */
-} spl_kmem_obj_t;
+ return (ptr);
+}
-typedef struct spl_kmem_slab {
- uint32_t sks_magic; /* Sanity magic */
- uint32_t sks_objs; /* Objects per slab */
- struct spl_kmem_cache *sks_cache; /* Owned by cache */
- struct list_head sks_list; /* Slab list linkage */
- struct list_head sks_free_list; /* Free object list */
- unsigned long sks_age; /* Last modify jiffie */
- uint32_t sks_ref; /* Ref count used objects */
-} spl_kmem_slab_t;
+#ifdef DEBUG_KMEM
-typedef struct spl_kmem_cache {
- uint32_t skc_magic; /* Sanity magic */
- uint32_t skc_name_size; /* Name length */
- char *skc_name; /* Name string */
- spl_kmem_magazine_t *skc_mag[NR_CPUS]; /* Per-CPU warm cache */
- uint32_t skc_mag_size; /* Magazine size */
- uint32_t skc_mag_refill; /* Magazine refill count */
- spl_kmem_ctor_t skc_ctor; /* Constructor */
- spl_kmem_dtor_t skc_dtor; /* Destructor */
- spl_kmem_reclaim_t skc_reclaim; /* Reclaimator */
- void *skc_private; /* Private data */
- void *skc_vmp; /* Unused */
- unsigned long skc_flags; /* Flags */
- uint32_t skc_obj_size; /* Object size */
- uint32_t skc_obj_align; /* Object alignment */
- uint32_t skc_slab_objs; /* Objects per slab */
- uint32_t skc_slab_size; /* Slab size */
- uint32_t skc_delay; /* Slab reclaim interval */
- uint32_t skc_reap; /* Slab reclaim count */
- atomic_t skc_ref; /* Ref count callers */
- struct delayed_work skc_work; /* Slab reclaim work */
- struct list_head skc_list; /* List of caches linkage */
- struct list_head skc_complete_list;/* Completely alloc'ed */
- struct list_head skc_partial_list; /* Partially alloc'ed */
- spinlock_t skc_lock; /* Cache lock */
- uint64_t skc_slab_fail; /* Slab alloc failures */
- uint64_t skc_slab_create;/* Slab creates */
- uint64_t skc_slab_destroy;/* Slab destroys */
- uint64_t skc_slab_total; /* Slab total current */
- uint64_t skc_slab_alloc; /* Slab alloc current */
- uint64_t skc_slab_max; /* Slab max historic */
- uint64_t skc_obj_total; /* Obj total current */
- uint64_t skc_obj_alloc; /* Obj alloc current */
- uint64_t skc_obj_max; /* Obj max historic */
-} spl_kmem_cache_t;
-#define kmem_cache_t spl_kmem_cache_t
+/*
+ * Memory accounting functions to be used only when DEBUG_KMEM is set.
+ */
+#ifdef HAVE_ATOMIC64_T
+#define kmem_alloc_used_add(size) atomic64_add(size, &kmem_alloc_used)
+#define kmem_alloc_used_sub(size) atomic64_sub(size, &kmem_alloc_used)
+#define kmem_alloc_used_read() atomic64_read(&kmem_alloc_used)
+#define kmem_alloc_used_set(size) atomic64_set(&kmem_alloc_used, size)
+extern atomic64_t kmem_alloc_used;
+extern unsigned long long kmem_alloc_max;
+#else /* HAVE_ATOMIC64_T */
+#define kmem_alloc_used_add(size) atomic_add(size, &kmem_alloc_used)
+#define kmem_alloc_used_sub(size) atomic_sub(size, &kmem_alloc_used)
+#define kmem_alloc_used_read() atomic_read(&kmem_alloc_used)
+#define kmem_alloc_used_set(size) atomic_set(&kmem_alloc_used, size)
+extern atomic_t kmem_alloc_used;
+extern unsigned long long kmem_alloc_max;
+#endif /* HAVE_ATOMIC64_T */
-extern spl_kmem_cache_t *
-spl_kmem_cache_create(char *name, size_t size, size_t align,
- spl_kmem_ctor_t ctor, spl_kmem_dtor_t dtor, spl_kmem_reclaim_t reclaim,
- void *priv, void *vmp, int flags);
+#ifdef DEBUG_KMEM_TRACKING
+/*
+ * DEBUG_KMEM && DEBUG_KMEM_TRACKING
+ *
+ * The maximum level of memory debugging. All memory will be accounted
+ * for and each allocation will be explicitly tracked. Any allocation
+ * which is leaked will be reported on module unload and the exact location
+ * where that memory was allocation will be reported. This level of memory
+ * tracking will have a significant impact on performance and should only
+ * be enabled for debugging. This feature may be enabled by passing
+ * --enable-debug-kmem-tracking to configure.
+ */
+#define kmem_alloc(sz, fl) kmem_alloc_track((sz), (fl), \
+ __FUNCTION__, __LINE__, 0, 0)
+#define kmem_zalloc(sz, fl) kmem_alloc_track((sz), (fl)|__GFP_ZERO,\
+ __FUNCTION__, __LINE__, 0, 0)
+#define kmem_alloc_node(sz, fl, nd) kmem_alloc_track((sz), (fl), \
+ __FUNCTION__, __LINE__, 1, nd)
+#define kmem_free(ptr, sz) kmem_free_track((ptr), (sz))
+
+extern void *kmem_alloc_track(size_t, int, const char *, int, int, int);
+extern void kmem_free_track(const void *, size_t);
+
+#else /* DEBUG_KMEM_TRACKING */
+/*
+ * DEBUG_KMEM && !DEBUG_KMEM_TRACKING
+ *
+ * The default build will set DEBUG_KEM. This provides basic memory
+ * accounting with little to no impact on performance. When the module
+ * is unloaded in any memory was leaked the total number of leaked bytes
+ * will be reported on the console. To disable this basic accounting
+ * pass the --disable-debug-kmem option to configure.
+ */
+#define kmem_alloc(sz, fl) kmem_alloc_debug((sz), (fl), \
+ __FUNCTION__, __LINE__, 0, 0)
+#define kmem_zalloc(sz, fl) kmem_alloc_debug((sz), (fl)|__GFP_ZERO,\
+ __FUNCTION__, __LINE__, 0, 0)
+#define kmem_alloc_node(sz, fl, nd) kmem_alloc_debug((sz), (fl), \
+ __FUNCTION__, __LINE__, 1, nd)
+#define kmem_free(ptr, sz) kmem_free_debug((ptr), (sz))
+
+extern void *kmem_alloc_debug(size_t, int, const char *, int, int, int);
+extern void kmem_free_debug(const void *, size_t);
+
+#endif /* DEBUG_KMEM_TRACKING */
+#else /* DEBUG_KMEM */
+/*
+ * !DEBUG_KMEM && !DEBUG_KMEM_TRACKING
+ *
+ * All debugging is disabled. There will be no overhead even for
+ * minimal memory accounting. To enable basic accounting pass the
+ * --enable-debug-kmem option to configure.
+ */
+#define kmem_alloc(sz, fl) kmalloc_nofail((sz), (fl))
+#define kmem_zalloc(sz, fl) kzalloc_nofail((sz), (fl))
+#define kmem_alloc_node(sz, fl, nd) kmalloc_node_nofail((sz), (fl), (nd))
+#define kmem_free(ptr, sz) ((void)(sz), kfree(ptr))
-extern void spl_kmem_cache_destroy(spl_kmem_cache_t *skc);
-extern void *spl_kmem_cache_alloc(spl_kmem_cache_t *skc, int flags);
-extern void spl_kmem_cache_free(spl_kmem_cache_t *skc, void *obj);
-extern void spl_kmem_cache_reap_now(spl_kmem_cache_t *skc);
-extern void spl_kmem_reap(void);
+#endif /* DEBUG_KMEM */
-int spl_kmem_init_kallsyms_lookup(void);
int spl_kmem_init(void);
void spl_kmem_fini(void);
-#define kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags) \
- spl_kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags)
-#define kmem_cache_destroy(skc) spl_kmem_cache_destroy(skc)
-#define kmem_cache_alloc(skc, flags) spl_kmem_cache_alloc(skc, flags)
-#define kmem_cache_free(skc, obj) spl_kmem_cache_free(skc, obj)
-#define kmem_cache_reap_now(skc) spl_kmem_cache_reap_now(skc)
-#define kmem_reap() spl_kmem_reap()
-#define kmem_virt(ptr) (((ptr) >= (void *)VMALLOC_START) && \
- ((ptr) < (void *)VMALLOC_END))
-
-#ifdef __cplusplus
-}
-#endif
+#define kmem_virt(ptr) (((ptr) >= (void *)VMALLOC_START) && \
+ ((ptr) < (void *)VMALLOC_END))
#endif /* _SPL_KMEM_H */
projects / mirror_spl.git / blobdiff