[mirror_spl-debian.git] / include / sys / kmem.h

/*****************************************************************************\
 *  Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
 *  Copyright (C) 2007 The Regents of the University of California.
 *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
 *  Written by Brian Behlendorf <behlendorf1@llnl.gov>.
 *  UCRL-CODE-235197
 *
 *  This file is part of the SPL, Solaris Porting Layer.
 *  For details, see <http://github.com/behlendorf/spl/>.
 *
 *  The SPL is free software; you can redistribute it and/or modify it
 *  under the terms of the GNU General Public License as published by the
 *  Free Software Foundation; either version 2 of the License, or (at your
 *  option) any later version.
 *
 *  The SPL is distributed in the hope that it will be useful, but WITHOUT
 *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 *  for more details.
 *
 *  You should have received a copy of the GNU General Public License along
 *  with the SPL.  If not, see <http://www.gnu.org/licenses/>.
\*****************************************************************************/

#ifndef _SPL_KMEM_H
#define	_SPL_KMEM_H

#undef DEBUG_KMEM_UNIMPLEMENTED

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/mm_compat.h>
#include <linux/spinlock.h>
#include <linux/rwsem.h>
#include <linux/hash.h>
#include <linux/ctype.h>
#include <asm/atomic.h>
#include <sys/types.h>
#include <sys/debug.h>
#include <sys/vmsystm.h>

/*
 * Memory allocation interfaces
 */
#define KM_SLEEP                        GFP_KERNEL
#define KM_NOSLEEP                      GFP_ATOMIC
#undef  KM_PANIC                        /* No linux analog */
#define KM_PUSHPAGE                     (KM_SLEEP | __GFP_HIGH)
#define KM_VMFLAGS                      GFP_LEVEL_MASK
#define KM_FLAGS                        __GFP_BITS_MASK
#define KM_NODEBUG                      __GFP_NOWARN

/*
 * Used internally, the kernel does not need to support this flag
 */
#ifndef __GFP_ZERO
# define __GFP_ZERO                     0x8000
#endif

/*
 * __GFP_NOFAIL looks like it will be removed from the kernel perhaps as
 * early as 2.6.32.  To avoid this issue when it occurs in upstream kernels
 * we retry the allocation here as long as it is not __GFP_WAIT (GFP_ATOMIC).
 * I would prefer the caller handle the failure case cleanly but we are
 * trying to emulate Solaris and those are not the Solaris semantics.
 */
static inline void *
kmalloc_nofail(size_t size, gfp_t flags)
{
	void *ptr;

	do {
		ptr = kmalloc(size, flags);
	} while (ptr == NULL && (flags & __GFP_WAIT));

	return ptr;
}

static inline void *
kzalloc_nofail(size_t size, gfp_t flags)
{
	void *ptr;

	do {
		ptr = kzalloc(size, flags);
	} while (ptr == NULL && (flags & __GFP_WAIT));

	return ptr;
}

#ifdef HAVE_KMALLOC_NODE
static inline void *
kmalloc_node_nofail(size_t size, gfp_t flags, int node)
{
	void *ptr;

	do {
		ptr = kmalloc_node(size, flags, node);
	} while (ptr == NULL && (flags & __GFP_WAIT));

	return ptr;
}
#endif /* HAVE_KMALLOC_NODE */

#ifdef DEBUG_KMEM
# ifdef HAVE_ATOMIC64_T

extern atomic64_t kmem_alloc_used;
extern unsigned long long kmem_alloc_max;
extern atomic64_t vmem_alloc_used;
extern unsigned long long vmem_alloc_max;

# define kmem_alloc_used_add(size)      atomic64_add(size, &kmem_alloc_used)
# define kmem_alloc_used_sub(size)      atomic64_sub(size, &kmem_alloc_used)
# define kmem_alloc_used_read()         atomic64_read(&kmem_alloc_used)
# define kmem_alloc_used_set(size)      atomic64_set(&kmem_alloc_used, size)
# define vmem_alloc_used_add(size)      atomic64_add(size, &vmem_alloc_used)
# define vmem_alloc_used_sub(size)      atomic64_sub(size, &vmem_alloc_used)
# define vmem_alloc_used_read()         atomic64_read(&vmem_alloc_used)
# define vmem_alloc_used_set(size)      atomic64_set(&vmem_alloc_used, size)

# else

extern atomic_t kmem_alloc_used;
extern unsigned long long kmem_alloc_max;
extern atomic_t vmem_alloc_used;
extern unsigned long long vmem_alloc_max;

# define kmem_alloc_used_add(size)      atomic_add(size, &kmem_alloc_used)
# define kmem_alloc_used_sub(size)      atomic_sub(size, &kmem_alloc_used)
# define kmem_alloc_used_read()         atomic_read(&kmem_alloc_used)
# define kmem_alloc_used_set(size)      atomic_set(&kmem_alloc_used, size)
# define vmem_alloc_used_add(size)      atomic_add(size, &vmem_alloc_used)
# define vmem_alloc_used_sub(size)      atomic_sub(size, &vmem_alloc_used)
# define vmem_alloc_used_read()         atomic_read(&vmem_alloc_used)
# define vmem_alloc_used_set(size)      atomic_set(&vmem_alloc_used, size)

# endif /* _LP64 */

# define kmem_alloc(size, flags)             __kmem_alloc((size), (flags), 0, 0)
# define kmem_zalloc(size, flags)            __kmem_alloc((size), ((flags) |  \
                                                 __GFP_ZERO), 0, 0)

/* The node alloc functions are only used by the SPL code itself */
# ifdef HAVE_KMALLOC_NODE
#  define kmem_alloc_node(size, flags, node) __kmem_alloc((size), (flags), 1, \
                                                 node)
# else
#  define kmem_alloc_node(size, flags, node) __kmem_alloc((size), (flags), 0, 0)
# endif

# define vmem_zalloc(size, flags)            vmem_alloc((size), ((flags) |    \
                                                 __GFP_ZERO))

# ifdef DEBUG_KMEM_TRACKING

extern void *kmem_alloc_track(size_t size, int flags, const char *func,
    int line, int node_alloc, int node);
extern void kmem_free_track(void *ptr, size_t size);
extern void *vmem_alloc_track(size_t size, int flags, const char *func,
    int line);
extern void vmem_free_track(void *ptr, size_t size);

#  define __kmem_alloc(size, flags, na, node) kmem_alloc_track((size),        \
                                                  (flags), __FUNCTION__,      \
                                                  __LINE__, (na), (node))
#  define kmem_free(ptr, size)                kmem_free_track((ptr), (size))
#  define vmem_alloc(size, flags)             vmem_alloc_track((size),        \
                                                  (flags),__FUNCTION__,       \
                                                  __LINE__)
#  define vmem_free(ptr, size)                vmem_free_track((ptr), (size))

# else /* DEBUG_KMEM_TRACKING */

extern void *kmem_alloc_debug(size_t size, int flags, const char *func,
    int line, int node_alloc, int node);
extern void kmem_free_debug(void *ptr, size_t size);
extern void *vmem_alloc_debug(size_t size, int flags, const char *func,
    int line);
extern void vmem_free_debug(void *ptr, size_t size);

#  define __kmem_alloc(size, flags, na, node) kmem_alloc_debug((size),        \
                                                  (flags), __FUNCTION__,      \
                                                  __LINE__, (na), (node))
#  define kmem_free(ptr, size)                kmem_free_debug((ptr), (size))
#  define vmem_alloc(size, flags)             vmem_alloc_debug((size),        \
                                                  (flags), __FUNCTION__,      \
                                                  __LINE__)
#  define vmem_free(ptr, size)                vmem_free_debug((ptr), (size))

# endif /* DEBUG_KMEM_TRACKING */

#else /* DEBUG_KMEM */

# define kmem_alloc(size, flags)              kmalloc_nofail((size), (flags))
# define kmem_zalloc(size, flags)             kzalloc_nofail((size), (flags))
# define kmem_free(ptr, size)                 ((void)(size), kfree(ptr))

# ifdef HAVE_KMALLOC_NODE
#  define kmem_alloc_node(size, flags, node)                                  \
          kmalloc_node_nofail((size), (flags), (node))
# else
#  define kmem_alloc_node(size, flags, node)                                  \
          kmalloc_nofail((size), (flags))
# endif

# define vmem_alloc(size, flags)              __vmalloc((size), ((flags) |    \
                                                  __GFP_HIGHMEM), PAGE_KERNEL)
# define vmem_zalloc(size, flags)                                             \
({                                                                            \
        void *_ptr_ = __vmalloc((size),((flags)|__GFP_HIGHMEM),PAGE_KERNEL);  \
        if (_ptr_)                                                            \
                memset(_ptr_, 0, (size));                                     \
        _ptr_;                                                                \
})
# define vmem_free(ptr, size)           ((void)(size), vfree(ptr))

#endif /* DEBUG_KMEM */

#ifdef DEBUG_KMEM_UNIMPLEMENTED
static __inline__ void *
kmem_alloc_tryhard(size_t size, size_t *alloc_size, int kmflags)
{
#error "kmem_alloc_tryhard() not implemented"
}
#endif /* DEBUG_KMEM_UNIMPLEMENTED */

/*
 * Slab allocation interfaces
 */
enum {
	KMC_BIT_NOTOUCH		= 0,	/* Don't update ages */
	KMC_BIT_NODEBUG		= 1,	/* Default behavior */
	KMC_BIT_NOMAGAZINE	= 2,	/* XXX: Unsupported */
	KMC_BIT_NOHASH		= 3,	/* XXX: Unsupported */
	KMC_BIT_QCACHE		= 4,	/* XXX: Unsupported */
	KMC_BIT_KMEM		= 5,	/* Use kmem cache */
	KMC_BIT_VMEM		= 6,	/* Use vmem cache */
	KMC_BIT_OFFSLAB		= 7,	/* Objects not on slab */
	KMC_BIT_REAPING		= 16,	/* Reaping in progress */
	KMC_BIT_DESTROY		= 17,	/* Destroy in progress */
};

#define KMC_NOTOUCH		(1 << KMC_BIT_NOTOUCH)
#define KMC_NODEBUG		(1 << KMC_BIT_NODEBUG)
#define KMC_NOMAGAZINE		(1 << KMC_BIT_NOMAGAZINE)
#define KMC_NOHASH		(1 << KMC_BIT_NOHASH)
#define KMC_QCACHE		(1 << KMC_BIT_QCACHE)
#define KMC_KMEM		(1 << KMC_BIT_KMEM)
#define KMC_VMEM		(1 << KMC_BIT_VMEM)
#define KMC_OFFSLAB		(1 << KMC_BIT_OFFSLAB)
#define KMC_REAPING		(1 << KMC_BIT_REAPING)
#define KMC_DESTROY		(1 << KMC_BIT_DESTROY)

#define KMC_REAP_CHUNK			INT_MAX
#define KMC_DEFAULT_SEEKS		1

#ifdef DEBUG_KMEM_UNIMPLEMENTED
static __inline__ void kmem_init(void) {
#error "kmem_init() not implemented"
}

static __inline__ void kmem_thread_init(void) {
#error "kmem_thread_init() not implemented"
}

static __inline__ void kmem_mp_init(void) {
#error "kmem_mp_init() not implemented"
}

static __inline__ void kmem_reap_idspace(void) {
#error "kmem_reap_idspace() not implemented"
}

static __inline__ size_t kmem_avail(void) {
#error "kmem_avail() not implemented"
}

static __inline__ size_t kmem_maxavail(void) {
#error "kmem_maxavail() not implemented"
}

static __inline__ uint64_t kmem_cache_stat(spl_kmem_cache_t *cache) {
#error "kmem_cache_stat() not implemented"
}
#endif /* DEBUG_KMEM_UNIMPLEMENTED */

/* XXX - Used by arc.c to adjust its memory footprint. We may want
 *       to use this hook in the future to adjust behavior based on
 *       debug levels.  For now it's safe to always return 0.
 */
static __inline__ int
kmem_debugging(void)
{
        return 0;
}

extern int kmem_set_warning(int flag);

extern struct list_head spl_kmem_cache_list;
extern struct rw_semaphore spl_kmem_cache_sem;

#define SKM_MAGIC			0x2e2e2e2e
#define SKO_MAGIC			0x20202020
#define SKS_MAGIC			0x22222222
#define SKC_MAGIC			0x2c2c2c2c

#define SPL_KMEM_CACHE_DELAY		15	/* Minimum slab release age */
#define SPL_KMEM_CACHE_REAP		0	/* Default reap everything */
#define SPL_KMEM_CACHE_OBJ_PER_SLAB	32	/* Target objects per slab */
#define SPL_KMEM_CACHE_OBJ_PER_SLAB_MIN	8	/* Minimum objects per slab */
#define SPL_KMEM_CACHE_ALIGN		8	/* Default object alignment */

typedef int (*spl_kmem_ctor_t)(void *, void *, int);
typedef void (*spl_kmem_dtor_t)(void *, void *);
typedef void (*spl_kmem_reclaim_t)(void *);

typedef struct spl_kmem_magazine {
	uint32_t		skm_magic;	/* Sanity magic */
	uint32_t		skm_avail;	/* Available objects */
	uint32_t		skm_size;	/* Magazine size */
	uint32_t		skm_refill;	/* Batch refill size */
	struct spl_kmem_cache	*skm_cache;	/* Owned by cache */
	struct delayed_work	skm_work;	/* Magazine reclaim work */
	unsigned long		skm_age;	/* Last cache access */
	void			*skm_objs[0];	/* Object pointers */
} spl_kmem_magazine_t;

typedef struct spl_kmem_obj {
        uint32_t		sko_magic;	/* Sanity magic */
	void			*sko_addr;	/* Buffer address */
	struct spl_kmem_slab	*sko_slab;	/* Owned by slab */
	struct list_head	sko_list;	/* Free object list linkage */
} spl_kmem_obj_t;

typedef struct spl_kmem_slab {
        uint32_t		sks_magic;	/* Sanity magic */
	uint32_t		sks_objs;	/* Objects per slab */
	struct spl_kmem_cache	*sks_cache;	/* Owned by cache */
	struct list_head	sks_list;	/* Slab list linkage */
	struct list_head	sks_free_list;	/* Free object list */
	unsigned long		sks_age;	/* Last modify jiffie */
	uint32_t		sks_ref;	/* Ref count used objects */
} spl_kmem_slab_t;

typedef struct spl_kmem_cache {
	uint32_t		skc_magic;	/* Sanity magic */
	uint32_t		skc_name_size;	/* Name length */
	char			*skc_name;	/* Name string */
	spl_kmem_magazine_t	*skc_mag[NR_CPUS]; /* Per-CPU warm cache */
	uint32_t		skc_mag_size;	/* Magazine size */
	uint32_t		skc_mag_refill;	/* Magazine refill count */
	spl_kmem_ctor_t		skc_ctor;	/* Constructor */
	spl_kmem_dtor_t		skc_dtor;	/* Destructor */
	spl_kmem_reclaim_t	skc_reclaim;	/* Reclaimator */
	void			*skc_private;	/* Private data */
	void			*skc_vmp;	/* Unused */
	unsigned long		skc_flags;	/* Flags */
	uint32_t		skc_obj_size;	/* Object size */
	uint32_t		skc_obj_align;	/* Object alignment */
	uint32_t		skc_slab_objs;	/* Objects per slab */
	uint32_t		skc_slab_size;	/* Slab size */
	uint32_t		skc_delay;	/* Slab reclaim interval */
	uint32_t		skc_reap;	/* Slab reclaim count */
	atomic_t		skc_ref;	/* Ref count callers */
	struct delayed_work	skc_work;	/* Slab reclaim work */
	struct list_head	skc_list;	/* List of caches linkage */
	struct list_head	skc_complete_list;/* Completely alloc'ed */
	struct list_head	skc_partial_list; /* Partially alloc'ed */
	spinlock_t		skc_lock;	/* Cache lock */
	uint64_t		skc_slab_fail;	/* Slab alloc failures */
	uint64_t		skc_slab_create;/* Slab creates */
	uint64_t		skc_slab_destroy;/* Slab destroys */
	uint64_t		skc_slab_total;	/* Slab total current */
	uint64_t		skc_slab_alloc;	/* Slab alloc current */
	uint64_t		skc_slab_max;	/* Slab max historic  */
	uint64_t		skc_obj_total;	/* Obj total current */
	uint64_t		skc_obj_alloc;	/* Obj alloc current */
	uint64_t		skc_obj_max;	/* Obj max historic */
} spl_kmem_cache_t;
#define kmem_cache_t		spl_kmem_cache_t

extern spl_kmem_cache_t *
spl_kmem_cache_create(char *name, size_t size, size_t align,
        spl_kmem_ctor_t ctor, spl_kmem_dtor_t dtor, spl_kmem_reclaim_t reclaim,
        void *priv, void *vmp, int flags);

extern void spl_kmem_cache_destroy(spl_kmem_cache_t *skc);
extern void *spl_kmem_cache_alloc(spl_kmem_cache_t *skc, int flags);
extern void spl_kmem_cache_free(spl_kmem_cache_t *skc, void *obj);
extern void spl_kmem_cache_reap_now(spl_kmem_cache_t *skc);
extern void spl_kmem_reap(void);

int spl_kmem_init_kallsyms_lookup(void);
int spl_kmem_init(void);
void spl_kmem_fini(void);

#define kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags) \
        spl_kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags)
#define kmem_cache_destroy(skc)		spl_kmem_cache_destroy(skc)
#define kmem_cache_alloc(skc, flags)	spl_kmem_cache_alloc(skc, flags)
#define kmem_cache_free(skc, obj)	spl_kmem_cache_free(skc, obj)
#define kmem_cache_reap_now(skc)	spl_kmem_cache_reap_now(skc)
#define kmem_reap()			spl_kmem_reap()
#define kmem_virt(ptr)			(((ptr) >= (void *)VMALLOC_START) && \
					 ((ptr) <  (void *)VMALLOC_END))

#endif	/* _SPL_KMEM_H */
Commit	Line	Data
716154c5 BB	1	/*****************************************************************************\
	2	* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
	3	* Copyright (C) 2007 The Regents of the University of California.
	4	* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
	5	* Written by Brian Behlendorf <behlendorf1@llnl.gov>.
715f6251	6	* UCRL-CODE-235197
715f6251	7	*
716154c5 BB	8	* This file is part of the SPL, Solaris Porting Layer.
	9	* For details, see <http://github.com/behlendorf/spl/>.
	10	*
	11	* The SPL is free software; you can redistribute it and/or modify it
	12	* under the terms of the GNU General Public License as published by the
	13	* Free Software Foundation; either version 2 of the License, or (at your
	14	* option) any later version.
715f6251	15	*
716154c5	16	* The SPL is distributed in the hope that it will be useful, but WITHOUT
715f6251	17	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	18	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	19	* for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License along
716154c5 BB	22	* with the SPL. If not, see <http://www.gnu.org/licenses/>.
716154c5 BB	23	\*****************************************************************************/
715f6251	24
09b414e8	25	#ifndef _SPL_KMEM_H
09b414e8	26	#define _SPL_KMEM_H
f1ca4da6	27
f1ca4da6	28	#undef DEBUG_KMEM_UNIMPLEMENTED
f1ca4da6	29
f1b59d26	30	#include <linux/module.h>
f1ca4da6	31	#include <linux/slab.h>
79b31f36	32	#include <linux/vmalloc.h>
baf2979e	33	#include <linux/mm_compat.h>
f1ca4da6	34	#include <linux/spinlock.h>
d6a26c6a	35	#include <linux/rwsem.h>
	36	#include <linux/hash.h>
	37	#include <linux/ctype.h>
d04c8a56	38	#include <asm/atomic.h>
57d86234	39	#include <sys/types.h>
937879f1	40	#include <sys/debug.h>
36b313da	41	#include <sys/vmsystm.h>
550f1705	42
f1ca4da6	43	/*
	44	* Memory allocation interfaces
	45	*/
c89fdee4	46	#define KM_SLEEP GFP_KERNEL
f1ca4da6	47	#define KM_NOSLEEP GFP_ATOMIC
f1ca4da6	48	#undef KM_PANIC /* No linux analog */
a0f6da3d	49	#define KM_PUSHPAGE (KM_SLEEP \| __GFP_HIGH)
f1ca4da6	50	#define KM_VMFLAGS GFP_LEVEL_MASK
f1ca4da6	51	#define KM_FLAGS __GFP_BITS_MASK
23d91792	52	#define KM_NODEBUG __GFP_NOWARN
f1ca4da6	53
3d061e9d	54	/*
	55	* Used internally, the kernel does not need to support this flag
	56	*/
	57	#ifndef __GFP_ZERO
a0f6da3d	58	# define __GFP_ZERO 0x8000
3d061e9d	59	#endif
3d061e9d	60
c89fdee4 BB	61	/*
	62	* __GFP_NOFAIL looks like it will be removed from the kernel perhaps as
	63	* early as 2.6.32. To avoid this issue when it occurs in upstream kernels
	64	* we retry the allocation here as long as it is not __GFP_WAIT (GFP_ATOMIC).
	65	* I would prefer the caller handle the failure case cleanly but we are
	66	* trying to emulate Solaris and those are not the Solaris semantics.
	67	*/
	68	static inline void *
	69	kmalloc_nofail(size_t size, gfp_t flags)
	70	{
	71	void *ptr;
	72
	73	do {
	74	ptr = kmalloc(size, flags);
	75	} while (ptr == NULL && (flags & __GFP_WAIT));
	76
	77	return ptr;
	78	}
	79
	80	static inline void *
	81	kzalloc_nofail(size_t size, gfp_t flags)
	82	{
	83	void *ptr;
	84
	85	do {
	86	ptr = kzalloc(size, flags);
	87	} while (ptr == NULL && (flags & __GFP_WAIT));
	88
	89	return ptr;
	90	}
	91
	92	#ifdef HAVE_KMALLOC_NODE
	93	static inline void *
	94	kmalloc_node_nofail(size_t size, gfp_t flags, int node)
	95	{
	96	void *ptr;
	97
	98	do {
	99	ptr = kmalloc_node(size, flags, node);
	100	} while (ptr == NULL && (flags & __GFP_WAIT));
	101
	102	return ptr;
	103	}
	104	#endif /* HAVE_KMALLOC_NODE */
	105
f1ca4da6	106	#ifdef DEBUG_KMEM
d04c8a56	107	# ifdef HAVE_ATOMIC64_T
a0f6da3d	108
c19c06f3	109	extern atomic64_t kmem_alloc_used;
a0f6da3d	110	extern unsigned long long kmem_alloc_max;
c19c06f3	111	extern atomic64_t vmem_alloc_used;
a0f6da3d	112	extern unsigned long long vmem_alloc_max;
a0f6da3d	113
d04c8a56 BB	114	# define kmem_alloc_used_add(size) atomic64_add(size, &kmem_alloc_used)
	115	# define kmem_alloc_used_sub(size) atomic64_sub(size, &kmem_alloc_used)
	116	# define kmem_alloc_used_read() atomic64_read(&kmem_alloc_used)
	117	# define kmem_alloc_used_set(size) atomic64_set(&kmem_alloc_used, size)
	118	# define vmem_alloc_used_add(size) atomic64_add(size, &vmem_alloc_used)
	119	# define vmem_alloc_used_sub(size) atomic64_sub(size, &vmem_alloc_used)
	120	# define vmem_alloc_used_read() atomic64_read(&vmem_alloc_used)
	121	# define vmem_alloc_used_set(size) atomic64_set(&vmem_alloc_used, size)
	122
	123	# else
	124
	125	extern atomic_t kmem_alloc_used;
	126	extern unsigned long long kmem_alloc_max;
	127	extern atomic_t vmem_alloc_used;
	128	extern unsigned long long vmem_alloc_max;
	129
	130	# define kmem_alloc_used_add(size) atomic_add(size, &kmem_alloc_used)
	131	# define kmem_alloc_used_sub(size) atomic_sub(size, &kmem_alloc_used)
	132	# define kmem_alloc_used_read() atomic_read(&kmem_alloc_used)
	133	# define kmem_alloc_used_set(size) atomic_set(&kmem_alloc_used, size)
	134	# define vmem_alloc_used_add(size) atomic_add(size, &vmem_alloc_used)
	135	# define vmem_alloc_used_sub(size) atomic_sub(size, &vmem_alloc_used)
	136	# define vmem_alloc_used_read() atomic_read(&vmem_alloc_used)
	137	# define vmem_alloc_used_set(size) atomic_set(&vmem_alloc_used, size)
	138
	139	# endif /* _LP64 */
	140
a0f6da3d	141	# define kmem_alloc(size, flags) __kmem_alloc((size), (flags), 0, 0)
	142	# define kmem_zalloc(size, flags) __kmem_alloc((size), ((flags) \| \
	143	__GFP_ZERO), 0, 0)
	144
	145	/* The node alloc functions are only used by the SPL code itself */
	146	# ifdef HAVE_KMALLOC_NODE
	147	# define kmem_alloc_node(size, flags, node) __kmem_alloc((size), (flags), 1, \
	148	node)
	149	# else
	150	# define kmem_alloc_node(size, flags, node) __kmem_alloc((size), (flags), 0, 0)
	151	# endif
	152
	153	# define vmem_zalloc(size, flags) vmem_alloc((size), ((flags) \| \
	154	__GFP_ZERO))
	155
	156	# ifdef DEBUG_KMEM_TRACKING
	157
	158	extern void kmem_alloc_track(size_t size, int flags, const char func,
	159	int line, int node_alloc, int node);
	160	extern void kmem_free_track(void *ptr, size_t size);
	161	extern void vmem_alloc_track(size_t size, int flags, const char func,
	162	int line);
	163	extern void vmem_free_track(void *ptr, size_t size);
	164
	165	# define __kmem_alloc(size, flags, na, node) kmem_alloc_track((size), \
	166	(flags), __FUNCTION__, \
	167	__LINE__, (na), (node))
	168	# define kmem_free(ptr, size) kmem_free_track((ptr), (size))
	169	# define vmem_alloc(size, flags) vmem_alloc_track((size), \
	170	(flags),__FUNCTION__, \
	171	__LINE__)
	172	# define vmem_free(ptr, size) vmem_free_track((ptr), (size))
	173
	174	# else /* DEBUG_KMEM_TRACKING */
	175
	176	extern void kmem_alloc_debug(size_t size, int flags, const char func,
	177	int line, int node_alloc, int node);
	178	extern void kmem_free_debug(void *ptr, size_t size);
	179	extern void vmem_alloc_debug(size_t size, int flags, const char func,
	180	int line);
	181	extern void vmem_free_debug(void *ptr, size_t size);
	182
	183	# define __kmem_alloc(size, flags, na, node) kmem_alloc_debug((size), \
	184	(flags), __FUNCTION__, \
	185	__LINE__, (na), (node))
	186	# define kmem_free(ptr, size) kmem_free_debug((ptr), (size))
	187	# define vmem_alloc(size, flags) vmem_alloc_debug((size), \
	188	(flags), __FUNCTION__, \
	189	__LINE__)
	190	# define vmem_free(ptr, size) vmem_free_debug((ptr), (size))
	191
	192	# endif /* DEBUG_KMEM_TRACKING */
ff449ac4	193
c6dc93d6	194	#else /* DEBUG_KMEM */
f1ca4da6	195
c89fdee4 BB	196	# define kmem_alloc(size, flags) kmalloc_nofail((size), (flags))
c89fdee4 BB	197	# define kmem_zalloc(size, flags) kzalloc_nofail((size), (flags))
a1cf80b4	198	# define kmem_free(ptr, size) ((void)(size), kfree(ptr))
a0f6da3d	199
	200	# ifdef HAVE_KMALLOC_NODE
	201	# define kmem_alloc_node(size, flags, node) \
c89fdee4	202	kmalloc_node_nofail((size), (flags), (node))
a0f6da3d	203	# else
a0f6da3d	204	# define kmem_alloc_node(size, flags, node) \
c89fdee4	205	kmalloc_nofail((size), (flags))
a0f6da3d	206	# endif
a0f6da3d	207
a1cf80b4 BB	208	# define vmem_alloc(size, flags) __vmalloc((size), ((flags) \| \
a1cf80b4 BB	209	__GFP_HIGHMEM), PAGE_KERNEL)
a0f6da3d	210	# define vmem_zalloc(size, flags) \
79b31f36	211	({ \
c6dc93d6	212	void *_ptr_ = __vmalloc((size),((flags)\|__GFP_HIGHMEM),PAGE_KERNEL); \
	213	if (_ptr_) \
	214	memset(_ptr_, 0, (size)); \
	215	_ptr_; \
79b31f36	216	})
bb01879e	217	# define vmem_free(ptr, size) ((void)(size), vfree(ptr))
79b31f36	218
f1ca4da6	219	#endif /* DEBUG_KMEM */
f1ca4da6	220
f1ca4da6	221	#ifdef DEBUG_KMEM_UNIMPLEMENTED
	222	static __inline__ void *
	223	kmem_alloc_tryhard(size_t size, size_t *alloc_size, int kmflags)
	224	{
	225	#error "kmem_alloc_tryhard() not implemented"
	226	}
	227	#endif /* DEBUG_KMEM_UNIMPLEMENTED */
	228
	229	/*
	230	* Slab allocation interfaces
	231	*/
ea3e6ca9 BB	232	enum {
	233	KMC_BIT_NOTOUCH = 0, /* Don't update ages */
	234	KMC_BIT_NODEBUG = 1, /* Default behavior */
	235	KMC_BIT_NOMAGAZINE = 2, /* XXX: Unsupported */
	236	KMC_BIT_NOHASH = 3, /* XXX: Unsupported */
	237	KMC_BIT_QCACHE = 4, /* XXX: Unsupported */
	238	KMC_BIT_KMEM = 5, /* Use kmem cache */
	239	KMC_BIT_VMEM = 6, /* Use vmem cache */
	240	KMC_BIT_OFFSLAB = 7, /* Objects not on slab */
	241	KMC_BIT_REAPING = 16, /* Reaping in progress */
	242	KMC_BIT_DESTROY = 17, /* Destroy in progress */
	243	};
	244
	245	#define KMC_NOTOUCH (1 << KMC_BIT_NOTOUCH)
	246	#define KMC_NODEBUG (1 << KMC_BIT_NODEBUG)
	247	#define KMC_NOMAGAZINE (1 << KMC_BIT_NOMAGAZINE)
	248	#define KMC_NOHASH (1 << KMC_BIT_NOHASH)
	249	#define KMC_QCACHE (1 << KMC_BIT_QCACHE)
	250	#define KMC_KMEM (1 << KMC_BIT_KMEM)
	251	#define KMC_VMEM (1 << KMC_BIT_VMEM)
	252	#define KMC_OFFSLAB (1 << KMC_BIT_OFFSLAB)
	253	#define KMC_REAPING (1 << KMC_BIT_REAPING)
	254	#define KMC_DESTROY (1 << KMC_BIT_DESTROY)
	255
	256	#define KMC_REAP_CHUNK INT_MAX
	257	#define KMC_DEFAULT_SEEKS 1
f1ca4da6	258
f1ca4da6	259	#ifdef DEBUG_KMEM_UNIMPLEMENTED
	260	static __inline__ void kmem_init(void) {
	261	#error "kmem_init() not implemented"
	262	}
	263
	264	static __inline__ void kmem_thread_init(void) {
	265	#error "kmem_thread_init() not implemented"
	266	}
	267
	268	static __inline__ void kmem_mp_init(void) {
	269	#error "kmem_mp_init() not implemented"
	270	}
	271
	272	static __inline__ void kmem_reap_idspace(void) {
	273	#error "kmem_reap_idspace() not implemented"
	274	}
	275
	276	static __inline__ size_t kmem_avail(void) {
	277	#error "kmem_avail() not implemented"
	278	}
	279
	280	static __inline__ size_t kmem_maxavail(void) {
	281	#error "kmem_maxavail() not implemented"
	282	}
	283
2fb9b26a	284	static __inline__ uint64_t kmem_cache_stat(spl_kmem_cache_t *cache) {
f1ca4da6	285	#error "kmem_cache_stat() not implemented"
	286	}
	287	#endif /* DEBUG_KMEM_UNIMPLEMENTED */
	288
	289	/* XXX - Used by arc.c to adjust its memory footprint. We may want
	290	* to use this hook in the future to adjust behavior based on
	291	* debug levels. For now it's safe to always return 0.
	292	*/
	293	static __inline__ int
	294	kmem_debugging(void)
	295	{
	296	return 0;
	297	}
	298
c19c06f3	299	extern int kmem_set_warning(int flag);
c19c06f3	300
ff449ac4	301	extern struct list_head spl_kmem_cache_list;
ff449ac4	302	extern struct rw_semaphore spl_kmem_cache_sem;
2fb9b26a	303
4afaaefa	304	#define SKM_MAGIC 0x2e2e2e2e
2fb9b26a	305	#define SKO_MAGIC 0x20202020
	306	#define SKS_MAGIC 0x22222222
	307	#define SKC_MAGIC 0x2c2c2c2c
	308
37db7d8c BB	309	#define SPL_KMEM_CACHE_DELAY 15 /* Minimum slab release age */
37db7d8c BB	310	#define SPL_KMEM_CACHE_REAP 0 /* Default reap everything */
ea3e6ca9 BB	311	#define SPL_KMEM_CACHE_OBJ_PER_SLAB 32 /* Target objects per slab */
	312	#define SPL_KMEM_CACHE_OBJ_PER_SLAB_MIN 8 /* Minimum objects per slab */
	313	#define SPL_KMEM_CACHE_ALIGN 8 /* Default object alignment */
2fb9b26a	314
	315	typedef int (spl_kmem_ctor_t)(void , void *, int);
	316	typedef void (spl_kmem_dtor_t)(void , void *);
	317	typedef void (spl_kmem_reclaim_t)(void );
	318
4afaaefa	319	typedef struct spl_kmem_magazine {
9b1b8e4c	320	uint32_t skm_magic; /* Sanity magic */
4afaaefa	321	uint32_t skm_avail; /* Available objects */
	322	uint32_t skm_size; /* Magazine size */
	323	uint32_t skm_refill; /* Batch refill size */
9b1b8e4c BB	324	struct spl_kmem_cache skm_cache; / Owned by cache */
9b1b8e4c BB	325	struct delayed_work skm_work; /* Magazine reclaim work */
4afaaefa	326	unsigned long skm_age; /* Last cache access */
	327	void skm_objs[0]; / Object pointers */
	328	} spl_kmem_magazine_t;
	329
2fb9b26a	330	typedef struct spl_kmem_obj {
2fb9b26a	331	uint32_t sko_magic; /* Sanity magic */
2fb9b26a	332	void sko_addr; / Buffer address */
	333	struct spl_kmem_slab sko_slab; / Owned by slab */
	334	struct list_head sko_list; /* Free object list linkage */
2fb9b26a	335	} spl_kmem_obj_t;
	336
	337	typedef struct spl_kmem_slab {
	338	uint32_t sks_magic; /* Sanity magic */
	339	uint32_t sks_objs; /* Objects per slab */
	340	struct spl_kmem_cache sks_cache; / Owned by cache */
	341	struct list_head sks_list; /* Slab list linkage */
	342	struct list_head sks_free_list; /* Free object list */
	343	unsigned long sks_age; /* Last modify jiffie */
4afaaefa	344	uint32_t sks_ref; /* Ref count used objects */
2fb9b26a	345	} spl_kmem_slab_t;
	346
	347	typedef struct spl_kmem_cache {
ea3e6ca9 BB	348	uint32_t skc_magic; /* Sanity magic */
	349	uint32_t skc_name_size; /* Name length */
	350	char skc_name; / Name string */
4afaaefa	351	spl_kmem_magazine_t skc_mag[NR_CPUS]; / Per-CPU warm cache */
	352	uint32_t skc_mag_size; /* Magazine size */
	353	uint32_t skc_mag_refill; /* Magazine refill count */
ea3e6ca9 BB	354	spl_kmem_ctor_t skc_ctor; /* Constructor */
	355	spl_kmem_dtor_t skc_dtor; /* Destructor */
	356	spl_kmem_reclaim_t skc_reclaim; /* Reclaimator */
	357	void skc_private; / Private data */
	358	void skc_vmp; / Unused */
31a033ec	359	unsigned long skc_flags; /* Flags */
2fb9b26a	360	uint32_t skc_obj_size; /* Object size */
48e0606a	361	uint32_t skc_obj_align; /* Object alignment */
a1502d76	362	uint32_t skc_slab_objs; /* Objects per slab */
ea3e6ca9 BB	363	uint32_t skc_slab_size; /* Slab size */
ea3e6ca9 BB	364	uint32_t skc_delay; /* Slab reclaim interval */
37db7d8c	365	uint32_t skc_reap; /* Slab reclaim count */
ea3e6ca9 BB	366	atomic_t skc_ref; /* Ref count callers */
ea3e6ca9 BB	367	struct delayed_work skc_work; /* Slab reclaim work */
ea3e6ca9	368	struct list_head skc_list; /* List of caches linkage */
2fb9b26a	369	struct list_head skc_complete_list;/* Completely alloc'ed */
2fb9b26a	370	struct list_head skc_partial_list; /* Partially alloc'ed */
d46630e0	371	spinlock_t skc_lock; /* Cache lock */
2fb9b26a	372	uint64_t skc_slab_fail; /* Slab alloc failures */
	373	uint64_t skc_slab_create;/* Slab creates */
	374	uint64_t skc_slab_destroy;/* Slab destroys */
d46630e0	375	uint64_t skc_slab_total; /* Slab total current */
ea3e6ca9	376	uint64_t skc_slab_alloc; /* Slab alloc current */
d46630e0	377	uint64_t skc_slab_max; /* Slab max historic */
	378	uint64_t skc_obj_total; /* Obj total current */
	379	uint64_t skc_obj_alloc; /* Obj alloc current */
	380	uint64_t skc_obj_max; /* Obj max historic */
2fb9b26a	381	} spl_kmem_cache_t;
7afde631	382	#define kmem_cache_t spl_kmem_cache_t
2fb9b26a	383
	384	extern spl_kmem_cache_t *
	385	spl_kmem_cache_create(char *name, size_t size, size_t align,
	386	spl_kmem_ctor_t ctor, spl_kmem_dtor_t dtor, spl_kmem_reclaim_t reclaim,
f1ca4da6	387	void priv, void vmp, int flags);
f1ca4da6	388
2fb9b26a	389	extern void spl_kmem_cache_destroy(spl_kmem_cache_t *skc);
	390	extern void spl_kmem_cache_alloc(spl_kmem_cache_t skc, int flags);
	391	extern void spl_kmem_cache_free(spl_kmem_cache_t skc, void obj);
	392	extern void spl_kmem_cache_reap_now(spl_kmem_cache_t *skc);
	393	extern void spl_kmem_reap(void);
f1ca4da6	394
d1ff2312	395	int spl_kmem_init_kallsyms_lookup(void);
2fb9b26a	396	int spl_kmem_init(void);
2fb9b26a	397	void spl_kmem_fini(void);
5d86345d	398
f1ca4da6	399	#define kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags) \
2fb9b26a	400	spl_kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags)
	401	#define kmem_cache_destroy(skc) spl_kmem_cache_destroy(skc)
	402	#define kmem_cache_alloc(skc, flags) spl_kmem_cache_alloc(skc, flags)
	403	#define kmem_cache_free(skc, obj) spl_kmem_cache_free(skc, obj)
	404	#define kmem_cache_reap_now(skc) spl_kmem_cache_reap_now(skc)
	405	#define kmem_reap() spl_kmem_reap()
a1502d76	406	#define kmem_virt(ptr) (((ptr) >= (void *)VMALLOC_START) && \
a1502d76	407	((ptr) < (void *)VMALLOC_END))
f1ca4da6	408
09b414e8	409	#endif /* _SPL_KMEM_H */