[mirror_zfs.git] / include / sys / kmem.h

#ifndef _SPL_KMEM_H
#define	_SPL_KMEM_H

#ifdef	__cplusplus
extern "C" {
#endif

#define DEBUG_KMEM
#undef DEBUG_KMEM_UNIMPLEMENTED

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/mm.h>
#include <linux/spinlock.h>
#include <sys/debug.h>
/*
 * Memory allocation interfaces
 */
#define KM_SLEEP                        GFP_KERNEL
#define KM_NOSLEEP                      GFP_ATOMIC
#undef  KM_PANIC                        /* No linux analog */
#define KM_PUSHPAGE			(GFP_KERNEL | GFP_HIGH)
#define KM_VMFLAGS                      GFP_LEVEL_MASK
#define KM_FLAGS                        __GFP_BITS_MASK

#ifdef DEBUG_KMEM
extern atomic64_t kmem_alloc_used;
extern unsigned long kmem_alloc_max;
extern atomic64_t vmem_alloc_used;
extern unsigned long vmem_alloc_max;
extern int kmem_warning_flag;

#define __kmem_alloc(size, flags, allocator)                                  \
({      void *_ptr_;                                                          \
                                                                              \
	/* Marked unlikely because we should never be doing this */           \
        if (unlikely((size) > (PAGE_SIZE * 4)) && kmem_warning_flag)          \
                __CDEBUG_LIMIT(S_KMEM, D_WARNING, "Warning large "            \
			       "kmem_alloc(%d, 0x%x) (%ld/%ld)\n",            \
			       (int)(size), (int)(flags),                     \
		               atomic64_read(&kmem_alloc_used),               \
			       kmem_alloc_max);                               \
                                                                              \
        _ptr_ = (void *)allocator((size), (flags));                           \
        if (_ptr_ == NULL) {                                                  \
                __CDEBUG_LIMIT(S_KMEM, D_WARNING, "Warning "                  \
			       "kmem_alloc(%d, 0x%x) failed (%ld/%ld)\n",     \
			       (int)(size), (int)(flags),                     \
		               atomic64_read(&kmem_alloc_used),               \
			       kmem_alloc_max);                               \
        } else {                                                              \
                atomic64_add((size), &kmem_alloc_used);                       \
                if (unlikely(atomic64_read(&kmem_alloc_used)>kmem_alloc_max)) \
                        kmem_alloc_max = atomic64_read(&kmem_alloc_used);     \
			                                                      \
                __CDEBUG_LIMIT(S_KMEM, D_INFO, "kmem_alloc(%d, 0x%x)'d "      \
			       "(%ld/%ld)\n", (int)(size), (int)(flags),      \
		               atomic64_read(&kmem_alloc_used),               \
			       kmem_alloc_max);                               \
        }                                                                     \
                                                                              \
        _ptr_;                                                                \
})

#define kmem_alloc(size, flags)         __kmem_alloc((size), (flags), kmalloc)
#define kmem_zalloc(size, flags)        __kmem_alloc((size), (flags), kzalloc)

#define kmem_free(ptr, size)                                                  \
({                                                                            \
        ASSERT((ptr) || (size > 0));                                          \
        atomic64_sub((size), &kmem_alloc_used);                               \
        __CDEBUG_LIMIT(S_KMEM, D_INFO, "kmem_free(%d)'d (%ld/%ld)\n",         \
		       (int)(size), atomic64_read(&kmem_alloc_used),          \
		       kmem_alloc_max);                                       \
        memset(ptr, 0x5a, (size)); /* Poison */                               \
        kfree(ptr);                                                           \
})

#define __vmem_alloc(size, flags)                                             \
({      void *_ptr_;                                                          \
                                                                              \
	ASSERT(flags & KM_SLEEP);                                             \
                                                                              \
        _ptr_ = (void *)__vmalloc((size),                                     \
				  (((flags) | __GFP_HIGHMEM) & ~__GFP_ZERO),  \
				  PAGE_KERNEL);                               \
        if (_ptr_ == NULL) {                                                  \
                __CDEBUG_LIMIT(S_KMEM, D_WARNING, "Warning "                  \
			       "vmem_alloc(%d, 0x%x) failed (%ld/%ld)\n",     \
			       (int)(size), (int)(flags),                     \
		              atomic64_read(&vmem_alloc_used),                \
			      vmem_alloc_max);                                \
        } else {                                                              \
                if (flags & __GFP_ZERO)                                       \
                        memset(_ptr_, 0, (size));                             \
                                                                              \
                atomic64_add((size), &vmem_alloc_used);                       \
                if (unlikely(atomic64_read(&vmem_alloc_used)>vmem_alloc_max)) \
                        vmem_alloc_max = atomic64_read(&vmem_alloc_used);     \
                                                                              \
                __CDEBUG_LIMIT(S_KMEM, D_INFO, "vmem_alloc(%d, 0x%x)'d "      \
			       "(%ld/%ld)\n", (int)(size), (int)(flags),      \
		               atomic64_read(&vmem_alloc_used),               \
			       vmem_alloc_max);                               \
        }                                                                     \
                                                                              \
        _ptr_;                                                                \
})

#define vmem_alloc(size, flags)         __vmem_alloc((size), (flags))
#define vmem_zalloc(size, flags)        __vmem_alloc((size), ((flags) |       \
                                                     __GFP_ZERO))

#define vmem_free(ptr, size)                                                  \
({                                                                            \
        ASSERT((ptr) || (size > 0));                                          \
        atomic64_sub((size), &vmem_alloc_used);                               \
        __CDEBUG_LIMIT(S_KMEM, D_INFO, "vmem_free(%d)'d (%ld/%ld)\n",         \
		       (int)(size), atomic64_read(&vmem_alloc_used),          \
		       vmem_alloc_max);                                       \
        memset(ptr, 0x5a, (size)); /* Poison */                               \
        vfree(ptr);                                                           \
})

#else

#define kmem_alloc(size, flags)         kmalloc((size), (flags))
#define kmem_zalloc(size, flags)        kzalloc((size), (flags))
#define kmem_free(ptr, size)                                                  \
({                                                                            \
        ASSERT((ptr) || (size > 0));                                          \
	kfree(ptr);                                                           \
})

#define vmem_alloc(size, flags)         __vmalloc((size), ((flags) |          \
						  __GFP_HIGHMEM), PAGE_KERNEL)
#define vmem_zalloc(size, flags)        __vmalloc((size), ((flags) |          \
						  __GFP_HIGHMEM | __GFP_ZERO) \
						  PAGE_KERNEL)
#define vmem_free(ptr, size)                                                  \
({                                                                            \
        ASSERT((ptr) || (size > 0));                                          \
	vfree(ptr);                                                           \
})

#endif /* DEBUG_KMEM */


#ifdef DEBUG_KMEM_UNIMPLEMENTED
static __inline__ void *
kmem_alloc_tryhard(size_t size, size_t *alloc_size, int kmflags)
{
#error "kmem_alloc_tryhard() not implemented"
}
#endif /* DEBUG_KMEM_UNIMPLEMENTED */

/*
 * Slab allocation interfaces
 */
#undef  KMC_NOTOUCH                     /* No linux analog */
#define KMC_NODEBUG                     0x00000000 /* Default behavior */
#define KMC_NOMAGAZINE                  /* No linux analog */
#define KMC_NOHASH                      /* No linux analog */
#define KMC_QCACHE                      /* No linux analog */

#define KMC_REAP_CHUNK                  256
#define KMC_DEFAULT_SEEKS               DEFAULT_SEEKS

/* Defined by linux slab.h
 * typedef struct kmem_cache_s kmem_cache_t;
 */

/* No linux analog
 * extern int kmem_ready;
 * extern pgcnt_t kmem_reapahead;
 */

#ifdef DEBUG_KMEM_UNIMPLEMENTED
static __inline__ void kmem_init(void) {
#error "kmem_init() not implemented"
}

static __inline__ void kmem_thread_init(void) {
#error "kmem_thread_init() not implemented"
}

static __inline__ void kmem_mp_init(void) {
#error "kmem_mp_init() not implemented"
}

static __inline__ void kmem_reap_idspace(void) {
#error "kmem_reap_idspace() not implemented"
}

static __inline__ size_t kmem_avail(void) {
#error "kmem_avail() not implemented"
}

static __inline__ size_t kmem_maxavail(void) {
#error "kmem_maxavail() not implemented"
}

static __inline__ uint64_t kmem_cache_stat(kmem_cache_t *cache) {
#error "kmem_cache_stat() not implemented"
}
#endif /* DEBUG_KMEM_UNIMPLEMENTED */

/* XXX - Used by arc.c to adjust its memory footprint. We may want
 *       to use this hook in the future to adjust behavior based on
 *       debug levels.  For now it's safe to always return 0.
 */
static __inline__ int
kmem_debugging(void)
{
        return 0;
}

typedef int (*kmem_constructor_t)(void *, void *, int);
typedef void (*kmem_destructor_t)(void *, void *);
typedef void (*kmem_reclaim_t)(void *);

extern int kmem_set_warning(int flag);

extern kmem_cache_t *
__kmem_cache_create(char *name, size_t size, size_t align,
        kmem_constructor_t constructor,
        kmem_destructor_t destructor,
        kmem_reclaim_t reclaim,
        void *priv, void *vmp, int flags);

int
extern __kmem_cache_destroy(kmem_cache_t *cache);

void
extern __kmem_reap(void);

int kmem_init(void);
void kmem_fini(void);

#define kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags) \
        __kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags)
#define kmem_cache_destroy(cache)       __kmem_cache_destroy(cache)
#define kmem_cache_alloc(cache, flags)  kmem_cache_alloc(cache, flags)
#define kmem_cache_free(cache, ptr)     kmem_cache_free(cache, ptr)
#define kmem_cache_reap_now(cache)      kmem_cache_shrink(cache)
#define kmem_reap()                     __kmem_reap()

#ifdef	__cplusplus
}
#endif

#endif	/* _SPL_KMEM_H */
Commit	Line	Data
09b414e8 BB	1	#ifndef _SPL_KMEM_H
09b414e8 BB	2	#define _SPL_KMEM_H
f1ca4da6 BB	3
	4	#ifdef __cplusplus
	5	extern "C" {
	6	#endif
	7
79b31f36	8	#define DEBUG_KMEM
f1ca4da6 BB	9	#undef DEBUG_KMEM_UNIMPLEMENTED
f1ca4da6 BB	10
f1b59d26	11	#include <linux/module.h>
f1ca4da6	12	#include <linux/slab.h>
79b31f36	13	#include <linux/vmalloc.h>
f1ca4da6 BB	14	#include <linux/mm.h>
f1ca4da6 BB	15	#include <linux/spinlock.h>
937879f1	16	#include <sys/debug.h>
f1ca4da6 BB	17	/*
	18	* Memory allocation interfaces
	19	*/
	20	#define KM_SLEEP GFP_KERNEL
	21	#define KM_NOSLEEP GFP_ATOMIC
	22	#undef KM_PANIC /* No linux analog */
	23	#define KM_PUSHPAGE (GFP_KERNEL \| GFP_HIGH)
	24	#define KM_VMFLAGS GFP_LEVEL_MASK
	25	#define KM_FLAGS __GFP_BITS_MASK
	26
	27	#ifdef DEBUG_KMEM
c19c06f3 BB	28	extern atomic64_t kmem_alloc_used;
	29	extern unsigned long kmem_alloc_max;
	30	extern atomic64_t vmem_alloc_used;
	31	extern unsigned long vmem_alloc_max;
	32	extern int kmem_warning_flag;
f1ca4da6	33
f1ca4da6 BB	34	#define __kmem_alloc(size, flags, allocator) \
	35	({ void *_ptr_; \
	36	\
	37	/* Marked unlikely because we should never be doing this */ \
6a585c61	38	if (unlikely((size) > (PAGE_SIZE * 4)) && kmem_warning_flag) \
839d8b43 BB	39	__CDEBUG_LIMIT(S_KMEM, D_WARNING, "Warning large " \
	40	"kmem_alloc(%d, 0x%x) (%ld/%ld)\n", \
	41	(int)(size), (int)(flags), \
3561541c BB	42	atomic64_read(&kmem_alloc_used), \
3561541c BB	43	kmem_alloc_max); \
f1ca4da6 BB	44	\
	45	_ptr_ = (void *)allocator((size), (flags)); \
	46	if (_ptr_ == NULL) { \
3561541c	47	__CDEBUG_LIMIT(S_KMEM, D_WARNING, "Warning " \
839d8b43 BB	48	"kmem_alloc(%d, 0x%x) failed (%ld/%ld)\n", \
839d8b43 BB	49	(int)(size), (int)(flags), \
3561541c BB	50	atomic64_read(&kmem_alloc_used), \
3561541c BB	51	kmem_alloc_max); \
c19c06f3 BB	52	} else { \
	53	atomic64_add((size), &kmem_alloc_used); \
	54	if (unlikely(atomic64_read(&kmem_alloc_used)>kmem_alloc_max)) \
	55	kmem_alloc_max = atomic64_read(&kmem_alloc_used); \
839d8b43 BB	56	\
	57	__CDEBUG_LIMIT(S_KMEM, D_INFO, "kmem_alloc(%d, 0x%x)'d " \
	58	"(%ld/%ld)\n", (int)(size), (int)(flags), \
	59	atomic64_read(&kmem_alloc_used), \
	60	kmem_alloc_max); \
f1ca4da6 BB	61	} \
	62	\
	63	_ptr_; \
	64	})
	65
4fd2f7ee BB	66	#define kmem_alloc(size, flags) __kmem_alloc((size), (flags), kmalloc)
4fd2f7ee BB	67	#define kmem_zalloc(size, flags) __kmem_alloc((size), (flags), kzalloc)
f1ca4da6 BB	68
	69	#define kmem_free(ptr, size) \
	70	({ \
937879f1	71	ASSERT((ptr) \|\| (size > 0)); \
c19c06f3	72	atomic64_sub((size), &kmem_alloc_used); \
839d8b43 BB	73	__CDEBUG_LIMIT(S_KMEM, D_INFO, "kmem_free(%d)'d (%ld/%ld)\n", \
	74	(int)(size), atomic64_read(&kmem_alloc_used), \
	75	kmem_alloc_max); \
f1ca4da6 BB	76	memset(ptr, 0x5a, (size)); /* Poison */ \
f1ca4da6 BB	77	kfree(ptr); \
f1ca4da6 BB	78	})
f1ca4da6 BB	79
79b31f36 BB	80	#define __vmem_alloc(size, flags) \
	81	({ void *_ptr_; \
	82	\
937879f1	83	ASSERT(flags & KM_SLEEP); \
79b31f36	84	\
3561541c BB	85	_ptr_ = (void *)__vmalloc((size), \
	86	(((flags) \| __GFP_HIGHMEM) & ~__GFP_ZERO), \
	87	PAGE_KERNEL); \
79b31f36	88	if (_ptr_ == NULL) { \
3561541c	89	__CDEBUG_LIMIT(S_KMEM, D_WARNING, "Warning " \
839d8b43 BB	90	"vmem_alloc(%d, 0x%x) failed (%ld/%ld)\n", \
839d8b43 BB	91	(int)(size), (int)(flags), \
3561541c BB	92	atomic64_read(&vmem_alloc_used), \
3561541c BB	93	vmem_alloc_max); \
c19c06f3	94	} else { \
8100fe56 HW	95	if (flags & __GFP_ZERO) \
8100fe56 HW	96	memset(_ptr_, 0, (size)); \
3561541c	97	\
c19c06f3 BB	98	atomic64_add((size), &vmem_alloc_used); \
	99	if (unlikely(atomic64_read(&vmem_alloc_used)>vmem_alloc_max)) \
	100	vmem_alloc_max = atomic64_read(&vmem_alloc_used); \
839d8b43 BB	101	\
	102	__CDEBUG_LIMIT(S_KMEM, D_INFO, "vmem_alloc(%d, 0x%x)'d " \
	103	"(%ld/%ld)\n", (int)(size), (int)(flags), \
	104	atomic64_read(&vmem_alloc_used), \
	105	vmem_alloc_max); \
79b31f36 BB	106	} \
	107	\
	108	_ptr_; \
	109	})
	110
4fd2f7ee BB	111	#define vmem_alloc(size, flags) __vmem_alloc((size), (flags))
	112	#define vmem_zalloc(size, flags) __vmem_alloc((size), ((flags) \| \
	113	__GFP_ZERO))
79b31f36 BB	114
	115	#define vmem_free(ptr, size) \
	116	({ \
937879f1	117	ASSERT((ptr) \|\| (size > 0)); \
c19c06f3	118	atomic64_sub((size), &vmem_alloc_used); \
839d8b43 BB	119	__CDEBUG_LIMIT(S_KMEM, D_INFO, "vmem_free(%d)'d (%ld/%ld)\n", \
	120	(int)(size), atomic64_read(&vmem_alloc_used), \
	121	vmem_alloc_max); \
79b31f36 BB	122	memset(ptr, 0x5a, (size)); /* Poison */ \
	123	vfree(ptr); \
	124	})
f1ca4da6 BB	125
	126	#else
	127
4fd2f7ee BB	128	#define kmem_alloc(size, flags) kmalloc((size), (flags))
4fd2f7ee BB	129	#define kmem_zalloc(size, flags) kzalloc((size), (flags))
3b3ba48f BB	130	#define kmem_free(ptr, size) \
3b3ba48f BB	131	({ \
937879f1	132	ASSERT((ptr) \|\| (size > 0)); \
3b3ba48f BB	133	kfree(ptr); \
3b3ba48f BB	134	})
f1ca4da6	135
4fd2f7ee BB	136	#define vmem_alloc(size, flags) __vmalloc((size), ((flags) \| \
	137	__GFP_HIGHMEM), PAGE_KERNEL)
	138	#define vmem_zalloc(size, flags) __vmalloc((size), ((flags) \| \
	139	__GFP_HIGHMEM \| __GFP_ZERO) \
	140	PAGE_KERNEL)
79b31f36 BB	141	#define vmem_free(ptr, size) \
79b31f36 BB	142	({ \
937879f1	143	ASSERT((ptr) \|\| (size > 0)); \
79b31f36 BB	144	vfree(ptr); \
	145	})
	146
f1ca4da6 BB	147	#endif /* DEBUG_KMEM */
	148
	149
	150	#ifdef DEBUG_KMEM_UNIMPLEMENTED
	151	static __inline__ void *
	152	kmem_alloc_tryhard(size_t size, size_t *alloc_size, int kmflags)
	153	{
	154	#error "kmem_alloc_tryhard() not implemented"
	155	}
	156	#endif /* DEBUG_KMEM_UNIMPLEMENTED */
	157
	158	/*
	159	* Slab allocation interfaces
	160	*/
	161	#undef KMC_NOTOUCH /* No linux analog */
48f940b9	162	#define KMC_NODEBUG 0x00000000 /* Default behavior */
f1ca4da6 BB	163	#define KMC_NOMAGAZINE /* No linux analog */
	164	#define KMC_NOHASH /* No linux analog */
	165	#define KMC_QCACHE /* No linux analog */
	166
	167	#define KMC_REAP_CHUNK 256
	168	#define KMC_DEFAULT_SEEKS DEFAULT_SEEKS
	169
	170	/* Defined by linux slab.h
	171	* typedef struct kmem_cache_s kmem_cache_t;
	172	*/
	173
	174	/* No linux analog
	175	* extern int kmem_ready;
	176	* extern pgcnt_t kmem_reapahead;
	177	*/
	178
	179	#ifdef DEBUG_KMEM_UNIMPLEMENTED
	180	static __inline__ void kmem_init(void) {
	181	#error "kmem_init() not implemented"
	182	}
	183
	184	static __inline__ void kmem_thread_init(void) {
	185	#error "kmem_thread_init() not implemented"
	186	}
	187
	188	static __inline__ void kmem_mp_init(void) {
	189	#error "kmem_mp_init() not implemented"
	190	}
	191
	192	static __inline__ void kmem_reap_idspace(void) {
	193	#error "kmem_reap_idspace() not implemented"
	194	}
	195
	196	static __inline__ size_t kmem_avail(void) {
	197	#error "kmem_avail() not implemented"
	198	}
	199
	200	static __inline__ size_t kmem_maxavail(void) {
	201	#error "kmem_maxavail() not implemented"
	202	}
	203
	204	static __inline__ uint64_t kmem_cache_stat(kmem_cache_t *cache) {
	205	#error "kmem_cache_stat() not implemented"
	206	}
	207	#endif /* DEBUG_KMEM_UNIMPLEMENTED */
	208
	209	/* XXX - Used by arc.c to adjust its memory footprint. We may want
	210	* to use this hook in the future to adjust behavior based on
	211	* debug levels. For now it's safe to always return 0.
	212	*/
	213	static __inline__ int
	214	kmem_debugging(void)
	215	{
	216	return 0;
	217	}
	218
	219	typedef int (kmem_constructor_t)(void , void *, int);
	220	typedef void (kmem_destructor_t)(void , void *);
	221	typedef void (kmem_reclaim_t)(void );
	222
c19c06f3 BB	223	extern int kmem_set_warning(int flag);
c19c06f3 BB	224
f1b59d26	225	extern kmem_cache_t *
f1ca4da6	226	__kmem_cache_create(char *name, size_t size, size_t align,
f1b59d26 BB	227	kmem_constructor_t constructor,
	228	kmem_destructor_t destructor,
	229	kmem_reclaim_t reclaim,
f1ca4da6 BB	230	void priv, void vmp, int flags);
f1ca4da6 BB	231
e4f1d29f	232	int
f1b59d26 BB	233	extern __kmem_cache_destroy(kmem_cache_t *cache);
	234
	235	void
	236	extern __kmem_reap(void);
f1ca4da6	237
5d86345d BB	238	int kmem_init(void);
	239	void kmem_fini(void);
	240
f1ca4da6 BB	241	#define kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags) \
	242	__kmem_cache_create(name,size,align,ctor,dtor,rclm,priv,vmp,flags)
	243	#define kmem_cache_destroy(cache) __kmem_cache_destroy(cache)
	244	#define kmem_cache_alloc(cache, flags) kmem_cache_alloc(cache, flags)
	245	#define kmem_cache_free(cache, ptr) kmem_cache_free(cache, ptr)
	246	#define kmem_cache_reap_now(cache) kmem_cache_shrink(cache)
f1b59d26	247	#define kmem_reap() __kmem_reap()
f1ca4da6 BB	248
	249	#ifdef __cplusplus
	250	}
	251	#endif
	252
09b414e8	253	#endif /* _SPL_KMEM_H */