[mirror_ubuntu-zesty-kernel.git] / include / linux / slab_def.h

#ifndef _LINUX_SLAB_DEF_H
#define	_LINUX_SLAB_DEF_H

/*
 * Definitions unique to the original Linux SLAB allocator.
 *
 * What we provide here is a way to optimize the frequent kmalloc
 * calls in the kernel by selecting the appropriate general cache
 * if kmalloc was called with a size that can be established at
 * compile time.
 */

#include <linux/init.h>
#include <asm/page.h>		/* kmalloc_sizes.h needs PAGE_SIZE */
#include <asm/cache.h>		/* kmalloc_sizes.h needs L1_CACHE_BYTES */
#include <linux/compiler.h>

#include <trace/events/kmem.h>

/*
 * Enforce a minimum alignment for the kmalloc caches.
 * Usually, the kmalloc caches are cache_line_size() aligned, except when
 * DEBUG and FORCED_DEBUG are enabled, then they are BYTES_PER_WORD aligned.
 * Some archs want to perform DMA into kmalloc caches and need a guaranteed
 * alignment larger than the alignment of a 64-bit integer.
 * ARCH_KMALLOC_MINALIGN allows that.
 * Note that increasing this value may disable some debug features.
 */
#ifdef ARCH_DMA_MINALIGN
#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN
#else
#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
#endif

#ifndef ARCH_SLAB_MINALIGN
/*
 * Enforce a minimum alignment for all caches.
 * Intended for archs that get misalignment faults even for BYTES_PER_WORD
 * aligned buffers. Includes ARCH_KMALLOC_MINALIGN.
 * If possible: Do not enable this flag for CONFIG_DEBUG_SLAB, it disables
 * some debug features.
 */
#define ARCH_SLAB_MINALIGN 0
#endif

/*
 * struct kmem_cache
 *
 * manages a cache.
 */

struct kmem_cache {
/* 1) per-cpu data, touched during every alloc/free */
	struct array_cache *array[NR_CPUS];
/* 2) Cache tunables. Protected by cache_chain_mutex */
	unsigned int batchcount;
	unsigned int limit;
	unsigned int shared;

	unsigned int buffer_size;
	u32 reciprocal_buffer_size;
/* 3) touched by every alloc & free from the backend */

	unsigned int flags;		/* constant flags */
	unsigned int num;		/* # of objs per slab */

/* 4) cache_grow/shrink */
	/* order of pgs per slab (2^n) */
	unsigned int gfporder;

	/* force GFP flags, e.g. GFP_DMA */
	gfp_t gfpflags;

	size_t colour;			/* cache colouring range */
	unsigned int colour_off;	/* colour offset */
	struct kmem_cache *slabp_cache;
	unsigned int slab_size;
	unsigned int dflags;		/* dynamic flags */

	/* constructor func */
	void (*ctor)(void *obj);

/* 5) cache creation/removal */
	const char *name;
	struct list_head next;

/* 6) statistics */
#ifdef CONFIG_DEBUG_SLAB
	unsigned long num_active;
	unsigned long num_allocations;
	unsigned long high_mark;
	unsigned long grown;
	unsigned long reaped;
	unsigned long errors;
	unsigned long max_freeable;
	unsigned long node_allocs;
	unsigned long node_frees;
	unsigned long node_overflow;
	atomic_t allochit;
	atomic_t allocmiss;
	atomic_t freehit;
	atomic_t freemiss;

	/*
	 * If debugging is enabled, then the allocator can add additional
	 * fields and/or padding to every object. buffer_size contains the total
	 * object size including these internal fields, the following two
	 * variables contain the offset to the user object and its size.
	 */
	int obj_offset;
	int obj_size;
#endif /* CONFIG_DEBUG_SLAB */

	/*
	 * We put nodelists[] at the end of kmem_cache, because we want to size
	 * this array to nr_node_ids slots instead of MAX_NUMNODES
	 * (see kmem_cache_init())
	 * We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
	 * is statically defined, so we reserve the max number of nodes.
	 */
	struct kmem_list3 *nodelists[MAX_NUMNODES];
	/*
	 * Do not add fields after nodelists[]
	 */
};

/* Size description struct for general caches. */
struct cache_sizes {
	size_t		 	cs_size;
	struct kmem_cache	*cs_cachep;
#ifdef CONFIG_ZONE_DMA
	struct kmem_cache	*cs_dmacachep;
#endif
};
extern struct cache_sizes malloc_sizes[];

void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
void *__kmalloc(size_t size, gfp_t flags);

#ifdef CONFIG_TRACING
extern void *kmem_cache_alloc_trace(size_t size,
				    struct kmem_cache *cachep, gfp_t flags);
extern size_t slab_buffer_size(struct kmem_cache *cachep);
#else
static __always_inline void *
kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags)
{
	return kmem_cache_alloc(cachep, flags);
}
static inline size_t slab_buffer_size(struct kmem_cache *cachep)
{
	return 0;
}
#endif

static __always_inline void *kmalloc(size_t size, gfp_t flags)
{
	struct kmem_cache *cachep;
	void *ret;

	if (__builtin_constant_p(size)) {
		int i = 0;

		if (!size)
			return ZERO_SIZE_PTR;

#define CACHE(x) \
		if (size <= x) \
			goto found; \
		else \
			i++;
#include <linux/kmalloc_sizes.h>
#undef CACHE
		return NULL;
found:
#ifdef CONFIG_ZONE_DMA
		if (flags & GFP_DMA)
			cachep = malloc_sizes[i].cs_dmacachep;
		else
#endif
			cachep = malloc_sizes[i].cs_cachep;

		ret = kmem_cache_alloc_trace(size, cachep, flags);

		return ret;
	}
	return __kmalloc(size, flags);
}

#ifdef CONFIG_NUMA
extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);

#ifdef CONFIG_TRACING
extern void *kmem_cache_alloc_node_trace(size_t size,
					 struct kmem_cache *cachep,
					 gfp_t flags,
					 int nodeid);
#else
static __always_inline void *
kmem_cache_alloc_node_trace(size_t size,
			    struct kmem_cache *cachep,
			    gfp_t flags,
			    int nodeid)
{
	return kmem_cache_alloc_node(cachep, flags, nodeid);
}
#endif

static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
{
	struct kmem_cache *cachep;

	if (__builtin_constant_p(size)) {
		int i = 0;

		if (!size)
			return ZERO_SIZE_PTR;

#define CACHE(x) \
		if (size <= x) \
			goto found; \
		else \
			i++;
#include <linux/kmalloc_sizes.h>
#undef CACHE
		return NULL;
found:
#ifdef CONFIG_ZONE_DMA
		if (flags & GFP_DMA)
			cachep = malloc_sizes[i].cs_dmacachep;
		else
#endif
			cachep = malloc_sizes[i].cs_cachep;

		return kmem_cache_alloc_node_trace(size, cachep, flags, node);
	}
	return __kmalloc_node(size, flags, node);
}

#endif	/* CONFIG_NUMA */

#endif	/* _LINUX_SLAB_DEF_H */
Commit	Line	Data
2e892f43 CL	1	#ifndef _LINUX_SLAB_DEF_H
	2	#define _LINUX_SLAB_DEF_H
	3
	4	/*
	5	* Definitions unique to the original Linux SLAB allocator.
	6	*
	7	* What we provide here is a way to optimize the frequent kmalloc
	8	* calls in the kernel by selecting the appropriate general cache
	9	* if kmalloc was called with a size that can be established at
	10	* compile time.
	11	*/
	12
	13	#include <linux/init.h>
	14	#include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */
	15	#include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */
	16	#include <linux/compiler.h>
039ca4e7 LZ	17
039ca4e7 LZ	18	#include <trace/events/kmem.h>
2e892f43	19
1f0ce8b3 DW	20	/*
	21	* Enforce a minimum alignment for the kmalloc caches.
	22	* Usually, the kmalloc caches are cache_line_size() aligned, except when
	23	* DEBUG and FORCED_DEBUG are enabled, then they are BYTES_PER_WORD aligned.
	24	* Some archs want to perform DMA into kmalloc caches and need a guaranteed
	25	* alignment larger than the alignment of a 64-bit integer.
	26	* ARCH_KMALLOC_MINALIGN allows that.
	27	* Note that increasing this value may disable some debug features.
	28	*/
a6eb9fe1 FT	29	#ifdef ARCH_DMA_MINALIGN
	30	#define ARCH_KMALLOC_MINALIGN ARCH_DMA_MINALIGN
	31	#else
1f0ce8b3 DW	32	#define ARCH_KMALLOC_MINALIGN __alignof__(unsigned long long)
	33	#endif
	34
	35	#ifndef ARCH_SLAB_MINALIGN
	36	/*
	37	* Enforce a minimum alignment for all caches.
	38	* Intended for archs that get misalignment faults even for BYTES_PER_WORD
	39	* aligned buffers. Includes ARCH_KMALLOC_MINALIGN.
	40	* If possible: Do not enable this flag for CONFIG_DEBUG_SLAB, it disables
	41	* some debug features.
	42	*/
	43	#define ARCH_SLAB_MINALIGN 0
	44	#endif
	45
8eae985f PE	46	/*
	47	* struct kmem_cache
	48	*
	49	* manages a cache.
	50	*/
	51
	52	struct kmem_cache {
	53	/* 1) per-cpu data, touched during every alloc/free */
	54	struct array_cache *array[NR_CPUS];
	55	/* 2) Cache tunables. Protected by cache_chain_mutex */
	56	unsigned int batchcount;
	57	unsigned int limit;
	58	unsigned int shared;
	59
	60	unsigned int buffer_size;
	61	u32 reciprocal_buffer_size;
	62	/* 3) touched by every alloc & free from the backend */
	63
	64	unsigned int flags; /* constant flags */
	65	unsigned int num; /* # of objs per slab */
	66
	67	/* 4) cache_grow/shrink */
	68	/* order of pgs per slab (2^n) */
	69	unsigned int gfporder;
	70
	71	/* force GFP flags, e.g. GFP_DMA */
	72	gfp_t gfpflags;
	73
	74	size_t colour; /* cache colouring range */
	75	unsigned int colour_off; /* colour offset */
	76	struct kmem_cache *slabp_cache;
	77	unsigned int slab_size;
	78	unsigned int dflags; /* dynamic flags */
	79
	80	/* constructor func */
	81	void (ctor)(void obj);
	82
	83	/* 5) cache creation/removal */
	84	const char *name;
	85	struct list_head next;
	86
	87	/* 6) statistics */
	88	#ifdef CONFIG_DEBUG_SLAB
	89	unsigned long num_active;
	90	unsigned long num_allocations;
	91	unsigned long high_mark;
	92	unsigned long grown;
	93	unsigned long reaped;
	94	unsigned long errors;
	95	unsigned long max_freeable;
	96	unsigned long node_allocs;
	97	unsigned long node_frees;
	98	unsigned long node_overflow;
	99	atomic_t allochit;
	100	atomic_t allocmiss;
	101	atomic_t freehit;
	102	atomic_t freemiss;
	103
	104	/*
	105	* If debugging is enabled, then the allocator can add additional
	106	* fields and/or padding to every object. buffer_size contains the total
	107	* object size including these internal fields, the following two
	108	* variables contain the offset to the user object and its size.
	109	*/
110	int obj_offset;
111	int obj_size;
112	#endif /* CONFIG_DEBUG_SLAB */
113
114	/*
115	* We put nodelists[] at the end of kmem_cache, because we want to size
116	* this array to nr_node_ids slots instead of MAX_NUMNODES
117	* (see kmem_cache_init())
118	* We still use [MAX_NUMNODES] and not [1] or [0] because cache_cache
119	* is statically defined, so we reserve the max number of nodes.
120	*/
121	struct kmem_list3 *nodelists[MAX_NUMNODES];
122	/*
123	* Do not add fields after nodelists[]
124	*/
125	};
126
2e892f43 CL	127	/* Size description struct for general caches. */
	128	struct cache_sizes {
	129	size_t cs_size;
	130	struct kmem_cache *cs_cachep;
4b51d669	131	#ifdef CONFIG_ZONE_DMA
2e892f43	132	struct kmem_cache *cs_dmacachep;
4b51d669	133	#endif
2e892f43 CL	134	};
	135	extern struct cache_sizes malloc_sizes[];
	136
6193a2ff PM	137	void kmem_cache_alloc(struct kmem_cache , gfp_t);
	138	void *__kmalloc(size_t size, gfp_t flags);
	139
0f24f128	140	#ifdef CONFIG_TRACING
85beb586 SR	141	extern void *kmem_cache_alloc_trace(size_t size,
85beb586 SR	142	struct kmem_cache *cachep, gfp_t flags);
36555751 EGM	143	extern size_t slab_buffer_size(struct kmem_cache *cachep);
	144	#else
	145	static __always_inline void *
85beb586	146	kmem_cache_alloc_trace(size_t size, struct kmem_cache *cachep, gfp_t flags)
2e892f43	147	{
36555751 EGM	148	return kmem_cache_alloc(cachep, flags);
	149	}
	150	static inline size_t slab_buffer_size(struct kmem_cache *cachep)
	151	{
	152	return 0;
	153	}
	154	#endif
	155
	156	static __always_inline void *kmalloc(size_t size, gfp_t flags)
	157	{
	158	struct kmem_cache *cachep;
	159	void *ret;
	160
2e892f43 CL	161	if (__builtin_constant_p(size)) {
2e892f43 CL	162	int i = 0;
6cb8f913 CL	163
	164	if (!size)
	165	return ZERO_SIZE_PTR;
	166
2e892f43 CL	167	#define CACHE(x) \
	168	if (size <= x) \
	169	goto found; \
	170	else \
	171	i++;
1c61fc40	172	#include <linux/kmalloc_sizes.h>
2e892f43	173	#undef CACHE
1cf3eb2f	174	return NULL;
2e892f43	175	found:
4b51d669 CL	176	#ifdef CONFIG_ZONE_DMA
4b51d669 CL	177	if (flags & GFP_DMA)
36555751 EGM	178	cachep = malloc_sizes[i].cs_dmacachep;
36555751 EGM	179	else
4b51d669	180	#endif
36555751 EGM	181	cachep = malloc_sizes[i].cs_cachep;
36555751 EGM	182
85beb586	183	ret = kmem_cache_alloc_trace(size, cachep, flags);
36555751 EGM	184
36555751 EGM	185	return ret;
2e892f43 CL	186	}
	187	return __kmalloc(size, flags);
	188	}
	189
2e892f43 CL	190	#ifdef CONFIG_NUMA
2e892f43 CL	191	extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
6193a2ff	192	extern void kmem_cache_alloc_node(struct kmem_cache , gfp_t flags, int node);
2e892f43	193
0f24f128	194	#ifdef CONFIG_TRACING
85beb586 SR	195	extern void *kmem_cache_alloc_node_trace(size_t size,
	196	struct kmem_cache *cachep,
	197	gfp_t flags,
	198	int nodeid);
36555751 EGM	199	#else
36555751 EGM	200	static __always_inline void *
85beb586 SR	201	kmem_cache_alloc_node_trace(size_t size,
	202	struct kmem_cache *cachep,
	203	gfp_t flags,
	204	int nodeid)
36555751 EGM	205	{
	206	return kmem_cache_alloc_node(cachep, flags, nodeid);
	207	}
	208	#endif
	209
	210	static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
2e892f43	211	{
36555751	212	struct kmem_cache *cachep;
36555751	213
2e892f43 CL	214	if (__builtin_constant_p(size)) {
2e892f43 CL	215	int i = 0;
6cb8f913 CL	216
	217	if (!size)
	218	return ZERO_SIZE_PTR;
	219
2e892f43 CL	220	#define CACHE(x) \
	221	if (size <= x) \
	222	goto found; \
	223	else \
	224	i++;
1c61fc40	225	#include <linux/kmalloc_sizes.h>
2e892f43	226	#undef CACHE
1cf3eb2f	227	return NULL;
2e892f43	228	found:
4b51d669 CL	229	#ifdef CONFIG_ZONE_DMA
4b51d669 CL	230	if (flags & GFP_DMA)
36555751 EGM	231	cachep = malloc_sizes[i].cs_dmacachep;
36555751 EGM	232	else
4b51d669	233	#endif
36555751 EGM	234	cachep = malloc_sizes[i].cs_cachep;
36555751 EGM	235
85beb586	236	return kmem_cache_alloc_node_trace(size, cachep, flags, node);
2e892f43 CL	237	}
	238	return __kmalloc_node(size, flags, node);
	239	}
	240
	241	#endif /* CONFIG_NUMA */
	242
	243	#endif /* _LINUX_SLAB_DEF_H */