[mirror_spl.git] / module / spl / spl-kmem.c

/*****************************************************************************\
 *  Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
 *  Copyright (C) 2007 The Regents of the University of California.
 *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
 *  Written by Brian Behlendorf <behlendorf1@llnl.gov>.
 *  UCRL-CODE-235197
 *
 *  This file is part of the SPL, Solaris Porting Layer.
 *  For details, see <http://zfsonlinux.org/>.
 *
 *  The SPL is free software; you can redistribute it and/or modify it
 *  under the terms of the GNU General Public License as published by the
 *  Free Software Foundation; either version 2 of the License, or (at your
 *  option) any later version.
 *
 *  The SPL is distributed in the hope that it will be useful, but WITHOUT
 *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 *  for more details.
 *
 *  You should have received a copy of the GNU General Public License along
 *  with the SPL.  If not, see <http://www.gnu.org/licenses/>.
 *****************************************************************************
 *  Solaris Porting Layer (SPL) Kmem Implementation.
\*****************************************************************************/

#include <sys/debug.h>
#include <sys/kmem.h>
#include <sys/vmem.h>

int
kmem_debugging(void)
{
	return 0;
}
EXPORT_SYMBOL(kmem_debugging);

char *
kmem_vasprintf(const char *fmt, va_list ap)
{
	va_list aq;
	char *ptr;

	do {
		va_copy(aq, ap);
		ptr = kvasprintf(GFP_KERNEL, fmt, aq);
		va_end(aq);
	} while (ptr == NULL);

	return ptr;
}
EXPORT_SYMBOL(kmem_vasprintf);

char *
kmem_asprintf(const char *fmt, ...)
{
	va_list ap;
	char *ptr;

	do {
		va_start(ap, fmt);
		ptr = kvasprintf(GFP_KERNEL, fmt, ap);
		va_end(ap);
	} while (ptr == NULL);

	return ptr;
}
EXPORT_SYMBOL(kmem_asprintf);

static char *
__strdup(const char *str, int flags)
{
	char *ptr;
	int n;

	n = strlen(str);
	ptr = kmalloc_nofail(n + 1, flags);
	if (ptr)
		memcpy(ptr, str, n + 1);

	return ptr;
}

char *
strdup(const char *str)
{
	return __strdup(str, KM_SLEEP);
}
EXPORT_SYMBOL(strdup);

void
strfree(char *str)
{
	kfree(str);
}
EXPORT_SYMBOL(strfree);

/*
 * Memory allocation interfaces and debugging for basic kmem_*
 * and vmem_* style memory allocation.  When DEBUG_KMEM is enabled
 * the SPL will keep track of the total memory allocated, and
 * report any memory leaked when the module is unloaded.
 */
#ifdef DEBUG_KMEM

/* Shim layer memory accounting */
# ifdef HAVE_ATOMIC64_T
atomic64_t kmem_alloc_used = ATOMIC64_INIT(0);
unsigned long long kmem_alloc_max = 0;
# else  /* HAVE_ATOMIC64_T */
atomic_t kmem_alloc_used = ATOMIC_INIT(0);
unsigned long long kmem_alloc_max = 0;
# endif /* HAVE_ATOMIC64_T */

EXPORT_SYMBOL(kmem_alloc_used);
EXPORT_SYMBOL(kmem_alloc_max);

/* When DEBUG_KMEM_TRACKING is enabled not only will total bytes be tracked
 * but also the location of every alloc and free.  When the SPL module is
 * unloaded a list of all leaked addresses and where they were allocated
 * will be dumped to the console.  Enabling this feature has a significant
 * impact on performance but it makes finding memory leaks straight forward.
 *
 * Not surprisingly with debugging enabled the xmem_locks are very highly
 * contended particularly on xfree().  If we want to run with this detailed
 * debugging enabled for anything other than debugging  we need to minimize
 * the contention by moving to a lock per xmem_table entry model.
 */
# ifdef DEBUG_KMEM_TRACKING

#  define KMEM_HASH_BITS          10
#  define KMEM_TABLE_SIZE         (1 << KMEM_HASH_BITS)

typedef struct kmem_debug {
	struct hlist_node kd_hlist;     /* Hash node linkage */
	struct list_head kd_list;       /* List of all allocations */
	void *kd_addr;                  /* Allocation pointer */
	size_t kd_size;                 /* Allocation size */
	const char *kd_func;            /* Allocation function */
	int kd_line;                    /* Allocation line */
} kmem_debug_t;

spinlock_t kmem_lock;
struct hlist_head kmem_table[KMEM_TABLE_SIZE];
struct list_head kmem_list;

EXPORT_SYMBOL(kmem_lock);
EXPORT_SYMBOL(kmem_table);
EXPORT_SYMBOL(kmem_list);

static kmem_debug_t *
kmem_del_init(spinlock_t *lock, struct hlist_head *table, int bits, const void *addr)
{
	struct hlist_head *head;
	struct hlist_node *node;
	struct kmem_debug *p;
	unsigned long flags;

	spin_lock_irqsave(lock, flags);

	head = &table[hash_ptr((void *)addr, bits)];
	hlist_for_each(node, head) {
		p = list_entry(node, struct kmem_debug, kd_hlist);
		if (p->kd_addr == addr) {
			hlist_del_init(&p->kd_hlist);
			list_del_init(&p->kd_list);
			spin_unlock_irqrestore(lock, flags);
			return p;
		}
	}

	spin_unlock_irqrestore(lock, flags);

	return (NULL);
}

void *
kmem_alloc_track(size_t size, int flags, const char *func, int line,
    int node_alloc, int node)
{
	void *ptr = NULL;
	kmem_debug_t *dptr;
	unsigned long irq_flags;

	/* Function may be called with KM_NOSLEEP so failure is possible */
	dptr = (kmem_debug_t *) kmalloc_nofail(sizeof(kmem_debug_t),
	    flags & ~__GFP_ZERO);

	if (unlikely(dptr == NULL)) {
		printk(KERN_WARNING "debug kmem_alloc(%ld, 0x%x) at %s:%d "
		    "failed (%lld/%llu)\n", sizeof(kmem_debug_t), flags,
		    func, line, kmem_alloc_used_read(), kmem_alloc_max);
	} else {
		/*
		 * Marked unlikely because we should never be doing this,
		 * we tolerate to up 2 pages but a single page is best.
		 */
		if (unlikely((size > PAGE_SIZE*2) && !(flags & KM_NODEBUG))) {
			printk(KERN_WARNING "large kmem_alloc(%llu, 0x%x) "
			    "at %s:%d failed (%lld/%llu)\n",
			    (unsigned long long)size, flags, func, line,
			    kmem_alloc_used_read(), kmem_alloc_max);
			spl_dumpstack();
		}

		/*
		 *  We use __strdup() below because the string pointed to by
		 * __FUNCTION__ might not be available by the time we want
		 * to print it since the module might have been unloaded.
		 * This can only fail in the KM_NOSLEEP case.
		 */
		dptr->kd_func = __strdup(func, flags & ~__GFP_ZERO);
		if (unlikely(dptr->kd_func == NULL)) {
			kfree(dptr);
			printk(KERN_WARNING "debug __strdup() at %s:%d "
			    "failed (%lld/%llu)\n", func, line,
			    kmem_alloc_used_read(), kmem_alloc_max);
			goto out;
		}

		/* Use the correct allocator */
		if (node_alloc) {
			ASSERT(!(flags & __GFP_ZERO));
			ptr = kmalloc_node_nofail(size, flags, node);
		} else if (flags & __GFP_ZERO) {
			ptr = kzalloc_nofail(size, flags & ~__GFP_ZERO);
		} else {
			ptr = kmalloc_nofail(size, flags);
		}

		if (unlikely(ptr == NULL)) {
			kfree(dptr->kd_func);
			kfree(dptr);
			printk(KERN_WARNING "kmem_alloc(%llu, 0x%x) "
			    "at %s:%d failed (%lld/%llu)\n",
			    (unsigned long long) size, flags, func, line,
			    kmem_alloc_used_read(), kmem_alloc_max);
			goto out;
		}

		kmem_alloc_used_add(size);
		if (unlikely(kmem_alloc_used_read() > kmem_alloc_max))
			kmem_alloc_max = kmem_alloc_used_read();

		INIT_HLIST_NODE(&dptr->kd_hlist);
		INIT_LIST_HEAD(&dptr->kd_list);

		dptr->kd_addr = ptr;
		dptr->kd_size = size;
		dptr->kd_line = line;

		spin_lock_irqsave(&kmem_lock, irq_flags);
		hlist_add_head(&dptr->kd_hlist,
		    &kmem_table[hash_ptr(ptr, KMEM_HASH_BITS)]);
		list_add_tail(&dptr->kd_list, &kmem_list);
		spin_unlock_irqrestore(&kmem_lock, irq_flags);
	}
out:
	return (ptr);
}
EXPORT_SYMBOL(kmem_alloc_track);

void
kmem_free_track(const void *ptr, size_t size)
{
	kmem_debug_t *dptr;

	ASSERTF(ptr || size > 0, "ptr: %p, size: %llu", ptr,
	    (unsigned long long) size);

	/* Must exist in hash due to kmem_alloc() */
	dptr = kmem_del_init(&kmem_lock, kmem_table, KMEM_HASH_BITS, ptr);
	ASSERT(dptr);

	/* Size must match */
	ASSERTF(dptr->kd_size == size, "kd_size (%llu) != size (%llu), "
	    "kd_func = %s, kd_line = %d\n", (unsigned long long) dptr->kd_size,
	    (unsigned long long) size, dptr->kd_func, dptr->kd_line);

	kmem_alloc_used_sub(size);
	kfree(dptr->kd_func);

	memset((void *)dptr, 0x5a, sizeof(kmem_debug_t));
	kfree(dptr);

	memset((void *)ptr, 0x5a, size);
	kfree(ptr);
}
EXPORT_SYMBOL(kmem_free_track);

# else /* DEBUG_KMEM_TRACKING */

void *
kmem_alloc_debug(size_t size, int flags, const char *func, int line,
    int node_alloc, int node)
{
	void *ptr;

	/*
	 * Marked unlikely because we should never be doing this,
	 * we tolerate to up 2 pages but a single page is best.
	 */
	if (unlikely((size > PAGE_SIZE * 2) && !(flags & KM_NODEBUG))) {
		printk(KERN_WARNING
		    "large kmem_alloc(%llu, 0x%x) at %s:%d (%lld/%llu)\n",
		    (unsigned long long)size, flags, func, line,
		    (unsigned long long)kmem_alloc_used_read(), kmem_alloc_max);
		spl_dumpstack();
	}

	/* Use the correct allocator */
	if (node_alloc) {
		ASSERT(!(flags & __GFP_ZERO));
		ptr = kmalloc_node_nofail(size, flags, node);
	} else if (flags & __GFP_ZERO) {
		ptr = kzalloc_nofail(size, flags & (~__GFP_ZERO));
	} else {
		ptr = kmalloc_nofail(size, flags);
	}

	if (unlikely(ptr == NULL)) {
		printk(KERN_WARNING
		    "kmem_alloc(%llu, 0x%x) at %s:%d failed (%lld/%llu)\n",
		    (unsigned long long)size, flags, func, line,
		    (unsigned long long)kmem_alloc_used_read(), kmem_alloc_max);
	} else {
		kmem_alloc_used_add(size);
		if (unlikely(kmem_alloc_used_read() > kmem_alloc_max))
			kmem_alloc_max = kmem_alloc_used_read();
	}

	return (ptr);
}
EXPORT_SYMBOL(kmem_alloc_debug);

void
kmem_free_debug(const void *ptr, size_t size)
{
	ASSERT(ptr || size > 0);
	kmem_alloc_used_sub(size);
	kfree(ptr);
}
EXPORT_SYMBOL(kmem_free_debug);

# endif /* DEBUG_KMEM_TRACKING */
#endif /* DEBUG_KMEM */

#if defined(DEBUG_KMEM) && defined(DEBUG_KMEM_TRACKING)
static char *
spl_sprintf_addr(kmem_debug_t *kd, char *str, int len, int min)
{
	int size = ((len - 1) < kd->kd_size) ? (len - 1) : kd->kd_size;
	int i, flag = 1;

	ASSERT(str != NULL && len >= 17);
	memset(str, 0, len);

	/* Check for a fully printable string, and while we are at
         * it place the printable characters in the passed buffer. */
	for (i = 0; i < size; i++) {
		str[i] = ((char *)(kd->kd_addr))[i];
		if (isprint(str[i])) {
			continue;
		} else {
			/* Minimum number of printable characters found
			 * to make it worthwhile to print this as ascii. */
			if (i > min)
				break;

			flag = 0;
			break;
		}
	}

	if (!flag) {
		sprintf(str, "%02x%02x%02x%02x%02x%02x%02x%02x",
		        *((uint8_t *)kd->kd_addr),
		        *((uint8_t *)kd->kd_addr + 2),
		        *((uint8_t *)kd->kd_addr + 4),
		        *((uint8_t *)kd->kd_addr + 6),
		        *((uint8_t *)kd->kd_addr + 8),
		        *((uint8_t *)kd->kd_addr + 10),
		        *((uint8_t *)kd->kd_addr + 12),
		        *((uint8_t *)kd->kd_addr + 14));
	}

	return str;
}

static int
spl_kmem_init_tracking(struct list_head *list, spinlock_t *lock, int size)
{
	int i;

	spin_lock_init(lock);
	INIT_LIST_HEAD(list);

	for (i = 0; i < size; i++)
		INIT_HLIST_HEAD(&kmem_table[i]);

	return (0);
}

static void
spl_kmem_fini_tracking(struct list_head *list, spinlock_t *lock)
{
	unsigned long flags;
	kmem_debug_t *kd;
	char str[17];

	spin_lock_irqsave(lock, flags);
	if (!list_empty(list))
		printk(KERN_WARNING "%-16s %-5s %-16s %s:%s\n", "address",
		       "size", "data", "func", "line");

	list_for_each_entry(kd, list, kd_list)
		printk(KERN_WARNING "%p %-5d %-16s %s:%d\n", kd->kd_addr,
		       (int)kd->kd_size, spl_sprintf_addr(kd, str, 17, 8),
		       kd->kd_func, kd->kd_line);

	spin_unlock_irqrestore(lock, flags);
}
#else /* DEBUG_KMEM && DEBUG_KMEM_TRACKING */
#define spl_kmem_init_tracking(list, lock, size)
#define spl_kmem_fini_tracking(list, lock)
#endif /* DEBUG_KMEM && DEBUG_KMEM_TRACKING */

int
spl_kmem_init(void)
{
	int rc = 0;

#ifdef DEBUG_KMEM
	kmem_alloc_used_set(0);
	spl_kmem_init_tracking(&kmem_list, &kmem_lock, KMEM_TABLE_SIZE);
#endif

	return (rc);
}

void
spl_kmem_fini(void)
{
#ifdef DEBUG_KMEM
	/* Display all unreclaimed memory addresses, including the
	 * allocation size and the first few bytes of what's located
	 * at that address to aid in debugging.  Performance is not
	 * a serious concern here since it is module unload time. */
	if (kmem_alloc_used_read() != 0)
		printk(KERN_WARNING "kmem leaked %ld/%llu bytes\n",
		    kmem_alloc_used_read(), kmem_alloc_max);

	spl_kmem_fini_tracking(&kmem_list, &kmem_lock);
#endif /* DEBUG_KMEM */
}
Commit	Line	Data
716154c5 BB	1	/*****************************************************************************\
	2	* Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
	3	* Copyright (C) 2007 The Regents of the University of California.
	4	* Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
	5	* Written by Brian Behlendorf <behlendorf1@llnl.gov>.
715f6251	6	* UCRL-CODE-235197
715f6251	7	*
716154c5	8	* This file is part of the SPL, Solaris Porting Layer.
3d6af2dd	9	* For details, see <http://zfsonlinux.org/>.
715f6251	10	*
716154c5 BB	11	* The SPL is free software; you can redistribute it and/or modify it
	12	* under the terms of the GNU General Public License as published by the
	13	* Free Software Foundation; either version 2 of the License, or (at your
	14	* option) any later version.
	15	*
	16	* The SPL is distributed in the hope that it will be useful, but WITHOUT
715f6251	17	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
	18	* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
	19	* for more details.
	20	*
	21	* You should have received a copy of the GNU General Public License along
716154c5 BB	22	* with the SPL. If not, see <http://www.gnu.org/licenses/>.
	23	*****************************************************************************
	24	* Solaris Porting Layer (SPL) Kmem Implementation.
	25	\*****************************************************************************/
715f6251	26
e5b9b344	27	#include <sys/debug.h>
f4b37741	28	#include <sys/kmem.h>
e5b9b344	29	#include <sys/vmem.h>
4ab13d3b	30
b868e22f BB	31	int
	32	kmem_debugging(void)
	33	{
	34	return 0;
	35	}
	36	EXPORT_SYMBOL(kmem_debugging);
	37
e6de04b7 BB	38	char *
	39	kmem_vasprintf(const char *fmt, va_list ap)
	40	{
	41	va_list aq;
	42	char *ptr;
	43
e6de04b7	44	do {
2c762de8	45	va_copy(aq, ap);
e6de04b7	46	ptr = kvasprintf(GFP_KERNEL, fmt, aq);
2c762de8	47	va_end(aq);
e6de04b7	48	} while (ptr == NULL);
e6de04b7 BB	49
	50	return ptr;
	51	}
	52	EXPORT_SYMBOL(kmem_vasprintf);
	53
b868e22f BB	54	char *
	55	kmem_asprintf(const char *fmt, ...)
	56	{
e6de04b7	57	va_list ap;
b868e22f BB	58	char *ptr;
b868e22f BB	59
b868e22f	60	do {
2c762de8	61	va_start(ap, fmt);
e6de04b7	62	ptr = kvasprintf(GFP_KERNEL, fmt, ap);
2c762de8	63	va_end(ap);
b868e22f	64	} while (ptr == NULL);
b868e22f BB	65
	66	return ptr;
	67	}
	68	EXPORT_SYMBOL(kmem_asprintf);
	69
10129680 BB	70	static char *
	71	__strdup(const char *str, int flags)
	72	{
	73	char *ptr;
	74	int n;
	75
	76	n = strlen(str);
	77	ptr = kmalloc_nofail(n + 1, flags);
	78	if (ptr)
	79	memcpy(ptr, str, n + 1);
	80
	81	return ptr;
	82	}
	83
	84	char *
	85	strdup(const char *str)
	86	{
	87	return __strdup(str, KM_SLEEP);
	88	}
	89	EXPORT_SYMBOL(strdup);
	90
	91	void
	92	strfree(char *str)
	93	{
41f84a8d	94	kfree(str);
10129680 BB	95	}
	96	EXPORT_SYMBOL(strfree);
	97
f1ca4da6	98	/*
2fb9b26a	99	* Memory allocation interfaces and debugging for basic kmem_*
055ffd98 BB	100	* and vmem_* style memory allocation. When DEBUG_KMEM is enabled
	101	* the SPL will keep track of the total memory allocated, and
	102	* report any memory leaked when the module is unloaded.
f1ca4da6	103	*/
f1ca4da6	104	#ifdef DEBUG_KMEM
d04c8a56	105
f1ca4da6	106	/* Shim layer memory accounting */
d04c8a56	107	# ifdef HAVE_ATOMIC64_T
550f1705	108	atomic64_t kmem_alloc_used = ATOMIC64_INIT(0);
a0f6da3d	109	unsigned long long kmem_alloc_max = 0;
10129680	110	# else /* HAVE_ATOMIC64_T */
d04c8a56 BB	111	atomic_t kmem_alloc_used = ATOMIC_INIT(0);
d04c8a56 BB	112	unsigned long long kmem_alloc_max = 0;
10129680	113	# endif /* HAVE_ATOMIC64_T */
79b31f36	114
ff449ac4	115	EXPORT_SYMBOL(kmem_alloc_used);
ff449ac4	116	EXPORT_SYMBOL(kmem_alloc_max);
ff449ac4	117
055ffd98 BB	118	/* When DEBUG_KMEM_TRACKING is enabled not only will total bytes be tracked
	119	* but also the location of every alloc and free. When the SPL module is
	120	* unloaded a list of all leaked addresses and where they were allocated
	121	* will be dumped to the console. Enabling this feature has a significant
	122	* impact on performance but it makes finding memory leaks straight forward.
	123	*
	124	* Not surprisingly with debugging enabled the xmem_locks are very highly
	125	* contended particularly on xfree(). If we want to run with this detailed
	126	* debugging enabled for anything other than debugging we need to minimize
	127	* the contention by moving to a lock per xmem_table entry model.
a0f6da3d	128	*/
055ffd98	129	# ifdef DEBUG_KMEM_TRACKING
a0f6da3d	130
	131	# define KMEM_HASH_BITS 10
	132	# define KMEM_TABLE_SIZE (1 << KMEM_HASH_BITS)
	133
a0f6da3d	134	typedef struct kmem_debug {
	135	struct hlist_node kd_hlist; /* Hash node linkage */
	136	struct list_head kd_list; /* List of all allocations */
	137	void kd_addr; / Allocation pointer */
	138	size_t kd_size; /* Allocation size */
	139	const char kd_func; / Allocation function */
	140	int kd_line; /* Allocation line */
	141	} kmem_debug_t;
	142
d6a26c6a	143	spinlock_t kmem_lock;
	144	struct hlist_head kmem_table[KMEM_TABLE_SIZE];
	145	struct list_head kmem_list;
	146
d6a26c6a	147	EXPORT_SYMBOL(kmem_lock);
	148	EXPORT_SYMBOL(kmem_table);
	149	EXPORT_SYMBOL(kmem_list);
	150
a0f6da3d	151	static kmem_debug_t *
973e8269	152	kmem_del_init(spinlock_t lock, struct hlist_head table, int bits, const void *addr)
a0f6da3d	153	{
	154	struct hlist_head *head;
	155	struct hlist_node *node;
	156	struct kmem_debug *p;
	157	unsigned long flags;
a0f6da3d	158
	159	spin_lock_irqsave(lock, flags);
	160
b1424add BB	161	head = &table[hash_ptr((void *)addr, bits)];
	162	hlist_for_each(node, head) {
	163	p = list_entry(node, struct kmem_debug, kd_hlist);
a0f6da3d	164	if (p->kd_addr == addr) {
	165	hlist_del_init(&p->kd_hlist);
	166	list_del_init(&p->kd_list);
	167	spin_unlock_irqrestore(lock, flags);
	168	return p;
	169	}
	170	}
	171
	172	spin_unlock_irqrestore(lock, flags);
	173
8d9a23e8	174	return (NULL);
a0f6da3d	175	}
	176
	177	void *
	178	kmem_alloc_track(size_t size, int flags, const char *func, int line,
	179	int node_alloc, int node)
	180	{
	181	void *ptr = NULL;
	182	kmem_debug_t *dptr;
	183	unsigned long irq_flags;
a0f6da3d	184
10129680	185	/* Function may be called with KM_NOSLEEP so failure is possible */
c89fdee4	186	dptr = (kmem_debug_t *) kmalloc_nofail(sizeof(kmem_debug_t),
a0f6da3d	187	flags & ~__GFP_ZERO);
a0f6da3d	188
10129680	189	if (unlikely(dptr == NULL)) {
8d9a23e8 BB	190	printk(KERN_WARNING "debug kmem_alloc(%ld, 0x%x) at %s:%d "
	191	"failed (%lld/%llu)\n", sizeof(kmem_debug_t), flags,
	192	func, line, kmem_alloc_used_read(), kmem_alloc_max);
a0f6da3d	193	} else {
10129680 BB	194	/*
	195	* Marked unlikely because we should never be doing this,
	196	* we tolerate to up 2 pages but a single page is best.
	197	*/
23d91792	198	if (unlikely((size > PAGE_SIZE*2) && !(flags & KM_NODEBUG))) {
8d9a23e8 BB	199	printk(KERN_WARNING "large kmem_alloc(%llu, 0x%x) "
	200	"at %s:%d failed (%lld/%llu)\n",
	201	(unsigned long long)size, flags, func, line,
d04c8a56	202	kmem_alloc_used_read(), kmem_alloc_max);
8d9a23e8	203	spl_dumpstack();
5198ea0e	204	}
a0f6da3d	205
10129680 BB	206	/*
10129680 BB	207	* We use __strdup() below because the string pointed to by
c8e60837	208	* __FUNCTION__ might not be available by the time we want
10129680 BB	209	* to print it since the module might have been unloaded.
	210	* This can only fail in the KM_NOSLEEP case.
	211	*/
	212	dptr->kd_func = __strdup(func, flags & ~__GFP_ZERO);
c8e60837	213	if (unlikely(dptr->kd_func == NULL)) {
c8e60837	214	kfree(dptr);
8d9a23e8 BB	215	printk(KERN_WARNING "debug __strdup() at %s:%d "
	216	"failed (%lld/%llu)\n", func, line,
	217	kmem_alloc_used_read(), kmem_alloc_max);
c8e60837	218	goto out;
	219	}
	220
a0f6da3d	221	/* Use the correct allocator */
	222	if (node_alloc) {
	223	ASSERT(!(flags & __GFP_ZERO));
c89fdee4	224	ptr = kmalloc_node_nofail(size, flags, node);
a0f6da3d	225	} else if (flags & __GFP_ZERO) {
c89fdee4	226	ptr = kzalloc_nofail(size, flags & ~__GFP_ZERO);
a0f6da3d	227	} else {
c89fdee4	228	ptr = kmalloc_nofail(size, flags);
a0f6da3d	229	}
	230
	231	if (unlikely(ptr == NULL)) {
c8e60837	232	kfree(dptr->kd_func);
a0f6da3d	233	kfree(dptr);
8d9a23e8 BB	234	printk(KERN_WARNING "kmem_alloc(%llu, 0x%x) "
8d9a23e8 BB	235	"at %s:%d failed (%lld/%llu)\n",
3cb77549	236	(unsigned long long) size, flags, func, line,
d04c8a56	237	kmem_alloc_used_read(), kmem_alloc_max);
a0f6da3d	238	goto out;
	239	}
	240
d04c8a56 BB	241	kmem_alloc_used_add(size);
	242	if (unlikely(kmem_alloc_used_read() > kmem_alloc_max))
	243	kmem_alloc_max = kmem_alloc_used_read();
a0f6da3d	244
	245	INIT_HLIST_NODE(&dptr->kd_hlist);
	246	INIT_LIST_HEAD(&dptr->kd_list);
	247
	248	dptr->kd_addr = ptr;
	249	dptr->kd_size = size;
a0f6da3d	250	dptr->kd_line = line;
	251
	252	spin_lock_irqsave(&kmem_lock, irq_flags);
b1424add	253	hlist_add_head(&dptr->kd_hlist,
a0f6da3d	254	&kmem_table[hash_ptr(ptr, KMEM_HASH_BITS)]);
	255	list_add_tail(&dptr->kd_list, &kmem_list);
	256	spin_unlock_irqrestore(&kmem_lock, irq_flags);
a0f6da3d	257	}
a0f6da3d	258	out:
8d9a23e8	259	return (ptr);
a0f6da3d	260	}
	261	EXPORT_SYMBOL(kmem_alloc_track);
	262
	263	void
973e8269	264	kmem_free_track(const void *ptr, size_t size)
a0f6da3d	265	{
a0f6da3d	266	kmem_debug_t *dptr;
a0f6da3d	267
	268	ASSERTF(ptr \|\| size > 0, "ptr: %p, size: %llu", ptr,
	269	(unsigned long long) size);
	270
10129680	271	/* Must exist in hash due to kmem_alloc() */
8d9a23e8	272	dptr = kmem_del_init(&kmem_lock, kmem_table, KMEM_HASH_BITS, ptr);
10129680	273	ASSERT(dptr);
a0f6da3d	274
	275	/* Size must match */
	276	ASSERTF(dptr->kd_size == size, "kd_size (%llu) != size (%llu), "
	277	"kd_func = %s, kd_line = %d\n", (unsigned long long) dptr->kd_size,
	278	(unsigned long long) size, dptr->kd_func, dptr->kd_line);
	279
d04c8a56	280	kmem_alloc_used_sub(size);
c8e60837	281	kfree(dptr->kd_func);
c8e60837	282
b1424add	283	memset((void *)dptr, 0x5a, sizeof(kmem_debug_t));
a0f6da3d	284	kfree(dptr);
a0f6da3d	285
b1424add	286	memset((void *)ptr, 0x5a, size);
a0f6da3d	287	kfree(ptr);
a0f6da3d	288	}
	289	EXPORT_SYMBOL(kmem_free_track);
	290
a0f6da3d	291	# else /* DEBUG_KMEM_TRACKING */
	292
	293	void *
	294	kmem_alloc_debug(size_t size, int flags, const char *func, int line,
	295	int node_alloc, int node)
	296	{
	297	void *ptr;
a0f6da3d	298
10129680 BB	299	/*
	300	* Marked unlikely because we should never be doing this,
	301	* we tolerate to up 2 pages but a single page is best.
	302	*/
23d91792	303	if (unlikely((size > PAGE_SIZE * 2) && !(flags & KM_NODEBUG))) {
8d9a23e8	304	printk(KERN_WARNING
10129680	305	"large kmem_alloc(%llu, 0x%x) at %s:%d (%lld/%llu)\n",
8d9a23e8 BB	306	(unsigned long long)size, flags, func, line,
	307	(unsigned long long)kmem_alloc_used_read(), kmem_alloc_max);
	308	spl_dumpstack();
5198ea0e	309	}
a0f6da3d	310
	311	/* Use the correct allocator */
	312	if (node_alloc) {
	313	ASSERT(!(flags & __GFP_ZERO));
c89fdee4	314	ptr = kmalloc_node_nofail(size, flags, node);
a0f6da3d	315	} else if (flags & __GFP_ZERO) {
c89fdee4	316	ptr = kzalloc_nofail(size, flags & (~__GFP_ZERO));
a0f6da3d	317	} else {
c89fdee4	318	ptr = kmalloc_nofail(size, flags);
a0f6da3d	319	}
a0f6da3d	320
10129680	321	if (unlikely(ptr == NULL)) {
8d9a23e8	322	printk(KERN_WARNING
3cb77549	323	"kmem_alloc(%llu, 0x%x) at %s:%d failed (%lld/%llu)\n",
8d9a23e8 BB	324	(unsigned long long)size, flags, func, line,
8d9a23e8 BB	325	(unsigned long long)kmem_alloc_used_read(), kmem_alloc_max);
a0f6da3d	326	} else {
d04c8a56 BB	327	kmem_alloc_used_add(size);
	328	if (unlikely(kmem_alloc_used_read() > kmem_alloc_max))
	329	kmem_alloc_max = kmem_alloc_used_read();
a0f6da3d	330	}
10129680	331
8d9a23e8	332	return (ptr);
a0f6da3d	333	}
	334	EXPORT_SYMBOL(kmem_alloc_debug);
	335
	336	void
973e8269	337	kmem_free_debug(const void *ptr, size_t size)
a0f6da3d	338	{
8d9a23e8	339	ASSERT(ptr \|\| size > 0);
d04c8a56	340	kmem_alloc_used_sub(size);
a0f6da3d	341	kfree(ptr);
a0f6da3d	342	}
	343	EXPORT_SYMBOL(kmem_free_debug);
	344
a0f6da3d	345	# endif /* DEBUG_KMEM_TRACKING */
	346	#endif /* DEBUG_KMEM */
	347
e5b9b344 BB	348	#if defined(DEBUG_KMEM) && defined(DEBUG_KMEM_TRACKING)
	349	static char *
	350	spl_sprintf_addr(kmem_debug_t kd, char str, int len, int min)
fece7c99	351	{
e5b9b344 BB	352	int size = ((len - 1) < kd->kd_size) ? (len - 1) : kd->kd_size;
e5b9b344 BB	353	int i, flag = 1;
fece7c99	354
e5b9b344 BB	355	ASSERT(str != NULL && len >= 17);
e5b9b344 BB	356	memset(str, 0, len);
fece7c99	357
e5b9b344 BB	358	/* Check for a fully printable string, and while we are at
	359	* it place the printable characters in the passed buffer. */
	360	for (i = 0; i < size; i++) {
	361	str[i] = ((char *)(kd->kd_addr))[i];
	362	if (isprint(str[i])) {
	363	continue;
	364	} else {
	365	/* Minimum number of printable characters found
	366	* to make it worthwhile to print this as ascii. */
	367	if (i > min)
	368	break;
8b45dda2	369
e5b9b344 BB	370	flag = 0;
	371	break;
	372	}
	373	}
06089b9e	374
e5b9b344 BB	375	if (!flag) {
	376	sprintf(str, "%02x%02x%02x%02x%02x%02x%02x%02x",
	377	((uint8_t )kd->kd_addr),
	378	((uint8_t )kd->kd_addr + 2),
	379	((uint8_t )kd->kd_addr + 4),
	380	((uint8_t )kd->kd_addr + 6),
	381	((uint8_t )kd->kd_addr + 8),
	382	((uint8_t )kd->kd_addr + 10),
	383	((uint8_t )kd->kd_addr + 12),
	384	((uint8_t )kd->kd_addr + 14));
	385	}
8b45dda2	386
e5b9b344	387	return str;
8b45dda2 BB	388	}
8b45dda2 BB	389
e5b9b344 BB	390	static int
e5b9b344 BB	391	spl_kmem_init_tracking(struct list_head list, spinlock_t lock, int size)
8b45dda2	392	{
e5b9b344	393	int i;
8b45dda2	394
e5b9b344 BB	395	spin_lock_init(lock);
e5b9b344 BB	396	INIT_LIST_HEAD(list);
8b45dda2	397
e5b9b344 BB	398	for (i = 0; i < size; i++)
e5b9b344 BB	399	INIT_HLIST_HEAD(&kmem_table[i]);
8b45dda2	400
e5b9b344	401	return (0);
fece7c99	402	}
fece7c99	403
e5b9b344 BB	404	static void
e5b9b344 BB	405	spl_kmem_fini_tracking(struct list_head list, spinlock_t lock)
fece7c99	406	{
e5b9b344 BB	407	unsigned long flags;
	408	kmem_debug_t *kd;
	409	char str[17];
a1502d76	410
e5b9b344 BB	411	spin_lock_irqsave(lock, flags);
	412	if (!list_empty(list))
	413	printk(KERN_WARNING "%-16s %-5s %-16s %s:%s\n", "address",
	414	"size", "data", "func", "line");
fece7c99	415
e5b9b344 BB	416	list_for_each_entry(kd, list, kd_list)
	417	printk(KERN_WARNING "%p %-5d %-16s %s:%d\n", kd->kd_addr,
	418	(int)kd->kd_size, spl_sprintf_addr(kd, str, 17, 8),
	419	kd->kd_func, kd->kd_line);
fece7c99	420
e5b9b344	421	spin_unlock_irqrestore(lock, flags);
fece7c99	422	}
e5b9b344 BB	423	#else /* DEBUG_KMEM && DEBUG_KMEM_TRACKING */
	424	#define spl_kmem_init_tracking(list, lock, size)
	425	#define spl_kmem_fini_tracking(list, lock)
	426	#endif /* DEBUG_KMEM && DEBUG_KMEM_TRACKING */
fece7c99	427
e5b9b344 BB	428	int
e5b9b344 BB	429	spl_kmem_init(void)
ea3e6ca9	430	{
e5b9b344	431	int rc = 0;
d6a26c6a	432
e5b9b344 BB	433	#ifdef DEBUG_KMEM
	434	kmem_alloc_used_set(0);
	435	spl_kmem_init_tracking(&kmem_list, &kmem_lock, KMEM_TABLE_SIZE);
	436	#endif
f1ca4da6	437
e5b9b344	438	return (rc);
2fb9b26a	439	}
d6a26c6a	440
e5b9b344 BB	441	void
e5b9b344 BB	442	spl_kmem_fini(void)
2fb9b26a	443	{
ff449ac4	444	#ifdef DEBUG_KMEM
	445	/* Display all unreclaimed memory addresses, including the
	446	* allocation size and the first few bytes of what's located
	447	* at that address to aid in debugging. Performance is not
	448	* a serious concern here since it is module unload time. */
d04c8a56	449	if (kmem_alloc_used_read() != 0)
8d9a23e8	450	printk(KERN_WARNING "kmem leaked %ld/%llu bytes\n",
3cb77549	451	kmem_alloc_used_read(), kmem_alloc_max);
ff449ac4	452
ff449ac4	453	spl_kmem_fini_tracking(&kmem_list, &kmem_lock);
ff449ac4	454	#endif /* DEBUG_KMEM */
5d86345d	455	}