[mirror_ubuntu-focal-kernel.git] / kernel / profile.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 *  linux/kernel/profile.c
 *  Simple profiling. Manages a direct-mapped profile hit count buffer,
 *  with configurable resolution, support for restricting the cpus on
 *  which profiling is done, and switching between cpu time and
 *  schedule() calls via kernel command line parameters passed at boot.
 *
 *  Scheduler profiling support, Arjan van de Ven and Ingo Molnar,
 *	Red Hat, July 2004
 *  Consolidation of architecture support code for profiling,
 *	Nadia Yvette Chambers, Oracle, July 2004
 *  Amortized hit count accounting via per-cpu open-addressed hashtables
 *	to resolve timer interrupt livelocks, Nadia Yvette Chambers,
 *	Oracle, 2004
 */

#include <linux/export.h>
#include <linux/profile.h>
#include <linux/memblock.h>
#include <linux/notifier.h>
#include <linux/mm.h>
#include <linux/cpumask.h>
#include <linux/cpu.h>
#include <linux/highmem.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <linux/vmalloc.h>
#include <linux/sched/stat.h>

#include <asm/sections.h>
#include <asm/irq_regs.h>
#include <asm/ptrace.h>

struct profile_hit {
	u32 pc, hits;
};
#define PROFILE_GRPSHIFT	3
#define PROFILE_GRPSZ		(1 << PROFILE_GRPSHIFT)
#define NR_PROFILE_HIT		(PAGE_SIZE/sizeof(struct profile_hit))
#define NR_PROFILE_GRP		(NR_PROFILE_HIT/PROFILE_GRPSZ)

static atomic_t *prof_buffer;
static unsigned long prof_len;
static unsigned short int prof_shift;

int prof_on __read_mostly;
EXPORT_SYMBOL_GPL(prof_on);

static cpumask_var_t prof_cpu_mask;
#if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
static DEFINE_PER_CPU(struct profile_hit *[2], cpu_profile_hits);
static DEFINE_PER_CPU(int, cpu_profile_flip);
static DEFINE_MUTEX(profile_flip_mutex);
#endif /* CONFIG_SMP */

int profile_setup(char *str)
{
	static const char schedstr[] = "schedule";
	static const char sleepstr[] = "sleep";
	static const char kvmstr[] = "kvm";
	int par;

	if (!strncmp(str, sleepstr, strlen(sleepstr))) {
#ifdef CONFIG_SCHEDSTATS
		force_schedstat_enabled();
		prof_on = SLEEP_PROFILING;
		if (str[strlen(sleepstr)] == ',')
			str += strlen(sleepstr) + 1;
		if (get_option(&str, &par))
			prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
		pr_info("kernel sleep profiling enabled (shift: %u)\n",
			prof_shift);
#else
		pr_warn("kernel sleep profiling requires CONFIG_SCHEDSTATS\n");
#endif /* CONFIG_SCHEDSTATS */
	} else if (!strncmp(str, schedstr, strlen(schedstr))) {
		prof_on = SCHED_PROFILING;
		if (str[strlen(schedstr)] == ',')
			str += strlen(schedstr) + 1;
		if (get_option(&str, &par))
			prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
		pr_info("kernel schedule profiling enabled (shift: %u)\n",
			prof_shift);
	} else if (!strncmp(str, kvmstr, strlen(kvmstr))) {
		prof_on = KVM_PROFILING;
		if (str[strlen(kvmstr)] == ',')
			str += strlen(kvmstr) + 1;
		if (get_option(&str, &par))
			prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
		pr_info("kernel KVM profiling enabled (shift: %u)\n",
			prof_shift);
	} else if (get_option(&str, &par)) {
		prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
		prof_on = CPU_PROFILING;
		pr_info("kernel profiling enabled (shift: %u)\n",
			prof_shift);
	}
	return 1;
}
__setup("profile=", profile_setup);


int __ref profile_init(void)
{
	int buffer_bytes;
	if (!prof_on)
		return 0;

	/* only text is profiled */
	prof_len = (_etext - _stext) >> prof_shift;
	buffer_bytes = prof_len*sizeof(atomic_t);

	if (!alloc_cpumask_var(&prof_cpu_mask, GFP_KERNEL))
		return -ENOMEM;

	cpumask_copy(prof_cpu_mask, cpu_possible_mask);

	prof_buffer = kzalloc(buffer_bytes, GFP_KERNEL|__GFP_NOWARN);
	if (prof_buffer)
		return 0;

	prof_buffer = alloc_pages_exact(buffer_bytes,
					GFP_KERNEL|__GFP_ZERO|__GFP_NOWARN);
	if (prof_buffer)
		return 0;

	prof_buffer = vzalloc(buffer_bytes);
	if (prof_buffer)
		return 0;

	free_cpumask_var(prof_cpu_mask);
	return -ENOMEM;
}

/* Profile event notifications */

static BLOCKING_NOTIFIER_HEAD(task_exit_notifier);
static ATOMIC_NOTIFIER_HEAD(task_free_notifier);
static BLOCKING_NOTIFIER_HEAD(munmap_notifier);

void profile_task_exit(struct task_struct *task)
{
	blocking_notifier_call_chain(&task_exit_notifier, 0, task);
}

int profile_handoff_task(struct task_struct *task)
{
	int ret;
	ret = atomic_notifier_call_chain(&task_free_notifier, 0, task);
	return (ret == NOTIFY_OK) ? 1 : 0;
}

void profile_munmap(unsigned long addr)
{
	blocking_notifier_call_chain(&munmap_notifier, 0, (void *)addr);
}

int task_handoff_register(struct notifier_block *n)
{
	return atomic_notifier_chain_register(&task_free_notifier, n);
}
EXPORT_SYMBOL_GPL(task_handoff_register);

int task_handoff_unregister(struct notifier_block *n)
{
	return atomic_notifier_chain_unregister(&task_free_notifier, n);
}
EXPORT_SYMBOL_GPL(task_handoff_unregister);

int profile_event_register(enum profile_type type, struct notifier_block *n)
{
	int err = -EINVAL;

	switch (type) {
	case PROFILE_TASK_EXIT:
		err = blocking_notifier_chain_register(
				&task_exit_notifier, n);
		break;
	case PROFILE_MUNMAP:
		err = blocking_notifier_chain_register(
				&munmap_notifier, n);
		break;
	}

	return err;
}
EXPORT_SYMBOL_GPL(profile_event_register);

int profile_event_unregister(enum profile_type type, struct notifier_block *n)
{
	int err = -EINVAL;

	switch (type) {
	case PROFILE_TASK_EXIT:
		err = blocking_notifier_chain_unregister(
				&task_exit_notifier, n);
		break;
	case PROFILE_MUNMAP:
		err = blocking_notifier_chain_unregister(
				&munmap_notifier, n);
		break;
	}

	return err;
}
EXPORT_SYMBOL_GPL(profile_event_unregister);

#if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
/*
 * Each cpu has a pair of open-addressed hashtables for pending
 * profile hits. read_profile() IPI's all cpus to request them
 * to flip buffers and flushes their contents to prof_buffer itself.
 * Flip requests are serialized by the profile_flip_mutex. The sole
 * use of having a second hashtable is for avoiding cacheline
 * contention that would otherwise happen during flushes of pending
 * profile hits required for the accuracy of reported profile hits
 * and so resurrect the interrupt livelock issue.
 *
 * The open-addressed hashtables are indexed by profile buffer slot
 * and hold the number of pending hits to that profile buffer slot on
 * a cpu in an entry. When the hashtable overflows, all pending hits
 * are accounted to their corresponding profile buffer slots with
 * atomic_add() and the hashtable emptied. As numerous pending hits
 * may be accounted to a profile buffer slot in a hashtable entry,
 * this amortizes a number of atomic profile buffer increments likely
 * to be far larger than the number of entries in the hashtable,
 * particularly given that the number of distinct profile buffer
 * positions to which hits are accounted during short intervals (e.g.
 * several seconds) is usually very small. Exclusion from buffer
 * flipping is provided by interrupt disablement (note that for
 * SCHED_PROFILING or SLEEP_PROFILING profile_hit() may be called from
 * process context).
 * The hash function is meant to be lightweight as opposed to strong,
 * and was vaguely inspired by ppc64 firmware-supported inverted
 * pagetable hash functions, but uses a full hashtable full of finite
 * collision chains, not just pairs of them.
 *
 * -- nyc
 */
static void __profile_flip_buffers(void *unused)
{
	int cpu = smp_processor_id();

	per_cpu(cpu_profile_flip, cpu) = !per_cpu(cpu_profile_flip, cpu);
}

static void profile_flip_buffers(void)
{
	int i, j, cpu;

	mutex_lock(&profile_flip_mutex);
	j = per_cpu(cpu_profile_flip, get_cpu());
	put_cpu();
	on_each_cpu(__profile_flip_buffers, NULL, 1);
	for_each_online_cpu(cpu) {
		struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[j];
		for (i = 0; i < NR_PROFILE_HIT; ++i) {
			if (!hits[i].hits) {
				if (hits[i].pc)
					hits[i].pc = 0;
				continue;
			}
			atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
			hits[i].hits = hits[i].pc = 0;
		}
	}
	mutex_unlock(&profile_flip_mutex);
}

static void profile_discard_flip_buffers(void)
{
	int i, cpu;

	mutex_lock(&profile_flip_mutex);
	i = per_cpu(cpu_profile_flip, get_cpu());
	put_cpu();
	on_each_cpu(__profile_flip_buffers, NULL, 1);
	for_each_online_cpu(cpu) {
		struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[i];
		memset(hits, 0, NR_PROFILE_HIT*sizeof(struct profile_hit));
	}
	mutex_unlock(&profile_flip_mutex);
}

static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
{
	unsigned long primary, secondary, flags, pc = (unsigned long)__pc;
	int i, j, cpu;
	struct profile_hit *hits;

	pc = min((pc - (unsigned long)_stext) >> prof_shift, prof_len - 1);
	i = primary = (pc & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
	secondary = (~(pc << 1) & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
	cpu = get_cpu();
	hits = per_cpu(cpu_profile_hits, cpu)[per_cpu(cpu_profile_flip, cpu)];
	if (!hits) {
		put_cpu();
		return;
	}
	/*
	 * We buffer the global profiler buffer into a per-CPU
	 * queue and thus reduce the number of global (and possibly
	 * NUMA-alien) accesses. The write-queue is self-coalescing:
	 */
	local_irq_save(flags);
	do {
		for (j = 0; j < PROFILE_GRPSZ; ++j) {
			if (hits[i + j].pc == pc) {
				hits[i + j].hits += nr_hits;
				goto out;
			} else if (!hits[i + j].hits) {
				hits[i + j].pc = pc;
				hits[i + j].hits = nr_hits;
				goto out;
			}
		}
		i = (i + secondary) & (NR_PROFILE_HIT - 1);
	} while (i != primary);

	/*
	 * Add the current hit(s) and flush the write-queue out
	 * to the global buffer:
	 */
	atomic_add(nr_hits, &prof_buffer[pc]);
	for (i = 0; i < NR_PROFILE_HIT; ++i) {
		atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
		hits[i].pc = hits[i].hits = 0;
	}
out:
	local_irq_restore(flags);
	put_cpu();
}

static int profile_dead_cpu(unsigned int cpu)
{
	struct page *page;
	int i;

	if (prof_cpu_mask != NULL)
		cpumask_clear_cpu(cpu, prof_cpu_mask);

	for (i = 0; i < 2; i++) {
		if (per_cpu(cpu_profile_hits, cpu)[i]) {
			page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[i]);
			per_cpu(cpu_profile_hits, cpu)[i] = NULL;
			__free_page(page);
		}
	}
	return 0;
}

static int profile_prepare_cpu(unsigned int cpu)
{
	int i, node = cpu_to_mem(cpu);
	struct page *page;

	per_cpu(cpu_profile_flip, cpu) = 0;

	for (i = 0; i < 2; i++) {
		if (per_cpu(cpu_profile_hits, cpu)[i])
			continue;

		page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
		if (!page) {
			profile_dead_cpu(cpu);
			return -ENOMEM;
		}
		per_cpu(cpu_profile_hits, cpu)[i] = page_address(page);

	}
	return 0;
}

static int profile_online_cpu(unsigned int cpu)
{
	if (prof_cpu_mask != NULL)
		cpumask_set_cpu(cpu, prof_cpu_mask);

	return 0;
}

#else /* !CONFIG_SMP */
#define profile_flip_buffers()		do { } while (0)
#define profile_discard_flip_buffers()	do { } while (0)

static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
{
	unsigned long pc;
	pc = ((unsigned long)__pc - (unsigned long)_stext) >> prof_shift;
	atomic_add(nr_hits, &prof_buffer[min(pc, prof_len - 1)]);
}
#endif /* !CONFIG_SMP */

void profile_hits(int type, void *__pc, unsigned int nr_hits)
{
	if (prof_on != type || !prof_buffer)
		return;
	do_profile_hits(type, __pc, nr_hits);
}
EXPORT_SYMBOL_GPL(profile_hits);

void profile_tick(int type)
{
	struct pt_regs *regs = get_irq_regs();

	if (!user_mode(regs) && prof_cpu_mask != NULL &&
	    cpumask_test_cpu(smp_processor_id(), prof_cpu_mask))
		profile_hit(type, (void *)profile_pc(regs));
}

#ifdef CONFIG_PROC_FS
#include <linux/proc_fs.h>
#include <linux/seq_file.h>
#include <linux/uaccess.h>

static int prof_cpu_mask_proc_show(struct seq_file *m, void *v)
{
	seq_printf(m, "%*pb\n", cpumask_pr_args(prof_cpu_mask));
	return 0;
}

static int prof_cpu_mask_proc_open(struct inode *inode, struct file *file)
{
	return single_open(file, prof_cpu_mask_proc_show, NULL);
}

static ssize_t prof_cpu_mask_proc_write(struct file *file,
	const char __user *buffer, size_t count, loff_t *pos)
{
	cpumask_var_t new_value;
	int err;

	if (!alloc_cpumask_var(&new_value, GFP_KERNEL))
		return -ENOMEM;

	err = cpumask_parse_user(buffer, count, new_value);
	if (!err) {
		cpumask_copy(prof_cpu_mask, new_value);
		err = count;
	}
	free_cpumask_var(new_value);
	return err;
}

static const struct file_operations prof_cpu_mask_proc_fops = {
	.open		= prof_cpu_mask_proc_open,
	.read		= seq_read,
	.llseek		= seq_lseek,
	.release	= single_release,
	.write		= prof_cpu_mask_proc_write,
};

void create_prof_cpu_mask(void)
{
	/* create /proc/irq/prof_cpu_mask */
	proc_create("irq/prof_cpu_mask", 0600, NULL, &prof_cpu_mask_proc_fops);
}

/*
 * This function accesses profiling information. The returned data is
 * binary: the sampling step and the actual contents of the profile
 * buffer. Use of the program readprofile is recommended in order to
 * get meaningful info out of these data.
 */
static ssize_t
read_profile(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
	unsigned long p = *ppos;
	ssize_t read;
	char *pnt;
	unsigned long sample_step = 1UL << prof_shift;

	profile_flip_buffers();
	if (p >= (prof_len+1)*sizeof(unsigned int))
		return 0;
	if (count > (prof_len+1)*sizeof(unsigned int) - p)
		count = (prof_len+1)*sizeof(unsigned int) - p;
	read = 0;

	while (p < sizeof(unsigned int) && count > 0) {
		if (put_user(*((char *)(&sample_step)+p), buf))
			return -EFAULT;
		buf++; p++; count--; read++;
	}
	pnt = (char *)prof_buffer + p - sizeof(atomic_t);
	if (copy_to_user(buf, (void *)pnt, count))
		return -EFAULT;
	read += count;
	*ppos += read;
	return read;
}

/*
 * Writing to /proc/profile resets the counters
 *
 * Writing a 'profiling multiplier' value into it also re-sets the profiling
 * interrupt frequency, on architectures that support this.
 */
static ssize_t write_profile(struct file *file, const char __user *buf,
			     size_t count, loff_t *ppos)
{
#ifdef CONFIG_SMP
	extern int setup_profiling_timer(unsigned int multiplier);

	if (count == sizeof(int)) {
		unsigned int multiplier;

		if (copy_from_user(&multiplier, buf, sizeof(int)))
			return -EFAULT;

		if (setup_profiling_timer(multiplier))
			return -EINVAL;
	}
#endif
	profile_discard_flip_buffers();
	memset(prof_buffer, 0, prof_len * sizeof(atomic_t));
	return count;
}

static const struct file_operations proc_profile_operations = {
	.read		= read_profile,
	.write		= write_profile,
	.llseek		= default_llseek,
};

int __ref create_proc_profile(void)
{
	struct proc_dir_entry *entry;
#ifdef CONFIG_SMP
	enum cpuhp_state online_state;
#endif

	int err = 0;

	if (!prof_on)
		return 0;
#ifdef CONFIG_SMP
	err = cpuhp_setup_state(CPUHP_PROFILE_PREPARE, "PROFILE_PREPARE",
				profile_prepare_cpu, profile_dead_cpu);
	if (err)
		return err;

	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "AP_PROFILE_ONLINE",
				profile_online_cpu, NULL);
	if (err < 0)
		goto err_state_prep;
	online_state = err;
	err = 0;
#endif
	entry = proc_create("profile", S_IWUSR | S_IRUGO,
			    NULL, &proc_profile_operations);
	if (!entry)
		goto err_state_onl;
	proc_set_size(entry, (1 + prof_len) * sizeof(atomic_t));

	return err;
err_state_onl:
#ifdef CONFIG_SMP
	cpuhp_remove_state(online_state);
err_state_prep:
	cpuhp_remove_state(CPUHP_PROFILE_PREPARE);
#endif
	return err;
}
subsys_initcall(create_proc_profile);
#endif /* CONFIG_PROC_FS */
Commit	Line	Data
457c8996	1	// SPDX-License-Identifier: GPL-2.0-only
1da177e4 LT	2	/*
	3	* linux/kernel/profile.c
	4	* Simple profiling. Manages a direct-mapped profile hit count buffer,
	5	* with configurable resolution, support for restricting the cpus on
	6	* which profiling is done, and switching between cpu time and
	7	* schedule() calls via kernel command line parameters passed at boot.
	8	*
	9	* Scheduler profiling support, Arjan van de Ven and Ingo Molnar,
	10	* Red Hat, July 2004
	11	* Consolidation of architecture support code for profiling,
6d49e352	12	* Nadia Yvette Chambers, Oracle, July 2004
1da177e4	13	* Amortized hit count accounting via per-cpu open-addressed hashtables
6d49e352 NYC	14	* to resolve timer interrupt livelocks, Nadia Yvette Chambers,
6d49e352 NYC	15	* Oracle, 2004
1da177e4 LT	16	*/
1da177e4 LT	17
9984de1a	18	#include <linux/export.h>
1da177e4	19	#include <linux/profile.h>
57c8a661	20	#include <linux/memblock.h>
1da177e4 LT	21	#include <linux/notifier.h>
	22	#include <linux/mm.h>
	23	#include <linux/cpumask.h>
	24	#include <linux/cpu.h>
1da177e4	25	#include <linux/highmem.h>
97d1f15b	26	#include <linux/mutex.h>
22b8ce94 DH	27	#include <linux/slab.h>
22b8ce94 DH	28	#include <linux/vmalloc.h>
3905f9ad IM	29	#include <linux/sched/stat.h>
3905f9ad IM	30
1da177e4	31	#include <asm/sections.h>
7d12e780	32	#include <asm/irq_regs.h>
e8edc6e0	33	#include <asm/ptrace.h>
1da177e4 LT	34
	35	struct profile_hit {
	36	u32 pc, hits;
	37	};
	38	#define PROFILE_GRPSHIFT 3
	39	#define PROFILE_GRPSZ (1 << PROFILE_GRPSHIFT)
	40	#define NR_PROFILE_HIT (PAGE_SIZE/sizeof(struct profile_hit))
	41	#define NR_PROFILE_GRP (NR_PROFILE_HIT/PROFILE_GRPSZ)
	42
1da177e4	43	static atomic_t *prof_buffer;
455d939e PS	44	static unsigned long prof_len;
455d939e PS	45	static unsigned short int prof_shift;
07031e14	46
ece8a684	47	int prof_on __read_mostly;
07031e14 IM	48	EXPORT_SYMBOL_GPL(prof_on);
07031e14 IM	49
c309b917	50	static cpumask_var_t prof_cpu_mask;
ade356b9	51	#if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
1da177e4 LT	52	static DEFINE_PER_CPU(struct profile_hit *[2], cpu_profile_hits);
1da177e4 LT	53	static DEFINE_PER_CPU(int, cpu_profile_flip);
97d1f15b	54	static DEFINE_MUTEX(profile_flip_mutex);
1da177e4 LT	55	#endif /* CONFIG_SMP */
1da177e4 LT	56
22b8ce94	57	int profile_setup(char *str)
1da177e4	58	{
f3da64d1 FF	59	static const char schedstr[] = "schedule";
	60	static const char sleepstr[] = "sleep";
	61	static const char kvmstr[] = "kvm";
1da177e4 LT	62	int par;
1da177e4 LT	63
ece8a684	64	if (!strncmp(str, sleepstr, strlen(sleepstr))) {
b3da2a73	65	#ifdef CONFIG_SCHEDSTATS
cb251765	66	force_schedstat_enabled();
ece8a684 IM	67	prof_on = SLEEP_PROFILING;
	68	if (str[strlen(sleepstr)] == ',')
	69	str += strlen(sleepstr) + 1;
	70	if (get_option(&str, &par))
455d939e PS	71	prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
455d939e PS	72	pr_info("kernel sleep profiling enabled (shift: %u)\n",
ece8a684	73	prof_shift);
b3da2a73	74	#else
aba871f1	75	pr_warn("kernel sleep profiling requires CONFIG_SCHEDSTATS\n");
b3da2a73	76	#endif /* CONFIG_SCHEDSTATS */
a75acf85	77	} else if (!strncmp(str, schedstr, strlen(schedstr))) {
1da177e4	78	prof_on = SCHED_PROFILING;
dfaa9c94 WLII	79	if (str[strlen(schedstr)] == ',')
	80	str += strlen(schedstr) + 1;
	81	if (get_option(&str, &par))
455d939e PS	82	prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
455d939e PS	83	pr_info("kernel schedule profiling enabled (shift: %u)\n",
dfaa9c94	84	prof_shift);
07031e14 IM	85	} else if (!strncmp(str, kvmstr, strlen(kvmstr))) {
	86	prof_on = KVM_PROFILING;
	87	if (str[strlen(kvmstr)] == ',')
	88	str += strlen(kvmstr) + 1;
	89	if (get_option(&str, &par))
455d939e PS	90	prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
455d939e PS	91	pr_info("kernel KVM profiling enabled (shift: %u)\n",
07031e14	92	prof_shift);
dfaa9c94	93	} else if (get_option(&str, &par)) {
455d939e	94	prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
1da177e4	95	prof_on = CPU_PROFILING;
455d939e	96	pr_info("kernel profiling enabled (shift: %u)\n",
1da177e4 LT	97	prof_shift);
	98	}
	99	return 1;
	100	}
	101	__setup("profile=", profile_setup);
	102
	103
ce05fcc3	104	int __ref profile_init(void)
1da177e4	105	{
22b8ce94	106	int buffer_bytes;
1ad82fd5	107	if (!prof_on)
22b8ce94	108	return 0;
1ad82fd5	109
1da177e4 LT	110	/* only text is profiled */
1da177e4 LT	111	prof_len = (_etext - _stext) >> prof_shift;
22b8ce94	112	buffer_bytes = prof_len*sizeof(atomic_t);
22b8ce94	113
c309b917 RR	114	if (!alloc_cpumask_var(&prof_cpu_mask, GFP_KERNEL))
	115	return -ENOMEM;
	116
acd89579 HD	117	cpumask_copy(prof_cpu_mask, cpu_possible_mask);
acd89579 HD	118
b62f495d	119	prof_buffer = kzalloc(buffer_bytes, GFP_KERNEL\|__GFP_NOWARN);
22b8ce94 DH	120	if (prof_buffer)
	121	return 0;
	122
b62f495d MG	123	prof_buffer = alloc_pages_exact(buffer_bytes,
b62f495d MG	124	GFP_KERNEL\|__GFP_ZERO\|__GFP_NOWARN);
22b8ce94 DH	125	if (prof_buffer)
	126	return 0;
	127
559fa6e7 JJ	128	prof_buffer = vzalloc(buffer_bytes);
559fa6e7 JJ	129	if (prof_buffer)
22b8ce94 DH	130	return 0;
22b8ce94 DH	131
c309b917	132	free_cpumask_var(prof_cpu_mask);
22b8ce94	133	return -ENOMEM;
1da177e4 LT	134	}
	135
	136	/* Profile event notifications */
1ad82fd5	137
e041c683 AS	138	static BLOCKING_NOTIFIER_HEAD(task_exit_notifier);
	139	static ATOMIC_NOTIFIER_HEAD(task_free_notifier);
	140	static BLOCKING_NOTIFIER_HEAD(munmap_notifier);
1ad82fd5 PC	141
1ad82fd5 PC	142	void profile_task_exit(struct task_struct *task)
1da177e4	143	{
e041c683	144	blocking_notifier_call_chain(&task_exit_notifier, 0, task);
1da177e4	145	}
1ad82fd5 PC	146
1ad82fd5 PC	147	int profile_handoff_task(struct task_struct *task)
1da177e4 LT	148	{
1da177e4 LT	149	int ret;
e041c683	150	ret = atomic_notifier_call_chain(&task_free_notifier, 0, task);
1da177e4 LT	151	return (ret == NOTIFY_OK) ? 1 : 0;
	152	}
	153
	154	void profile_munmap(unsigned long addr)
	155	{
e041c683	156	blocking_notifier_call_chain(&munmap_notifier, 0, (void *)addr);
1da177e4 LT	157	}
1da177e4 LT	158
1ad82fd5	159	int task_handoff_register(struct notifier_block *n)
1da177e4	160	{
e041c683	161	return atomic_notifier_chain_register(&task_free_notifier, n);
1da177e4	162	}
1ad82fd5	163	EXPORT_SYMBOL_GPL(task_handoff_register);
1da177e4	164
1ad82fd5	165	int task_handoff_unregister(struct notifier_block *n)
1da177e4	166	{
e041c683	167	return atomic_notifier_chain_unregister(&task_free_notifier, n);
1da177e4	168	}
1ad82fd5	169	EXPORT_SYMBOL_GPL(task_handoff_unregister);
1da177e4	170
1ad82fd5	171	int profile_event_register(enum profile_type type, struct notifier_block *n)
1da177e4 LT	172	{
1da177e4 LT	173	int err = -EINVAL;
1ad82fd5	174
1da177e4	175	switch (type) {
1ad82fd5 PC	176	case PROFILE_TASK_EXIT:
	177	err = blocking_notifier_chain_register(
	178	&task_exit_notifier, n);
	179	break;
	180	case PROFILE_MUNMAP:
	181	err = blocking_notifier_chain_register(
	182	&munmap_notifier, n);
	183	break;
1da177e4	184	}
1ad82fd5	185
1da177e4 LT	186	return err;
1da177e4 LT	187	}
1ad82fd5	188	EXPORT_SYMBOL_GPL(profile_event_register);
1da177e4	189
1ad82fd5	190	int profile_event_unregister(enum profile_type type, struct notifier_block *n)
1da177e4 LT	191	{
1da177e4 LT	192	int err = -EINVAL;
1ad82fd5	193
1da177e4	194	switch (type) {
1ad82fd5 PC	195	case PROFILE_TASK_EXIT:
	196	err = blocking_notifier_chain_unregister(
	197	&task_exit_notifier, n);
	198	break;
	199	case PROFILE_MUNMAP:
	200	err = blocking_notifier_chain_unregister(
	201	&munmap_notifier, n);
	202	break;
1da177e4 LT	203	}
1da177e4 LT	204
1da177e4 LT	205	return err;
1da177e4 LT	206	}
1ad82fd5	207	EXPORT_SYMBOL_GPL(profile_event_unregister);
1da177e4	208
ade356b9	209	#if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
1da177e4 LT	210	/*
	211	* Each cpu has a pair of open-addressed hashtables for pending
	212	* profile hits. read_profile() IPI's all cpus to request them
	213	* to flip buffers and flushes their contents to prof_buffer itself.
	214	* Flip requests are serialized by the profile_flip_mutex. The sole
	215	* use of having a second hashtable is for avoiding cacheline
	216	* contention that would otherwise happen during flushes of pending
	217	* profile hits required for the accuracy of reported profile hits
	218	* and so resurrect the interrupt livelock issue.
	219	*
	220	* The open-addressed hashtables are indexed by profile buffer slot
	221	* and hold the number of pending hits to that profile buffer slot on
	222	* a cpu in an entry. When the hashtable overflows, all pending hits
	223	* are accounted to their corresponding profile buffer slots with
	224	* atomic_add() and the hashtable emptied. As numerous pending hits
	225	* may be accounted to a profile buffer slot in a hashtable entry,
	226	* this amortizes a number of atomic profile buffer increments likely
	227	* to be far larger than the number of entries in the hashtable,
	228	* particularly given that the number of distinct profile buffer
	229	* positions to which hits are accounted during short intervals (e.g.
	230	* several seconds) is usually very small. Exclusion from buffer
	231	* flipping is provided by interrupt disablement (note that for
ece8a684 IM	232	* SCHED_PROFILING or SLEEP_PROFILING profile_hit() may be called from
ece8a684 IM	233	* process context).
1da177e4 LT	234	* The hash function is meant to be lightweight as opposed to strong,
	235	* and was vaguely inspired by ppc64 firmware-supported inverted
	236	* pagetable hash functions, but uses a full hashtable full of finite
	237	* collision chains, not just pairs of them.
	238	*
6d49e352	239	* -- nyc
1da177e4 LT	240	*/
	241	static void __profile_flip_buffers(void *unused)
	242	{
	243	int cpu = smp_processor_id();
	244
	245	per_cpu(cpu_profile_flip, cpu) = !per_cpu(cpu_profile_flip, cpu);
	246	}
	247
	248	static void profile_flip_buffers(void)
	249	{
	250	int i, j, cpu;
	251
97d1f15b	252	mutex_lock(&profile_flip_mutex);
1da177e4 LT	253	j = per_cpu(cpu_profile_flip, get_cpu());
1da177e4 LT	254	put_cpu();
15c8b6c1	255	on_each_cpu(__profile_flip_buffers, NULL, 1);
1da177e4 LT	256	for_each_online_cpu(cpu) {
	257	struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[j];
	258	for (i = 0; i < NR_PROFILE_HIT; ++i) {
	259	if (!hits[i].hits) {
	260	if (hits[i].pc)
	261	hits[i].pc = 0;
	262	continue;
	263	}
	264	atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
	265	hits[i].hits = hits[i].pc = 0;
	266	}
	267	}
97d1f15b	268	mutex_unlock(&profile_flip_mutex);
1da177e4 LT	269	}
	270
	271	static void profile_discard_flip_buffers(void)
	272	{
	273	int i, cpu;
	274
97d1f15b	275	mutex_lock(&profile_flip_mutex);
1da177e4 LT	276	i = per_cpu(cpu_profile_flip, get_cpu());
1da177e4 LT	277	put_cpu();
15c8b6c1	278	on_each_cpu(__profile_flip_buffers, NULL, 1);
1da177e4 LT	279	for_each_online_cpu(cpu) {
	280	struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[i];
	281	memset(hits, 0, NR_PROFILE_HIT*sizeof(struct profile_hit));
	282	}
97d1f15b	283	mutex_unlock(&profile_flip_mutex);
1da177e4 LT	284	}
1da177e4 LT	285
6f7bd76f	286	static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
1da177e4 LT	287	{
	288	unsigned long primary, secondary, flags, pc = (unsigned long)__pc;
	289	int i, j, cpu;
	290	struct profile_hit *hits;
	291
1da177e4 LT	292	pc = min((pc - (unsigned long)_stext) >> prof_shift, prof_len - 1);
	293	i = primary = (pc & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
	294	secondary = (~(pc << 1) & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
	295	cpu = get_cpu();
	296	hits = per_cpu(cpu_profile_hits, cpu)[per_cpu(cpu_profile_flip, cpu)];
	297	if (!hits) {
	298	put_cpu();
	299	return;
	300	}
ece8a684 IM	301	/*
	302	* We buffer the global profiler buffer into a per-CPU
	303	* queue and thus reduce the number of global (and possibly
	304	* NUMA-alien) accesses. The write-queue is self-coalescing:
	305	*/
1da177e4 LT	306	local_irq_save(flags);
	307	do {
	308	for (j = 0; j < PROFILE_GRPSZ; ++j) {
	309	if (hits[i + j].pc == pc) {
ece8a684	310	hits[i + j].hits += nr_hits;
1da177e4 LT	311	goto out;
	312	} else if (!hits[i + j].hits) {
	313	hits[i + j].pc = pc;
ece8a684	314	hits[i + j].hits = nr_hits;
1da177e4 LT	315	goto out;
	316	}
	317	}
	318	i = (i + secondary) & (NR_PROFILE_HIT - 1);
	319	} while (i != primary);
ece8a684 IM	320
	321	/*
	322	* Add the current hit(s) and flush the write-queue out
	323	* to the global buffer:
	324	*/
	325	atomic_add(nr_hits, &prof_buffer[pc]);
1da177e4 LT	326	for (i = 0; i < NR_PROFILE_HIT; ++i) {
	327	atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
	328	hits[i].pc = hits[i].hits = 0;
	329	}
	330	out:
	331	local_irq_restore(flags);
	332	put_cpu();
	333	}
	334
e722d8da	335	static int profile_dead_cpu(unsigned int cpu)
1da177e4	336	{
1da177e4	337	struct page *page;
e722d8da	338	int i;
1da177e4	339
e722d8da SAS	340	if (prof_cpu_mask != NULL)
	341	cpumask_clear_cpu(cpu, prof_cpu_mask);
	342
	343	for (i = 0; i < 2; i++) {
	344	if (per_cpu(cpu_profile_hits, cpu)[i]) {
	345	page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[i]);
	346	per_cpu(cpu_profile_hits, cpu)[i] = NULL;
1da177e4 LT	347	__free_page(page);
1da177e4 LT	348	}
e722d8da SAS	349	}
	350	return 0;
	351	}
	352
	353	static int profile_prepare_cpu(unsigned int cpu)
	354	{
	355	int i, node = cpu_to_mem(cpu);
	356	struct page *page;
	357
	358	per_cpu(cpu_profile_flip, cpu) = 0;
	359
	360	for (i = 0; i < 2; i++) {
	361	if (per_cpu(cpu_profile_hits, cpu)[i])
	362	continue;
	363
	364	page = __alloc_pages_node(node, GFP_KERNEL \| __GFP_ZERO, 0);
	365	if (!page) {
	366	profile_dead_cpu(cpu);
	367	return -ENOMEM;
1da177e4	368	}
e722d8da SAS	369	per_cpu(cpu_profile_hits, cpu)[i] = page_address(page);
e722d8da SAS	370
1da177e4	371	}
e722d8da SAS	372	return 0;
	373	}
	374
	375	static int profile_online_cpu(unsigned int cpu)
	376	{
	377	if (prof_cpu_mask != NULL)
	378	cpumask_set_cpu(cpu, prof_cpu_mask);
	379
	380	return 0;
1da177e4	381	}
e722d8da	382
1da177e4 LT	383	#else /* !CONFIG_SMP */
	384	#define profile_flip_buffers() do { } while (0)
	385	#define profile_discard_flip_buffers() do { } while (0)
	386
6f7bd76f	387	static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
1da177e4 LT	388	{
1da177e4 LT	389	unsigned long pc;
1da177e4	390	pc = ((unsigned long)__pc - (unsigned long)_stext) >> prof_shift;
ece8a684	391	atomic_add(nr_hits, &prof_buffer[min(pc, prof_len - 1)]);
1da177e4 LT	392	}
1da177e4 LT	393	#endif /* !CONFIG_SMP */
6f7bd76f RM	394
	395	void profile_hits(int type, void *__pc, unsigned int nr_hits)
	396	{
	397	if (prof_on != type \|\| !prof_buffer)
	398	return;
	399	do_profile_hits(type, __pc, nr_hits);
	400	}
bbe1a59b AM	401	EXPORT_SYMBOL_GPL(profile_hits);
bbe1a59b AM	402
7d12e780	403	void profile_tick(int type)
1da177e4	404	{
7d12e780 DH	405	struct pt_regs *regs = get_irq_regs();
7d12e780 DH	406
c309b917 RR	407	if (!user_mode(regs) && prof_cpu_mask != NULL &&
c309b917 RR	408	cpumask_test_cpu(smp_processor_id(), prof_cpu_mask))
1da177e4 LT	409	profile_hit(type, (void *)profile_pc(regs));
	410	}
	411
	412	#ifdef CONFIG_PROC_FS
	413	#include <linux/proc_fs.h>
583a22e7	414	#include <linux/seq_file.h>
7c0f6ba6	415	#include <linux/uaccess.h>
1da177e4	416
583a22e7	417	static int prof_cpu_mask_proc_show(struct seq_file m, void v)
1da177e4	418	{
ccbd59c1	419	seq_printf(m, "%*pb\n", cpumask_pr_args(prof_cpu_mask));
583a22e7 AD	420	return 0;
	421	}
	422
	423	static int prof_cpu_mask_proc_open(struct inode inode, struct file file)
	424	{
	425	return single_open(file, prof_cpu_mask_proc_show, NULL);
1da177e4 LT	426	}
1da177e4 LT	427
583a22e7 AD	428	static ssize_t prof_cpu_mask_proc_write(struct file *file,
583a22e7 AD	429	const char __user buffer, size_t count, loff_t pos)
1da177e4	430	{
c309b917	431	cpumask_var_t new_value;
583a22e7	432	int err;
1da177e4	433
c309b917 RR	434	if (!alloc_cpumask_var(&new_value, GFP_KERNEL))
c309b917 RR	435	return -ENOMEM;
1da177e4	436
c309b917 RR	437	err = cpumask_parse_user(buffer, count, new_value);
c309b917 RR	438	if (!err) {
583a22e7 AD	439	cpumask_copy(prof_cpu_mask, new_value);
583a22e7 AD	440	err = count;
c309b917 RR	441	}
	442	free_cpumask_var(new_value);
	443	return err;
1da177e4 LT	444	}
1da177e4 LT	445
583a22e7 AD	446	static const struct file_operations prof_cpu_mask_proc_fops = {
	447	.open = prof_cpu_mask_proc_open,
	448	.read = seq_read,
	449	.llseek = seq_lseek,
	450	.release = single_release,
	451	.write = prof_cpu_mask_proc_write,
	452	};
	453
fbd387ae	454	void create_prof_cpu_mask(void)
1da177e4	455	{
1da177e4	456	/* create /proc/irq/prof_cpu_mask */
fbd387ae	457	proc_create("irq/prof_cpu_mask", 0600, NULL, &prof_cpu_mask_proc_fops);
1da177e4 LT	458	}
	459
	460	/*
	461	* This function accesses profiling information. The returned data is
	462	* binary: the sampling step and the actual contents of the profile
	463	* buffer. Use of the program readprofile is recommended in order to
	464	* get meaningful info out of these data.
	465	*/
	466	static ssize_t
	467	read_profile(struct file file, char __user buf, size_t count, loff_t *ppos)
	468	{
	469	unsigned long p = *ppos;
	470	ssize_t read;
1ad82fd5	471	char *pnt;
455d939e	472	unsigned long sample_step = 1UL << prof_shift;
1da177e4 LT	473
	474	profile_flip_buffers();
	475	if (p >= (prof_len+1)*sizeof(unsigned int))
	476	return 0;
	477	if (count > (prof_len+1)*sizeof(unsigned int) - p)
	478	count = (prof_len+1)*sizeof(unsigned int) - p;
	479	read = 0;
	480
	481	while (p < sizeof(unsigned int) && count > 0) {
1ad82fd5	482	if (put_user(((char )(&sample_step)+p), buf))
064b022c	483	return -EFAULT;
1da177e4 LT	484	buf++; p++; count--; read++;
	485	}
	486	pnt = (char *)prof_buffer + p - sizeof(atomic_t);
1ad82fd5	487	if (copy_to_user(buf, (void *)pnt, count))
1da177e4 LT	488	return -EFAULT;
	489	read += count;
	490	*ppos += read;
	491	return read;
	492	}
	493
	494	/*
	495	* Writing to /proc/profile resets the counters
	496	*
	497	* Writing a 'profiling multiplier' value into it also re-sets the profiling
	498	* interrupt frequency, on architectures that support this.
	499	*/
	500	static ssize_t write_profile(struct file file, const char __user buf,
	501	size_t count, loff_t *ppos)
	502	{
	503	#ifdef CONFIG_SMP
1ad82fd5	504	extern int setup_profiling_timer(unsigned int multiplier);
1da177e4 LT	505
	506	if (count == sizeof(int)) {
	507	unsigned int multiplier;
	508
	509	if (copy_from_user(&multiplier, buf, sizeof(int)))
	510	return -EFAULT;
	511
	512	if (setup_profiling_timer(multiplier))
	513	return -EINVAL;
	514	}
	515	#endif
	516	profile_discard_flip_buffers();
	517	memset(prof_buffer, 0, prof_len * sizeof(atomic_t));
	518	return count;
	519	}
	520
15ad7cdc	521	static const struct file_operations proc_profile_operations = {
1da177e4 LT	522	.read = read_profile,
1da177e4 LT	523	.write = write_profile,
6038f373	524	.llseek = default_llseek,
1da177e4 LT	525	};
1da177e4 LT	526
e722d8da	527	int __ref create_proc_profile(void)
1da177e4	528	{
e722d8da SAS	529	struct proc_dir_entry *entry;
	530	#ifdef CONFIG_SMP
	531	enum cpuhp_state online_state;
1da177e4 LT	532	#endif
1da177e4 LT	533
c270a817	534	int err = 0;
1da177e4 LT	535
	536	if (!prof_on)
	537	return 0;
e722d8da SAS	538	#ifdef CONFIG_SMP
	539	err = cpuhp_setup_state(CPUHP_PROFILE_PREPARE, "PROFILE_PREPARE",
	540	profile_prepare_cpu, profile_dead_cpu);
	541	if (err)
	542	return err;
	543
	544	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "AP_PROFILE_ONLINE",
	545	profile_online_cpu, NULL);
	546	if (err < 0)
	547	goto err_state_prep;
	548	online_state = err;
	549	err = 0;
	550	#endif
c33fff0a DL	551	entry = proc_create("profile", S_IWUSR \| S_IRUGO,
c33fff0a DL	552	NULL, &proc_profile_operations);
1ad82fd5	553	if (!entry)
e722d8da	554	goto err_state_onl;
271a15ea	555	proc_set_size(entry, (1 + prof_len) * sizeof(atomic_t));
c270a817	556
e722d8da SAS	557	return err;
	558	err_state_onl:
	559	#ifdef CONFIG_SMP
	560	cpuhp_remove_state(online_state);
	561	err_state_prep:
	562	cpuhp_remove_state(CPUHP_PROFILE_PREPARE);
	563	#endif
c270a817	564	return err;
1da177e4	565	}
c96d6660	566	subsys_initcall(create_proc_profile);
1da177e4	567	#endif /* CONFIG_PROC_FS */