[mirror_ubuntu-jammy-kernel.git] / arch / x86 / kernel / perf_regs.c

// SPDX-License-Identifier: GPL-2.0
#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#include <linux/perf_event.h>
#include <linux/bug.h>
#include <linux/stddef.h>
#include <asm/perf_regs.h>
#include <asm/ptrace.h>

#ifdef CONFIG_X86_32
#define PERF_REG_X86_MAX PERF_REG_X86_32_MAX
#else
#define PERF_REG_X86_MAX PERF_REG_X86_64_MAX
#endif

#define PT_REGS_OFFSET(id, r) [id] = offsetof(struct pt_regs, r)

static unsigned int pt_regs_offset[PERF_REG_X86_MAX] = {
	PT_REGS_OFFSET(PERF_REG_X86_AX, ax),
	PT_REGS_OFFSET(PERF_REG_X86_BX, bx),
	PT_REGS_OFFSET(PERF_REG_X86_CX, cx),
	PT_REGS_OFFSET(PERF_REG_X86_DX, dx),
	PT_REGS_OFFSET(PERF_REG_X86_SI, si),
	PT_REGS_OFFSET(PERF_REG_X86_DI, di),
	PT_REGS_OFFSET(PERF_REG_X86_BP, bp),
	PT_REGS_OFFSET(PERF_REG_X86_SP, sp),
	PT_REGS_OFFSET(PERF_REG_X86_IP, ip),
	PT_REGS_OFFSET(PERF_REG_X86_FLAGS, flags),
	PT_REGS_OFFSET(PERF_REG_X86_CS, cs),
	PT_REGS_OFFSET(PERF_REG_X86_SS, ss),
#ifdef CONFIG_X86_32
	PT_REGS_OFFSET(PERF_REG_X86_DS, ds),
	PT_REGS_OFFSET(PERF_REG_X86_ES, es),
	PT_REGS_OFFSET(PERF_REG_X86_FS, fs),
	PT_REGS_OFFSET(PERF_REG_X86_GS, gs),
#else
	/*
	 * The pt_regs struct does not store
	 * ds, es, fs, gs in 64 bit mode.
	 */
	(unsigned int) -1,
	(unsigned int) -1,
	(unsigned int) -1,
	(unsigned int) -1,
#endif
#ifdef CONFIG_X86_64
	PT_REGS_OFFSET(PERF_REG_X86_R8, r8),
	PT_REGS_OFFSET(PERF_REG_X86_R9, r9),
	PT_REGS_OFFSET(PERF_REG_X86_R10, r10),
	PT_REGS_OFFSET(PERF_REG_X86_R11, r11),
	PT_REGS_OFFSET(PERF_REG_X86_R12, r12),
	PT_REGS_OFFSET(PERF_REG_X86_R13, r13),
	PT_REGS_OFFSET(PERF_REG_X86_R14, r14),
	PT_REGS_OFFSET(PERF_REG_X86_R15, r15),
#endif
};

u64 perf_reg_value(struct pt_regs *regs, int idx)
{
	struct x86_perf_regs *perf_regs;

	if (idx >= PERF_REG_X86_XMM0 && idx < PERF_REG_X86_XMM_MAX) {
		perf_regs = container_of(regs, struct x86_perf_regs, regs);
		if (!perf_regs->xmm_regs)
			return 0;
		return perf_regs->xmm_regs[idx - PERF_REG_X86_XMM0];
	}

	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(pt_regs_offset)))
		return 0;

	return regs_get_register(regs, pt_regs_offset[idx]);
}

#define PERF_REG_X86_RESERVED	(((1ULL << PERF_REG_X86_XMM0) - 1) & \
				 ~((1ULL << PERF_REG_X86_MAX) - 1))

#ifdef CONFIG_X86_32
#define REG_NOSUPPORT ((1ULL << PERF_REG_X86_R8) | \
		       (1ULL << PERF_REG_X86_R9) | \
		       (1ULL << PERF_REG_X86_R10) | \
		       (1ULL << PERF_REG_X86_R11) | \
		       (1ULL << PERF_REG_X86_R12) | \
		       (1ULL << PERF_REG_X86_R13) | \
		       (1ULL << PERF_REG_X86_R14) | \
		       (1ULL << PERF_REG_X86_R15))

int perf_reg_validate(u64 mask)
{
	if (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)))
		return -EINVAL;

	return 0;
}

u64 perf_reg_abi(struct task_struct *task)
{
	return PERF_SAMPLE_REGS_ABI_32;
}

void perf_get_regs_user(struct perf_regs *regs_user,
			struct pt_regs *regs)
{
	regs_user->regs = task_pt_regs(current);
	regs_user->abi = perf_reg_abi(current);
}
#else /* CONFIG_X86_64 */
#define REG_NOSUPPORT ((1ULL << PERF_REG_X86_DS) | \
		       (1ULL << PERF_REG_X86_ES) | \
		       (1ULL << PERF_REG_X86_FS) | \
		       (1ULL << PERF_REG_X86_GS))

int perf_reg_validate(u64 mask)
{
	if (!mask || (mask & (REG_NOSUPPORT | PERF_REG_X86_RESERVED)))
		return -EINVAL;

	return 0;
}

u64 perf_reg_abi(struct task_struct *task)
{
	if (!user_64bit_mode(task_pt_regs(task)))
		return PERF_SAMPLE_REGS_ABI_32;
	else
		return PERF_SAMPLE_REGS_ABI_64;
}

static DEFINE_PER_CPU(struct pt_regs, nmi_user_regs);

void perf_get_regs_user(struct perf_regs *regs_user,
			struct pt_regs *regs)
{
	struct pt_regs *regs_user_copy = this_cpu_ptr(&nmi_user_regs);
	struct pt_regs *user_regs = task_pt_regs(current);

	if (!in_nmi()) {
		regs_user->regs = user_regs;
		regs_user->abi = perf_reg_abi(current);
		return;
	}

	/*
	 * If we're in an NMI that interrupted task_pt_regs setup, then
	 * we can't sample user regs at all.  This check isn't really
	 * sufficient, though, as we could be in an NMI inside an interrupt
	 * that happened during task_pt_regs setup.
	 */
	if (regs->sp > (unsigned long)&user_regs->r11 &&
	    regs->sp <= (unsigned long)(user_regs + 1)) {
		regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
		regs_user->regs = NULL;
		return;
	}

	/*
	 * These registers are always saved on 64-bit syscall entry.
	 * On 32-bit entry points, they are saved too except r8..r11.
	 */
	regs_user_copy->ip = user_regs->ip;
	regs_user_copy->ax = user_regs->ax;
	regs_user_copy->cx = user_regs->cx;
	regs_user_copy->dx = user_regs->dx;
	regs_user_copy->si = user_regs->si;
	regs_user_copy->di = user_regs->di;
	regs_user_copy->r8 = user_regs->r8;
	regs_user_copy->r9 = user_regs->r9;
	regs_user_copy->r10 = user_regs->r10;
	regs_user_copy->r11 = user_regs->r11;
	regs_user_copy->orig_ax = user_regs->orig_ax;
	regs_user_copy->flags = user_regs->flags;
	regs_user_copy->sp = user_regs->sp;
	regs_user_copy->cs = user_regs->cs;
	regs_user_copy->ss = user_regs->ss;
	/*
	 * Store user space frame-pointer value on sample
	 * to facilitate stack unwinding for cases when
	 * user space executable code has such support
	 * enabled at compile time:
	 */
	regs_user_copy->bp = user_regs->bp;

	regs_user_copy->bx = -1;
	regs_user_copy->r12 = -1;
	regs_user_copy->r13 = -1;
	regs_user_copy->r14 = -1;
	regs_user_copy->r15 = -1;
	/*
	 * For this to be at all useful, we need a reasonable guess for
	 * the ABI.  Be careful: we're in NMI context, and we're
	 * considering current to be the current task, so we should
	 * be careful not to look at any other percpu variables that might
	 * change during context switches.
	 */
	regs_user->abi = user_64bit_mode(user_regs) ?
		PERF_SAMPLE_REGS_ABI_64 : PERF_SAMPLE_REGS_ABI_32;

	regs_user->regs = regs_user_copy;
}
#endif /* CONFIG_X86_32 */
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
c5e63197 JO	2	#include <linux/errno.h>
c5e63197 JO	3	#include <linux/kernel.h>
4018994f	4	#include <linux/sched.h>
68db0cf1	5	#include <linux/sched/task_stack.h>
4018994f	6	#include <linux/perf_event.h>
c5e63197 JO	7	#include <linux/bug.h>
	8	#include <linux/stddef.h>
	9	#include <asm/perf_regs.h>
	10	#include <asm/ptrace.h>
	11
	12	#ifdef CONFIG_X86_32
	13	#define PERF_REG_X86_MAX PERF_REG_X86_32_MAX
	14	#else
	15	#define PERF_REG_X86_MAX PERF_REG_X86_64_MAX
	16	#endif
	17
	18	#define PT_REGS_OFFSET(id, r) [id] = offsetof(struct pt_regs, r)
	19
	20	static unsigned int pt_regs_offset[PERF_REG_X86_MAX] = {
	21	PT_REGS_OFFSET(PERF_REG_X86_AX, ax),
	22	PT_REGS_OFFSET(PERF_REG_X86_BX, bx),
	23	PT_REGS_OFFSET(PERF_REG_X86_CX, cx),
	24	PT_REGS_OFFSET(PERF_REG_X86_DX, dx),
	25	PT_REGS_OFFSET(PERF_REG_X86_SI, si),
	26	PT_REGS_OFFSET(PERF_REG_X86_DI, di),
	27	PT_REGS_OFFSET(PERF_REG_X86_BP, bp),
	28	PT_REGS_OFFSET(PERF_REG_X86_SP, sp),
	29	PT_REGS_OFFSET(PERF_REG_X86_IP, ip),
	30	PT_REGS_OFFSET(PERF_REG_X86_FLAGS, flags),
	31	PT_REGS_OFFSET(PERF_REG_X86_CS, cs),
	32	PT_REGS_OFFSET(PERF_REG_X86_SS, ss),
	33	#ifdef CONFIG_X86_32
	34	PT_REGS_OFFSET(PERF_REG_X86_DS, ds),
	35	PT_REGS_OFFSET(PERF_REG_X86_ES, es),
	36	PT_REGS_OFFSET(PERF_REG_X86_FS, fs),
	37	PT_REGS_OFFSET(PERF_REG_X86_GS, gs),
	38	#else
	39	/*
	40	* The pt_regs struct does not store
	41	* ds, es, fs, gs in 64 bit mode.
	42	*/
	43	(unsigned int) -1,
	44	(unsigned int) -1,
	45	(unsigned int) -1,
	46	(unsigned int) -1,
	47	#endif
	48	#ifdef CONFIG_X86_64
	49	PT_REGS_OFFSET(PERF_REG_X86_R8, r8),
	50	PT_REGS_OFFSET(PERF_REG_X86_R9, r9),
	51	PT_REGS_OFFSET(PERF_REG_X86_R10, r10),
	52	PT_REGS_OFFSET(PERF_REG_X86_R11, r11),
	53	PT_REGS_OFFSET(PERF_REG_X86_R12, r12),
	54	PT_REGS_OFFSET(PERF_REG_X86_R13, r13),
	55	PT_REGS_OFFSET(PERF_REG_X86_R14, r14),
	56	PT_REGS_OFFSET(PERF_REG_X86_R15, r15),
	57	#endif
	58	};
	59
	60	u64 perf_reg_value(struct pt_regs *regs, int idx)
	61	{
878068ea KL	62	struct x86_perf_regs *perf_regs;
	63
	64	if (idx >= PERF_REG_X86_XMM0 && idx < PERF_REG_X86_XMM_MAX) {
	65	perf_regs = container_of(regs, struct x86_perf_regs, regs);
	66	if (!perf_regs->xmm_regs)
	67	return 0;
	68	return perf_regs->xmm_regs[idx - PERF_REG_X86_XMM0];
	69	}
	70
1e6dd8ad	71	if (WARN_ON_ONCE(idx >= ARRAY_SIZE(pt_regs_offset)))
c5e63197 JO	72	return 0;
	73
	74	return regs_get_register(regs, pt_regs_offset[idx]);
	75	}
	76
90d42491 KL	77	#define PERF_REG_X86_RESERVED (((1ULL << PERF_REG_X86_XMM0) - 1) & \
	78	~((1ULL << PERF_REG_X86_MAX) - 1))
	79
c5e63197	80	#ifdef CONFIG_X86_32
878068ea KL	81	#define REG_NOSUPPORT ((1ULL << PERF_REG_X86_R8) \| \
	82	(1ULL << PERF_REG_X86_R9) \| \
	83	(1ULL << PERF_REG_X86_R10) \| \
	84	(1ULL << PERF_REG_X86_R11) \| \
	85	(1ULL << PERF_REG_X86_R12) \| \
	86	(1ULL << PERF_REG_X86_R13) \| \
	87	(1ULL << PERF_REG_X86_R14) \| \
	88	(1ULL << PERF_REG_X86_R15))
	89
c5e63197 JO	90	int perf_reg_validate(u64 mask)
c5e63197 JO	91	{
90d42491	92	if (!mask \|\| (mask & (REG_NOSUPPORT \| PERF_REG_X86_RESERVED)))
c5e63197 JO	93	return -EINVAL;
	94
	95	return 0;
	96	}
4018994f JO	97
	98	u64 perf_reg_abi(struct task_struct *task)
	99	{
	100	return PERF_SAMPLE_REGS_ABI_32;
	101	}
88a7c26a AL	102
88a7c26a AL	103	void perf_get_regs_user(struct perf_regs *regs_user,
76a4efa8	104	struct pt_regs *regs)
88a7c26a AL	105	{
	106	regs_user->regs = task_pt_regs(current);
	107	regs_user->abi = perf_reg_abi(current);
	108	}
c5e63197 JO	109	#else /* CONFIG_X86_64 */
	110	#define REG_NOSUPPORT ((1ULL << PERF_REG_X86_DS) \| \
	111	(1ULL << PERF_REG_X86_ES) \| \
	112	(1ULL << PERF_REG_X86_FS) \| \
	113	(1ULL << PERF_REG_X86_GS))
	114
	115	int perf_reg_validate(u64 mask)
	116	{
90d42491	117	if (!mask \|\| (mask & (REG_NOSUPPORT \| PERF_REG_X86_RESERVED)))
c5e63197 JO	118	return -EINVAL;
	119
	120	return 0;
	121	}
4018994f JO	122
	123	u64 perf_reg_abi(struct task_struct *task)
	124	{
375d4bfd	125	if (!user_64bit_mode(task_pt_regs(task)))
4018994f JO	126	return PERF_SAMPLE_REGS_ABI_32;
	127	else
	128	return PERF_SAMPLE_REGS_ABI_64;
	129	}
88a7c26a	130
76a4efa8 PZ	131	static DEFINE_PER_CPU(struct pt_regs, nmi_user_regs);
76a4efa8 PZ	132
88a7c26a	133	void perf_get_regs_user(struct perf_regs *regs_user,
76a4efa8	134	struct pt_regs *regs)
88a7c26a	135	{
76a4efa8	136	struct pt_regs *regs_user_copy = this_cpu_ptr(&nmi_user_regs);
86c269fe AL	137	struct pt_regs *user_regs = task_pt_regs(current);
86c269fe AL	138
76a4efa8 PZ	139	if (!in_nmi()) {
	140	regs_user->regs = user_regs;
	141	regs_user->abi = perf_reg_abi(current);
	142	return;
	143	}
	144
86c269fe AL	145	/*
	146	* If we're in an NMI that interrupted task_pt_regs setup, then
	147	* we can't sample user regs at all. This check isn't really
	148	* sufficient, though, as we could be in an NMI inside an interrupt
	149	* that happened during task_pt_regs setup.
	150	*/
	151	if (regs->sp > (unsigned long)&user_regs->r11 &&
	152	regs->sp <= (unsigned long)(user_regs + 1)) {
	153	regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE;
	154	regs_user->regs = NULL;
	155	return;
	156	}
	157
	158	/*
aa21df04 DV	159	* These registers are always saved on 64-bit syscall entry.
aa21df04 DV	160	* On 32-bit entry points, they are saved too except r8..r11.
86c269fe AL	161	*/
86c269fe AL	162	regs_user_copy->ip = user_regs->ip;
3b75232d	163	regs_user_copy->ax = user_regs->ax;
86c269fe AL	164	regs_user_copy->cx = user_regs->cx;
	165	regs_user_copy->dx = user_regs->dx;
	166	regs_user_copy->si = user_regs->si;
	167	regs_user_copy->di = user_regs->di;
	168	regs_user_copy->r8 = user_regs->r8;
	169	regs_user_copy->r9 = user_regs->r9;
	170	regs_user_copy->r10 = user_regs->r10;
	171	regs_user_copy->r11 = user_regs->r11;
	172	regs_user_copy->orig_ax = user_regs->orig_ax;
	173	regs_user_copy->flags = user_regs->flags;
aa21df04 DV	174	regs_user_copy->sp = user_regs->sp;
	175	regs_user_copy->cs = user_regs->cs;
	176	regs_user_copy->ss = user_regs->ss;
86c269fe	177	/*
10b11050 AB	178	* Store user space frame-pointer value on sample
	179	* to facilitate stack unwinding for cases when
	180	* user space executable code has such support
	181	* enabled at compile time:
86c269fe	182	*/
10b11050 AB	183	regs_user_copy->bp = user_regs->bp;
10b11050 AB	184
86c269fe	185	regs_user_copy->bx = -1;
86c269fe AL	186	regs_user_copy->r12 = -1;
	187	regs_user_copy->r13 = -1;
	188	regs_user_copy->r14 = -1;
	189	regs_user_copy->r15 = -1;
86c269fe AL	190	/*
86c269fe AL	191	* For this to be at all useful, we need a reasonable guess for
aa21df04	192	* the ABI. Be careful: we're in NMI context, and we're
86c269fe AL	193	* considering current to be the current task, so we should
	194	* be careful not to look at any other percpu variables that might
	195	* change during context switches.
	196	*/
32caa060 DV	197	regs_user->abi = user_64bit_mode(user_regs) ?
32caa060 DV	198	PERF_SAMPLE_REGS_ABI_64 : PERF_SAMPLE_REGS_ABI_32;
86c269fe AL	199
86c269fe AL	200	regs_user->regs = regs_user_copy;
88a7c26a	201	}
c5e63197	202	#endif /* CONFIG_X86_32 */