[mirror_ubuntu-focal-kernel.git] / arch / x86 / kernel / cpu / perf_event_intel_ds.c

#ifdef CONFIG_CPU_SUP_INTEL

/* The maximal number of PEBS events: */
#define MAX_PEBS_EVENTS		4

/* The size of a BTS record in bytes: */
#define BTS_RECORD_SIZE		24

#define BTS_BUFFER_SIZE		(PAGE_SIZE << 4)
#define PEBS_BUFFER_SIZE	PAGE_SIZE

/*
 * pebs_record_32 for p4 and core not supported

struct pebs_record_32 {
	u32 flags, ip;
	u32 ax, bc, cx, dx;
	u32 si, di, bp, sp;
};

 */

struct pebs_record_core {
	u64 flags, ip;
	u64 ax, bx, cx, dx;
	u64 si, di, bp, sp;
	u64 r8,  r9,  r10, r11;
	u64 r12, r13, r14, r15;
};

struct pebs_record_nhm {
	u64 flags, ip;
	u64 ax, bx, cx, dx;
	u64 si, di, bp, sp;
	u64 r8,  r9,  r10, r11;
	u64 r12, r13, r14, r15;
	u64 status, dla, dse, lat;
};

/*
 * Bits in the debugctlmsr controlling branch tracing.
 */
#define X86_DEBUGCTL_TR			(1 << 6)
#define X86_DEBUGCTL_BTS		(1 << 7)
#define X86_DEBUGCTL_BTINT		(1 << 8)
#define X86_DEBUGCTL_BTS_OFF_OS		(1 << 9)
#define X86_DEBUGCTL_BTS_OFF_USR	(1 << 10)

/*
 * A debug store configuration.
 *
 * We only support architectures that use 64bit fields.
 */
struct debug_store {
	u64	bts_buffer_base;
	u64	bts_index;
	u64	bts_absolute_maximum;
	u64	bts_interrupt_threshold;
	u64	pebs_buffer_base;
	u64	pebs_index;
	u64	pebs_absolute_maximum;
	u64	pebs_interrupt_threshold;
	u64	pebs_event_reset[MAX_PEBS_EVENTS];
};

static void init_debug_store_on_cpu(int cpu)
{
	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;

	if (!ds)
		return;

	wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
		     (u32)((u64)(unsigned long)ds),
		     (u32)((u64)(unsigned long)ds >> 32));
}

static void fini_debug_store_on_cpu(int cpu)
{
	if (!per_cpu(cpu_hw_events, cpu).ds)
		return;

	wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
}

static void release_ds_buffers(void)
{
	int cpu;

	if (!x86_pmu.bts && !x86_pmu.pebs)
		return;

	get_online_cpus();

	for_each_online_cpu(cpu)
		fini_debug_store_on_cpu(cpu);

	for_each_possible_cpu(cpu) {
		struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;

		if (!ds)
			continue;

		per_cpu(cpu_hw_events, cpu).ds = NULL;

		kfree((void *)(unsigned long)ds->pebs_buffer_base);
		kfree((void *)(unsigned long)ds->bts_buffer_base);
		kfree(ds);
	}

	put_online_cpus();
}

static int reserve_ds_buffers(void)
{
	int cpu, err = 0;

	if (!x86_pmu.bts && !x86_pmu.pebs)
		return 0;

	get_online_cpus();

	for_each_possible_cpu(cpu) {
		struct debug_store *ds;
		void *buffer;
		int max, thresh;

		err = -ENOMEM;
		ds = kzalloc(sizeof(*ds), GFP_KERNEL);
		if (unlikely(!ds)) {
			kfree(buffer);
			break;
		}
		per_cpu(cpu_hw_events, cpu).ds = ds;

		if (x86_pmu.bts) {
			buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
			if (unlikely(!buffer))
				break;

			max = BTS_BUFFER_SIZE / BTS_RECORD_SIZE;
			thresh = max / 16;

			ds->bts_buffer_base = (u64)(unsigned long)buffer;
			ds->bts_index = ds->bts_buffer_base;
			ds->bts_absolute_maximum = ds->bts_buffer_base +
				max * BTS_RECORD_SIZE;
			ds->bts_interrupt_threshold = ds->bts_absolute_maximum -
				thresh * BTS_RECORD_SIZE;
		}

		if (x86_pmu.pebs) {
			buffer = kzalloc(PEBS_BUFFER_SIZE, GFP_KERNEL);
			if (unlikely(!buffer))
				break;

			max = PEBS_BUFFER_SIZE / x86_pmu.pebs_record_size;

			ds->pebs_buffer_base = (u64)(unsigned long)buffer;
			ds->pebs_index = ds->pebs_buffer_base;
			ds->pebs_absolute_maximum = ds->pebs_buffer_base +
				max * x86_pmu.pebs_record_size;
			/*
			 * Always use single record PEBS
			 */
			ds->pebs_interrupt_threshold = ds->pebs_buffer_base +
				x86_pmu.pebs_record_size;
		}

		err = 0;
	}

	if (err)
		release_ds_buffers();
	else {
		for_each_online_cpu(cpu)
			init_debug_store_on_cpu(cpu);
	}

	put_online_cpus();

	return err;
}

/*
 * BTS
 */

static struct event_constraint bts_constraint =
	EVENT_CONSTRAINT(0, 1ULL << X86_PMC_IDX_FIXED_BTS, 0);

static void intel_pmu_enable_bts(u64 config)
{
	unsigned long debugctlmsr;

	debugctlmsr = get_debugctlmsr();

	debugctlmsr |= X86_DEBUGCTL_TR;
	debugctlmsr |= X86_DEBUGCTL_BTS;
	debugctlmsr |= X86_DEBUGCTL_BTINT;

	if (!(config & ARCH_PERFMON_EVENTSEL_OS))
		debugctlmsr |= X86_DEBUGCTL_BTS_OFF_OS;

	if (!(config & ARCH_PERFMON_EVENTSEL_USR))
		debugctlmsr |= X86_DEBUGCTL_BTS_OFF_USR;

	update_debugctlmsr(debugctlmsr);
}

static void intel_pmu_disable_bts(void)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	unsigned long debugctlmsr;

	if (!cpuc->ds)
		return;

	debugctlmsr = get_debugctlmsr();

	debugctlmsr &=
		~(X86_DEBUGCTL_TR | X86_DEBUGCTL_BTS | X86_DEBUGCTL_BTINT |
		  X86_DEBUGCTL_BTS_OFF_OS | X86_DEBUGCTL_BTS_OFF_USR);

	update_debugctlmsr(debugctlmsr);
}

static void intel_pmu_drain_bts_buffer(void)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	struct debug_store *ds = cpuc->ds;
	struct bts_record {
		u64	from;
		u64	to;
		u64	flags;
	};
	struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
	struct bts_record *at, *top;
	struct perf_output_handle handle;
	struct perf_event_header header;
	struct perf_sample_data data;
	struct pt_regs regs;

	if (!event)
		return;

	if (!ds)
		return;

	at  = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
	top = (struct bts_record *)(unsigned long)ds->bts_index;

	if (top <= at)
		return;

	ds->bts_index = ds->bts_buffer_base;

	perf_sample_data_init(&data, 0);
	data.period = event->hw.last_period;
	regs.ip     = 0;

	/*
	 * Prepare a generic sample, i.e. fill in the invariant fields.
	 * We will overwrite the from and to address before we output
	 * the sample.
	 */
	perf_prepare_sample(&header, &data, event, &regs);

	if (perf_output_begin(&handle, event, header.size * (top - at), 1, 1))
		return;

	for (; at < top; at++) {
		data.ip		= at->from;
		data.addr	= at->to;

		perf_output_sample(&handle, &header, &data, event);
	}

	perf_output_end(&handle);

	/* There's new data available. */
	event->hw.interrupts++;
	event->pending_kill = POLL_IN;
}

/*
 * PEBS
 */

static struct event_constraint intel_core_pebs_events[] = {
	PEBS_EVENT_CONSTRAINT(0x00c0, 0x1), /* INSTR_RETIRED.ANY */
	PEBS_EVENT_CONSTRAINT(0xfec1, 0x1), /* X87_OPS_RETIRED.ANY */
	PEBS_EVENT_CONSTRAINT(0x00c5, 0x1), /* BR_INST_RETIRED.MISPRED */
	PEBS_EVENT_CONSTRAINT(0x1fc7, 0x1), /* SIMD_INST_RETURED.ANY */
	PEBS_EVENT_CONSTRAINT(0x01cb, 0x1), /* MEM_LOAD_RETIRED.L1D_MISS */
	PEBS_EVENT_CONSTRAINT(0x02cb, 0x1), /* MEM_LOAD_RETIRED.L1D_LINE_MISS */
	PEBS_EVENT_CONSTRAINT(0x04cb, 0x1), /* MEM_LOAD_RETIRED.L2_MISS */
	PEBS_EVENT_CONSTRAINT(0x08cb, 0x1), /* MEM_LOAD_RETIRED.L2_LINE_MISS */
	PEBS_EVENT_CONSTRAINT(0x10cb, 0x1), /* MEM_LOAD_RETIRED.DTLB_MISS */
	EVENT_CONSTRAINT_END
};

static struct event_constraint intel_nehalem_pebs_events[] = {
	PEBS_EVENT_CONSTRAINT(0x00c0, 0xf), /* INSTR_RETIRED.ANY */
	PEBS_EVENT_CONSTRAINT(0xfec1, 0xf), /* X87_OPS_RETIRED.ANY */
	PEBS_EVENT_CONSTRAINT(0x00c5, 0xf), /* BR_INST_RETIRED.MISPRED */
	PEBS_EVENT_CONSTRAINT(0x1fc7, 0xf), /* SIMD_INST_RETURED.ANY */
	PEBS_EVENT_CONSTRAINT(0x01cb, 0xf), /* MEM_LOAD_RETIRED.L1D_MISS */
	PEBS_EVENT_CONSTRAINT(0x02cb, 0xf), /* MEM_LOAD_RETIRED.L1D_LINE_MISS */
	PEBS_EVENT_CONSTRAINT(0x04cb, 0xf), /* MEM_LOAD_RETIRED.L2_MISS */
	PEBS_EVENT_CONSTRAINT(0x08cb, 0xf), /* MEM_LOAD_RETIRED.L2_LINE_MISS */
	PEBS_EVENT_CONSTRAINT(0x10cb, 0xf), /* MEM_LOAD_RETIRED.DTLB_MISS */
	EVENT_CONSTRAINT_END
};

static struct event_constraint *
intel_pebs_constraints(struct perf_event *event)
{
	struct event_constraint *c;

	if (!event->attr.precise)
		return NULL;

	if (x86_pmu.pebs_constraints) {
		for_each_event_constraint(c, x86_pmu.pebs_constraints) {
			if ((event->hw.config & c->cmask) == c->code)
				return c;
		}
	}

	return &emptyconstraint;
}

static void intel_pmu_pebs_enable(struct hw_perf_event *hwc)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	u64 val = cpuc->pebs_enabled;

	hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;

	val |= 1ULL << hwc->idx;
	wrmsrl(MSR_IA32_PEBS_ENABLE, val);
}

static void intel_pmu_pebs_disable(struct hw_perf_event *hwc)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	u64 val = cpuc->pebs_enabled;

	val &= ~(1ULL << hwc->idx);
	wrmsrl(MSR_IA32_PEBS_ENABLE, val);

	hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
}

static void intel_pmu_pebs_enable_all(void)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);

	if (cpuc->pebs_enabled)
		wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
}

static void intel_pmu_pebs_disable_all(void)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);

	if (cpuc->pebs_enabled)
		wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
}

static int intel_pmu_save_and_restart(struct perf_event *event);
static void intel_pmu_disable_event(struct perf_event *event);

static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	struct debug_store *ds = cpuc->ds;
	struct perf_event *event = cpuc->events[0]; /* PMC0 only */
	struct pebs_record_core *at, *top;
	struct perf_sample_data data;
	struct pt_regs regs;
	int n;

	if (!event || !ds || !x86_pmu.pebs)
		return;

	intel_pmu_pebs_disable_all();

	at  = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;
	top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;

	if (top <= at)
		goto out;

	ds->pebs_index = ds->pebs_buffer_base;

	if (!intel_pmu_save_and_restart(event))
		goto out;

	perf_sample_data_init(&data, 0);
	data.period = event->hw.last_period;

	n = top - at;

	/*
	 * Should not happen, we program the threshold at 1 and do not
	 * set a reset value.
	 */
	WARN_ON_ONCE(n > 1);

	/*
	 * We use the interrupt regs as a base because the PEBS record
	 * does not contain a full regs set, specifically it seems to
	 * lack segment descriptors, which get used by things like
	 * user_mode().
	 *
	 * In the simple case fix up only the IP and BP,SP regs, for
	 * PERF_SAMPLE_IP and PERF_SAMPLE_CALLCHAIN to function properly.
	 * A possible PERF_SAMPLE_REGS will have to transfer all regs.
	 */
	regs = *iregs;
	regs.ip = at->ip;
	regs.bp = at->bp;
	regs.sp = at->sp;

	if (perf_event_overflow(event, 1, &data, &regs))
		intel_pmu_disable_event(event);

out:
	intel_pmu_pebs_enable_all();
}

static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)
{
	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
	struct debug_store *ds = cpuc->ds;
	struct pebs_record_nhm *at, *top;
	struct perf_sample_data data;
	struct perf_event *event = NULL;
	struct pt_regs regs;
	int bit, n;

	if (!ds || !x86_pmu.pebs)
		return;

	intel_pmu_pebs_disable_all();

	at  = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
	top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;

	if (top <= at)
		goto out;

	ds->pebs_index = ds->pebs_buffer_base;

	n = top - at;

	/*
	 * Should not happen, we program the threshold at 1 and do not
	 * set a reset value.
	 */
	WARN_ON_ONCE(n > MAX_PEBS_EVENTS);

	for ( ; at < top; at++) {
		for_each_bit(bit, (unsigned long *)&at->status, MAX_PEBS_EVENTS) {
			if (!cpuc->events[bit]->attr.precise)
				continue;

			event = cpuc->events[bit];
		}

		if (!event)
			continue;

		if (!intel_pmu_save_and_restart(event))
			continue;

		perf_sample_data_init(&data, 0);
		data.period = event->hw.last_period;

		/*
		 * See the comment in intel_pmu_drain_pebs_core()
		 */
		regs = *iregs;
		regs.ip = at->ip;
		regs.bp = at->bp;
		regs.sp = at->sp;

		if (perf_event_overflow(event, 1, &data, &regs))
			intel_pmu_disable_event(event);
	}
out:
	intel_pmu_pebs_enable_all();
}

/*
 * BTS, PEBS probe and setup
 */

static void intel_ds_init(void)
{
	/*
	 * No support for 32bit formats
	 */
	if (!boot_cpu_has(X86_FEATURE_DTES64))
		return;

	x86_pmu.bts  = boot_cpu_has(X86_FEATURE_BTS);
	x86_pmu.pebs = boot_cpu_has(X86_FEATURE_PEBS);
	if (x86_pmu.pebs) {
		int format = 0;

		if (x86_pmu.version > 1) {
			u64 capabilities;
			/*
			 * v2+ has a PEBS format field
			 */
			rdmsrl(MSR_IA32_PERF_CAPABILITIES, capabilities);
			format = (capabilities >> 8) & 0xf;
		}

		switch (format) {
		case 0:
			printk(KERN_CONT "PEBS v0, ");
			x86_pmu.pebs_record_size = sizeof(struct pebs_record_core);
			x86_pmu.drain_pebs = intel_pmu_drain_pebs_core;
			x86_pmu.pebs_constraints = intel_core_pebs_events;
			break;

		case 1:
			printk(KERN_CONT "PEBS v1, ");
			x86_pmu.pebs_record_size = sizeof(struct pebs_record_nhm);
			x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
			x86_pmu.pebs_constraints = intel_nehalem_pebs_events;
			break;

		default:
			printk(KERN_CONT "PEBS unknown format: %d, ", format);
			x86_pmu.pebs = 0;
			break;
		}
	}
}

#else /* CONFIG_CPU_SUP_INTEL */

static int reseve_ds_buffers(void)
{
	return 0;
}

static void release_ds_buffers(void)
{
}

#endif /* CONFIG_CPU_SUP_INTEL */
Commit	Line	Data
ca037701 PZ	1	#ifdef CONFIG_CPU_SUP_INTEL
	2
	3	/* The maximal number of PEBS events: */
	4	#define MAX_PEBS_EVENTS 4
	5
	6	/* The size of a BTS record in bytes: */
	7	#define BTS_RECORD_SIZE 24
	8
	9	#define BTS_BUFFER_SIZE (PAGE_SIZE << 4)
	10	#define PEBS_BUFFER_SIZE PAGE_SIZE
	11
	12	/*
	13	* pebs_record_32 for p4 and core not supported
	14
	15	struct pebs_record_32 {
	16	u32 flags, ip;
	17	u32 ax, bc, cx, dx;
	18	u32 si, di, bp, sp;
	19	};
	20
	21	*/
	22
	23	struct pebs_record_core {
	24	u64 flags, ip;
	25	u64 ax, bx, cx, dx;
	26	u64 si, di, bp, sp;
	27	u64 r8, r9, r10, r11;
	28	u64 r12, r13, r14, r15;
	29	};
	30
	31	struct pebs_record_nhm {
	32	u64 flags, ip;
	33	u64 ax, bx, cx, dx;
	34	u64 si, di, bp, sp;
	35	u64 r8, r9, r10, r11;
	36	u64 r12, r13, r14, r15;
	37	u64 status, dla, dse, lat;
	38	};
	39
	40	/*
	41	* Bits in the debugctlmsr controlling branch tracing.
	42	*/
	43	#define X86_DEBUGCTL_TR (1 << 6)
	44	#define X86_DEBUGCTL_BTS (1 << 7)
	45	#define X86_DEBUGCTL_BTINT (1 << 8)
	46	#define X86_DEBUGCTL_BTS_OFF_OS (1 << 9)
	47	#define X86_DEBUGCTL_BTS_OFF_USR (1 << 10)
	48
	49	/*
	50	* A debug store configuration.
	51	*
	52	* We only support architectures that use 64bit fields.
	53	*/
	54	struct debug_store {
	55	u64 bts_buffer_base;
	56	u64 bts_index;
	57	u64 bts_absolute_maximum;
	58	u64 bts_interrupt_threshold;
	59	u64 pebs_buffer_base;
	60	u64 pebs_index;
	61	u64 pebs_absolute_maximum;
	62	u64 pebs_interrupt_threshold;
	63	u64 pebs_event_reset[MAX_PEBS_EVENTS];
	64	};
65
66	static void init_debug_store_on_cpu(int cpu)
67	{
68	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
69
70	if (!ds)
71	return;
72
73	wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
74	(u32)((u64)(unsigned long)ds),
75	(u32)((u64)(unsigned long)ds >> 32));
76	}
77
78	static void fini_debug_store_on_cpu(int cpu)
79	{
80	if (!per_cpu(cpu_hw_events, cpu).ds)
81	return;
82
83	wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
84	}
85
86	static void release_ds_buffers(void)
87	{
88	int cpu;
89
90	if (!x86_pmu.bts && !x86_pmu.pebs)
91	return;
92
93	get_online_cpus();
94
95	for_each_online_cpu(cpu)
96	fini_debug_store_on_cpu(cpu);
97
98	for_each_possible_cpu(cpu) {
99	struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
100
101	if (!ds)
102	continue;
103
104	per_cpu(cpu_hw_events, cpu).ds = NULL;
105
106	kfree((void *)(unsigned long)ds->pebs_buffer_base);
107	kfree((void *)(unsigned long)ds->bts_buffer_base);
108	kfree(ds);
109	}
110
111	put_online_cpus();
112	}
113
114	static int reserve_ds_buffers(void)
115	{
116	int cpu, err = 0;
117
118	if (!x86_pmu.bts && !x86_pmu.pebs)
119	return 0;
120
121	get_online_cpus();
122
123	for_each_possible_cpu(cpu) {
124	struct debug_store *ds;
125	void *buffer;
126	int max, thresh;
127
128	err = -ENOMEM;
129	ds = kzalloc(sizeof(*ds), GFP_KERNEL);
130	if (unlikely(!ds)) {
131	kfree(buffer);
132	break;
133	}
134	per_cpu(cpu_hw_events, cpu).ds = ds;
135
136	if (x86_pmu.bts) {
137	buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
138	if (unlikely(!buffer))
139	break;
140
141	max = BTS_BUFFER_SIZE / BTS_RECORD_SIZE;
142	thresh = max / 16;
143
144	ds->bts_buffer_base = (u64)(unsigned long)buffer;
145	ds->bts_index = ds->bts_buffer_base;
146	ds->bts_absolute_maximum = ds->bts_buffer_base +
147	max * BTS_RECORD_SIZE;
148	ds->bts_interrupt_threshold = ds->bts_absolute_maximum -
149	thresh * BTS_RECORD_SIZE;
150	}
151
152	if (x86_pmu.pebs) {
153	buffer = kzalloc(PEBS_BUFFER_SIZE, GFP_KERNEL);
154	if (unlikely(!buffer))
155	break;
156
157	max = PEBS_BUFFER_SIZE / x86_pmu.pebs_record_size;
158
159	ds->pebs_buffer_base = (u64)(unsigned long)buffer;
160	ds->pebs_index = ds->pebs_buffer_base;
161	ds->pebs_absolute_maximum = ds->pebs_buffer_base +
162	max * x86_pmu.pebs_record_size;
163	/*
164	* Always use single record PEBS
165	*/
166	ds->pebs_interrupt_threshold = ds->pebs_buffer_base +
167	x86_pmu.pebs_record_size;
168	}
169
170	err = 0;
171	}
172
173	if (err)
174	release_ds_buffers();
175	else {
176	for_each_online_cpu(cpu)
177	init_debug_store_on_cpu(cpu);
178	}
179
180	put_online_cpus();
181
182	return err;
183	}
184
185	/*
186	* BTS
187	*/
188
189	static struct event_constraint bts_constraint =
190	EVENT_CONSTRAINT(0, 1ULL << X86_PMC_IDX_FIXED_BTS, 0);
191
192	static void intel_pmu_enable_bts(u64 config)
193	{
194	unsigned long debugctlmsr;
195
196	debugctlmsr = get_debugctlmsr();
197
198	debugctlmsr \|= X86_DEBUGCTL_TR;
199	debugctlmsr \|= X86_DEBUGCTL_BTS;
200	debugctlmsr \|= X86_DEBUGCTL_BTINT;
201
202	if (!(config & ARCH_PERFMON_EVENTSEL_OS))
203	debugctlmsr \|= X86_DEBUGCTL_BTS_OFF_OS;
204
205	if (!(config & ARCH_PERFMON_EVENTSEL_USR))
206	debugctlmsr \|= X86_DEBUGCTL_BTS_OFF_USR;
207
208	update_debugctlmsr(debugctlmsr);
209	}
210
211	static void intel_pmu_disable_bts(void)
212	{
213	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
214	unsigned long debugctlmsr;
215
216	if (!cpuc->ds)
217	return;
218
219	debugctlmsr = get_debugctlmsr();
220
221	debugctlmsr &=
222	~(X86_DEBUGCTL_TR \| X86_DEBUGCTL_BTS \| X86_DEBUGCTL_BTINT \|
223	X86_DEBUGCTL_BTS_OFF_OS \| X86_DEBUGCTL_BTS_OFF_USR);
224
225	update_debugctlmsr(debugctlmsr);
226	}
227
228	static void intel_pmu_drain_bts_buffer(void)
229	{
230	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
231	struct debug_store *ds = cpuc->ds;
232	struct bts_record {
233	u64 from;
234	u64 to;
235	u64 flags;
236	};
237	struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
238	struct bts_record at, top;
239	struct perf_output_handle handle;
240	struct perf_event_header header;
241	struct perf_sample_data data;
242	struct pt_regs regs;
243
244	if (!event)
245	return;
246
247	if (!ds)
248	return;
249
250	at = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
251	top = (struct bts_record *)(unsigned long)ds->bts_index;
252
253	if (top <= at)
254	return;
255
256	ds->bts_index = ds->bts_buffer_base;
257
258	perf_sample_data_init(&data, 0);
259	data.period = event->hw.last_period;
260	regs.ip = 0;
261
262	/*
263	* Prepare a generic sample, i.e. fill in the invariant fields.
264	* We will overwrite the from and to address before we output
265	* the sample.
266	*/
267	perf_prepare_sample(&header, &data, event, &regs);
268
269	if (perf_output_begin(&handle, event, header.size * (top - at), 1, 1))
270	return;
271
272	for (; at < top; at++) {
273	data.ip = at->from;
274	data.addr = at->to;
275
276	perf_output_sample(&handle, &header, &data, event);
277	}
278
279	perf_output_end(&handle);
280
281	/* There's new data available. */
282	event->hw.interrupts++;
283	event->pending_kill = POLL_IN;
284	}
285
286	/*
287	* PEBS
288	*/
289
290	static struct event_constraint intel_core_pebs_events[] = {
291	PEBS_EVENT_CONSTRAINT(0x00c0, 0x1), /* INSTR_RETIRED.ANY */
292	PEBS_EVENT_CONSTRAINT(0xfec1, 0x1), /* X87_OPS_RETIRED.ANY */
293	PEBS_EVENT_CONSTRAINT(0x00c5, 0x1), /* BR_INST_RETIRED.MISPRED */
294	PEBS_EVENT_CONSTRAINT(0x1fc7, 0x1), /* SIMD_INST_RETURED.ANY */
295	PEBS_EVENT_CONSTRAINT(0x01cb, 0x1), /* MEM_LOAD_RETIRED.L1D_MISS */
296	PEBS_EVENT_CONSTRAINT(0x02cb, 0x1), /* MEM_LOAD_RETIRED.L1D_LINE_MISS */
297	PEBS_EVENT_CONSTRAINT(0x04cb, 0x1), /* MEM_LOAD_RETIRED.L2_MISS */
298	PEBS_EVENT_CONSTRAINT(0x08cb, 0x1), /* MEM_LOAD_RETIRED.L2_LINE_MISS */
299	PEBS_EVENT_CONSTRAINT(0x10cb, 0x1), /* MEM_LOAD_RETIRED.DTLB_MISS */
300	EVENT_CONSTRAINT_END
301	};
302
303	static struct event_constraint intel_nehalem_pebs_events[] = {
304	PEBS_EVENT_CONSTRAINT(0x00c0, 0xf), /* INSTR_RETIRED.ANY */
305	PEBS_EVENT_CONSTRAINT(0xfec1, 0xf), /* X87_OPS_RETIRED.ANY */
306	PEBS_EVENT_CONSTRAINT(0x00c5, 0xf), /* BR_INST_RETIRED.MISPRED */
307	PEBS_EVENT_CONSTRAINT(0x1fc7, 0xf), /* SIMD_INST_RETURED.ANY */
308	PEBS_EVENT_CONSTRAINT(0x01cb, 0xf), /* MEM_LOAD_RETIRED.L1D_MISS */
309	PEBS_EVENT_CONSTRAINT(0x02cb, 0xf), /* MEM_LOAD_RETIRED.L1D_LINE_MISS */
310	PEBS_EVENT_CONSTRAINT(0x04cb, 0xf), /* MEM_LOAD_RETIRED.L2_MISS */
311	PEBS_EVENT_CONSTRAINT(0x08cb, 0xf), /* MEM_LOAD_RETIRED.L2_LINE_MISS */
312	PEBS_EVENT_CONSTRAINT(0x10cb, 0xf), /* MEM_LOAD_RETIRED.DTLB_MISS */
313	EVENT_CONSTRAINT_END
314	};
315
316	static struct event_constraint *
317	intel_pebs_constraints(struct perf_event *event)
318	{
319	struct event_constraint *c;
320
321	if (!event->attr.precise)
322	return NULL;
323
324	if (x86_pmu.pebs_constraints) {
325	for_each_event_constraint(c, x86_pmu.pebs_constraints) {
326	if ((event->hw.config & c->cmask) == c->code)
327	return c;
328	}
329	}
330
331	return &emptyconstraint;
332	}
333
334	static void intel_pmu_pebs_enable(struct hw_perf_event *hwc)
335	{
336	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
337	u64 val = cpuc->pebs_enabled;
338
339	hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
340
341	val \|= 1ULL << hwc->idx;
342	wrmsrl(MSR_IA32_PEBS_ENABLE, val);
343	}
344
345	static void intel_pmu_pebs_disable(struct hw_perf_event *hwc)
346	{
347	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
348	u64 val = cpuc->pebs_enabled;
349
350	val &= ~(1ULL << hwc->idx);
351	wrmsrl(MSR_IA32_PEBS_ENABLE, val);
352
353	hwc->config \|= ARCH_PERFMON_EVENTSEL_INT;
354	}
355
356	static void intel_pmu_pebs_enable_all(void)
357	{
358	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
359
360	if (cpuc->pebs_enabled)
361	wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
362	}
363
364	static void intel_pmu_pebs_disable_all(void)
365	{
366	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
367
368	if (cpuc->pebs_enabled)
369	wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
370	}
371
372	static int intel_pmu_save_and_restart(struct perf_event *event);
373	static void intel_pmu_disable_event(struct perf_event *event);
374
375	static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
376	{
377	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
378	struct debug_store *ds = cpuc->ds;
379	struct perf_event event = cpuc->events[0]; / PMC0 only */
380	struct pebs_record_core at, top;
381	struct perf_sample_data data;
382	struct pt_regs regs;
383	int n;
384
385	if (!event \|\| !ds \|\| !x86_pmu.pebs)
386	return;
387
388	intel_pmu_pebs_disable_all();
389
390	at = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;
391	top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;
392
393	if (top <= at)
394	goto out;
395
396	ds->pebs_index = ds->pebs_buffer_base;
397
398	if (!intel_pmu_save_and_restart(event))
399	goto out;
400
401	perf_sample_data_init(&data, 0);
402	data.period = event->hw.last_period;
403
404	n = top - at;
405
406	/*
407	* Should not happen, we program the threshold at 1 and do not
408	* set a reset value.
409	*/
410	WARN_ON_ONCE(n > 1);
411
412	/*
413	* We use the interrupt regs as a base because the PEBS record
414	* does not contain a full regs set, specifically it seems to
415	* lack segment descriptors, which get used by things like
416	* user_mode().
417	*
418	* In the simple case fix up only the IP and BP,SP regs, for
419	* PERF_SAMPLE_IP and PERF_SAMPLE_CALLCHAIN to function properly.
420	* A possible PERF_SAMPLE_REGS will have to transfer all regs.
421	*/
422	regs = *iregs;
423	regs.ip = at->ip;
424	regs.bp = at->bp;
425	regs.sp = at->sp;
426
427	if (perf_event_overflow(event, 1, &data, &regs))
428	intel_pmu_disable_event(event);
429
430	out:
431	intel_pmu_pebs_enable_all();
432	}
433
434	static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)
435	{
436	struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
437	struct debug_store *ds = cpuc->ds;
438	struct pebs_record_nhm at, top;
439	struct perf_sample_data data;
440	struct perf_event *event = NULL;
441	struct pt_regs regs;
442	int bit, n;
443
444	if (!ds \|\| !x86_pmu.pebs)
445	return;
446
447	intel_pmu_pebs_disable_all();
448
449	at = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
450	top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;
451
452	if (top <= at)
453	goto out;
454
455	ds->pebs_index = ds->pebs_buffer_base;
456
457	n = top - at;
458
459	/*
460	* Should not happen, we program the threshold at 1 and do not
461	* set a reset value.
462	*/
463	WARN_ON_ONCE(n > MAX_PEBS_EVENTS);
464
465	for ( ; at < top; at++) {
466	for_each_bit(bit, (unsigned long *)&at->status, MAX_PEBS_EVENTS) {
467	if (!cpuc->events[bit]->attr.precise)
468	continue;
469
470	event = cpuc->events[bit];
471	}
472
473	if (!event)
474	continue;
475
476	if (!intel_pmu_save_and_restart(event))
477	continue;
478
479	perf_sample_data_init(&data, 0);
480	data.period = event->hw.last_period;
481
482	/*
483	* See the comment in intel_pmu_drain_pebs_core()
484	*/
485	regs = *iregs;
486	regs.ip = at->ip;
487	regs.bp = at->bp;
488	regs.sp = at->sp;
489
490	if (perf_event_overflow(event, 1, &data, &regs))
491	intel_pmu_disable_event(event);
492	}
493	out:
494	intel_pmu_pebs_enable_all();
495	}
496
497	/*
498	* BTS, PEBS probe and setup
499	*/
500
501	static void intel_ds_init(void)
502	{
503	/*
504	* No support for 32bit formats
505	*/
506	if (!boot_cpu_has(X86_FEATURE_DTES64))
507	return;
508
509	x86_pmu.bts = boot_cpu_has(X86_FEATURE_BTS);
510	x86_pmu.pebs = boot_cpu_has(X86_FEATURE_PEBS);
511	if (x86_pmu.pebs) {
512	int format = 0;
513
514	if (x86_pmu.version > 1) {
515	u64 capabilities;
516	/*
517	* v2+ has a PEBS format field
518	*/
519	rdmsrl(MSR_IA32_PERF_CAPABILITIES, capabilities);
520	format = (capabilities >> 8) & 0xf;
521	}
522
523	switch (format) {
524	case 0:
525	printk(KERN_CONT "PEBS v0, ");
526	x86_pmu.pebs_record_size = sizeof(struct pebs_record_core);
527	x86_pmu.drain_pebs = intel_pmu_drain_pebs_core;
528	x86_pmu.pebs_constraints = intel_core_pebs_events;
529	break;
530
531	case 1:
532	printk(KERN_CONT "PEBS v1, ");
533	x86_pmu.pebs_record_size = sizeof(struct pebs_record_nhm);
534	x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
535	x86_pmu.pebs_constraints = intel_nehalem_pebs_events;
536	break;
537
538	default:
539	printk(KERN_CONT "PEBS unknown format: %d, ", format);
540	x86_pmu.pebs = 0;
541	break;
542	}
543	}
544	}
545
546	#else /* CONFIG_CPU_SUP_INTEL */
547
548	static int reseve_ds_buffers(void)
549	{
550	return 0;
551	}
552
553	static void release_ds_buffers(void)
554	{
555	}
556
557	#endif /* CONFIG_CPU_SUP_INTEL */