2 * Copyright © 2015-2016 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 * Robert Bragg <robert@sixbynine.org>
29 * DOC: i915 Perf Overview
31 * Gen graphics supports a large number of performance counters that can help
32 * driver and application developers understand and optimize their use of the
35 * This i915 perf interface enables userspace to configure and open a file
36 * descriptor representing a stream of GPU metrics which can then be read() as
37 * a stream of sample records.
39 * The interface is particularly suited to exposing buffered metrics that are
40 * captured by DMA from the GPU, unsynchronized with and unrelated to the CPU.
42 * Streams representing a single context are accessible to applications with a
43 * corresponding drm file descriptor, such that OpenGL can use the interface
44 * without special privileges. Access to system-wide metrics requires root
45 * privileges by default, unless changed via the dev.i915.perf_event_paranoid
51 * DOC: i915 Perf History and Comparison with Core Perf
53 * The interface was initially inspired by the core Perf infrastructure but
54 * some notable differences are:
56 * i915 perf file descriptors represent a "stream" instead of an "event"; where
57 * a perf event primarily corresponds to a single 64bit value, while a stream
58 * might sample sets of tightly-coupled counters, depending on the
59 * configuration. For example the Gen OA unit isn't designed to support
60 * orthogonal configurations of individual counters; it's configured for a set
61 * of related counters. Samples for an i915 perf stream capturing OA metrics
62 * will include a set of counter values packed in a compact HW specific format.
63 * The OA unit supports a number of different packing formats which can be
64 * selected by the user opening the stream. Perf has support for grouping
65 * events, but each event in the group is configured, validated and
66 * authenticated individually with separate system calls.
68 * i915 perf stream configurations are provided as an array of u64 (key,value)
69 * pairs, instead of a fixed struct with multiple miscellaneous config members,
70 * interleaved with event-type specific members.
72 * i915 perf doesn't support exposing metrics via an mmap'd circular buffer.
73 * The supported metrics are being written to memory by the GPU unsynchronized
74 * with the CPU, using HW specific packing formats for counter sets. Sometimes
75 * the constraints on HW configuration require reports to be filtered before it
76 * would be acceptable to expose them to unprivileged applications - to hide
77 * the metrics of other processes/contexts. For these use cases a read() based
78 * interface is a good fit, and provides an opportunity to filter data as it
79 * gets copied from the GPU mapped buffers to userspace buffers.
82 * Issues hit with first prototype based on Core Perf
83 * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
85 * The first prototype of this driver was based on the core perf
86 * infrastructure, and while we did make that mostly work, with some changes to
87 * perf, we found we were breaking or working around too many assumptions baked
88 * into perf's currently cpu centric design.
90 * In the end we didn't see a clear benefit to making perf's implementation and
91 * interface more complex by changing design assumptions while we knew we still
92 * wouldn't be able to use any existing perf based userspace tools.
94 * Also considering the Gen specific nature of the Observability hardware and
95 * how userspace will sometimes need to combine i915 perf OA metrics with
96 * side-band OA data captured via MI_REPORT_PERF_COUNT commands; we're
97 * expecting the interface to be used by a platform specific userspace such as
98 * OpenGL or tools. This is to say; we aren't inherently missing out on having
99 * a standard vendor/architecture agnostic interface by not using perf.
102 * For posterity, in case we might re-visit trying to adapt core perf to be
103 * better suited to exposing i915 metrics these were the main pain points we
106 * - The perf based OA PMU driver broke some significant design assumptions:
108 * Existing perf pmus are used for profiling work on a cpu and we were
109 * introducing the idea of _IS_DEVICE pmus with different security
110 * implications, the need to fake cpu-related data (such as user/kernel
111 * registers) to fit with perf's current design, and adding _DEVICE records
112 * as a way to forward device-specific status records.
114 * The OA unit writes reports of counters into a circular buffer, without
115 * involvement from the CPU, making our PMU driver the first of a kind.
117 * Given the way we were periodically forward data from the GPU-mapped, OA
118 * buffer to perf's buffer, those bursts of sample writes looked to perf like
119 * we were sampling too fast and so we had to subvert its throttling checks.
121 * Perf supports groups of counters and allows those to be read via
122 * transactions internally but transactions currently seem designed to be
123 * explicitly initiated from the cpu (say in response to a userspace read())
124 * and while we could pull a report out of the OA buffer we can't
125 * trigger a report from the cpu on demand.
127 * Related to being report based; the OA counters are configured in HW as a
128 * set while perf generally expects counter configurations to be orthogonal.
129 * Although counters can be associated with a group leader as they are
130 * opened, there's no clear precedent for being able to provide group-wide
131 * configuration attributes (for example we want to let userspace choose the
132 * OA unit report format used to capture all counters in a set, or specify a
133 * GPU context to filter metrics on). We avoided using perf's grouping
134 * feature and forwarded OA reports to userspace via perf's 'raw' sample
135 * field. This suited our userspace well considering how coupled the counters
136 * are when dealing with normalizing. It would be inconvenient to split
137 * counters up into separate events, only to require userspace to recombine
138 * them. For Mesa it's also convenient to be forwarded raw, periodic reports
139 * for combining with the side-band raw reports it captures using
140 * MI_REPORT_PERF_COUNT commands.
142 * - As a side note on perf's grouping feature; there was also some concern
143 * that using PERF_FORMAT_GROUP as a way to pack together counter values
144 * would quite drastically inflate our sample sizes, which would likely
145 * lower the effective sampling resolutions we could use when the available
146 * memory bandwidth is limited.
148 * With the OA unit's report formats, counters are packed together as 32
149 * or 40bit values, with the largest report size being 256 bytes.
151 * PERF_FORMAT_GROUP values are 64bit, but there doesn't appear to be a
152 * documented ordering to the values, implying PERF_FORMAT_ID must also be
153 * used to add a 64bit ID before each value; giving 16 bytes per counter.
155 * Related to counter orthogonality; we can't time share the OA unit, while
156 * event scheduling is a central design idea within perf for allowing
157 * userspace to open + enable more events than can be configured in HW at any
158 * one time. The OA unit is not designed to allow re-configuration while in
159 * use. We can't reconfigure the OA unit without losing internal OA unit
160 * state which we can't access explicitly to save and restore. Reconfiguring
161 * the OA unit is also relatively slow, involving ~100 register writes. From
162 * userspace Mesa also depends on a stable OA configuration when emitting
163 * MI_REPORT_PERF_COUNT commands and importantly the OA unit can't be
164 * disabled while there are outstanding MI_RPC commands lest we hang the
167 * The contents of sample records aren't extensible by device drivers (i.e.
168 * the sample_type bits). As an example; Sourab Gupta had been looking to
169 * attach GPU timestamps to our OA samples. We were shoehorning OA reports
170 * into sample records by using the 'raw' field, but it's tricky to pack more
171 * than one thing into this field because events/core.c currently only lets a
172 * pmu give a single raw data pointer plus len which will be copied into the
173 * ring buffer. To include more than the OA report we'd have to copy the
174 * report into an intermediate larger buffer. I'd been considering allowing a
175 * vector of data+len values to be specified for copying the raw data, but
176 * it felt like a kludge to being using the raw field for this purpose.
178 * - It felt like our perf based PMU was making some technical compromises
179 * just for the sake of using perf:
181 * perf_event_open() requires events to either relate to a pid or a specific
182 * cpu core, while our device pmu related to neither. Events opened with a
183 * pid will be automatically enabled/disabled according to the scheduling of
184 * that process - so not appropriate for us. When an event is related to a
185 * cpu id, perf ensures pmu methods will be invoked via an inter process
186 * interrupt on that core. To avoid invasive changes our userspace opened OA
187 * perf events for a specific cpu. This was workable but it meant the
188 * majority of the OA driver ran in atomic context, including all OA report
189 * forwarding, which wasn't really necessary in our case and seems to make
190 * our locking requirements somewhat complex as we handled the interaction
191 * with the rest of the i915 driver.
194 #include <linux/anon_inodes.h>
195 #include <linux/sizes.h>
196 #include <linux/uuid.h>
198 #include "i915_drv.h"
199 #include "i915_oa_hsw.h"
200 #include "i915_oa_bdw.h"
201 #include "i915_oa_chv.h"
202 #include "i915_oa_sklgt2.h"
203 #include "i915_oa_sklgt3.h"
204 #include "i915_oa_sklgt4.h"
205 #include "i915_oa_bxt.h"
206 #include "i915_oa_kblgt2.h"
207 #include "i915_oa_kblgt3.h"
208 #include "i915_oa_glk.h"
210 /* HW requires this to be a power of two, between 128k and 16M, though driver
211 * is currently generally designed assuming the largest 16M size is used such
212 * that the overflow cases are unlikely in normal operation.
214 #define OA_BUFFER_SIZE SZ_16M
216 #define OA_TAKEN(tail, head) ((tail - head) & (OA_BUFFER_SIZE - 1))
219 * DOC: OA Tail Pointer Race
221 * There's a HW race condition between OA unit tail pointer register updates and
222 * writes to memory whereby the tail pointer can sometimes get ahead of what's
223 * been written out to the OA buffer so far (in terms of what's visible to the
226 * Although this can be observed explicitly while copying reports to userspace
227 * by checking for a zeroed report-id field in tail reports, we want to account
228 * for this earlier, as part of the oa_buffer_check to avoid lots of redundant
231 * In effect we define a tail pointer for reading that lags the real tail
232 * pointer by at least %OA_TAIL_MARGIN_NSEC nanoseconds, which gives enough
233 * time for the corresponding reports to become visible to the CPU.
235 * To manage this we actually track two tail pointers:
236 * 1) An 'aging' tail with an associated timestamp that is tracked until we
237 * can trust the corresponding data is visible to the CPU; at which point
238 * it is considered 'aged'.
239 * 2) An 'aged' tail that can be used for read()ing.
241 * The two separate pointers let us decouple read()s from tail pointer aging.
243 * The tail pointers are checked and updated at a limited rate within a hrtimer
244 * callback (the same callback that is used for delivering POLLIN events)
246 * Initially the tails are marked invalid with %INVALID_TAIL_PTR which
247 * indicates that an updated tail pointer is needed.
249 * Most of the implementation details for this workaround are in
250 * oa_buffer_check_unlocked() and _append_oa_reports()
252 * Note for posterity: previously the driver used to define an effective tail
253 * pointer that lagged the real pointer by a 'tail margin' measured in bytes
254 * derived from %OA_TAIL_MARGIN_NSEC and the configured sampling frequency.
255 * This was flawed considering that the OA unit may also automatically generate
256 * non-periodic reports (such as on context switch) or the OA unit may be
257 * enabled without any periodic sampling.
259 #define OA_TAIL_MARGIN_NSEC 100000ULL
260 #define INVALID_TAIL_PTR 0xffffffff
262 /* frequency for checking whether the OA unit has written new reports to the
263 * circular OA buffer...
265 #define POLL_FREQUENCY 200
266 #define POLL_PERIOD (NSEC_PER_SEC / POLL_FREQUENCY)
268 /* for sysctl proc_dointvec_minmax of dev.i915.perf_stream_paranoid */
271 static u32 i915_perf_stream_paranoid
= true;
273 /* The maximum exponent the hardware accepts is 63 (essentially it selects one
274 * of the 64bit timestamp bits to trigger reports from) but there's currently
275 * no known use case for sampling as infrequently as once per 47 thousand years.
277 * Since the timestamps included in OA reports are only 32bits it seems
278 * reasonable to limit the OA exponent where it's still possible to account for
279 * overflow in OA report timestamps.
281 #define OA_EXPONENT_MAX 31
283 #define INVALID_CTX_ID 0xffffffff
285 /* On Gen8+ automatically triggered OA reports include a 'reason' field... */
286 #define OAREPORT_REASON_MASK 0x3f
287 #define OAREPORT_REASON_SHIFT 19
288 #define OAREPORT_REASON_TIMER (1<<0)
289 #define OAREPORT_REASON_CTX_SWITCH (1<<3)
290 #define OAREPORT_REASON_CLK_RATIO (1<<5)
293 /* For sysctl proc_dointvec_minmax of i915_oa_max_sample_rate
295 * The highest sampling frequency we can theoretically program the OA unit
296 * with is always half the timestamp frequency: E.g. 6.25Mhz for Haswell.
298 * Initialized just before we register the sysctl parameter.
300 static int oa_sample_rate_hard_limit
;
302 /* Theoretically we can program the OA unit to sample every 160ns but don't
303 * allow that by default unless root...
305 * The default threshold of 100000Hz is based on perf's similar
306 * kernel.perf_event_max_sample_rate sysctl parameter.
308 static u32 i915_oa_max_sample_rate
= 100000;
310 /* XXX: beware if future OA HW adds new report formats that the current
311 * code assumes all reports have a power-of-two size and ~(size - 1) can
312 * be used as a mask to align the OA tail pointer.
314 static struct i915_oa_format hsw_oa_formats
[I915_OA_FORMAT_MAX
] = {
315 [I915_OA_FORMAT_A13
] = { 0, 64 },
316 [I915_OA_FORMAT_A29
] = { 1, 128 },
317 [I915_OA_FORMAT_A13_B8_C8
] = { 2, 128 },
318 /* A29_B8_C8 Disallowed as 192 bytes doesn't factor into buffer size */
319 [I915_OA_FORMAT_B4_C8
] = { 4, 64 },
320 [I915_OA_FORMAT_A45_B8_C8
] = { 5, 256 },
321 [I915_OA_FORMAT_B4_C8_A16
] = { 6, 128 },
322 [I915_OA_FORMAT_C4_B8
] = { 7, 64 },
325 static struct i915_oa_format gen8_plus_oa_formats
[I915_OA_FORMAT_MAX
] = {
326 [I915_OA_FORMAT_A12
] = { 0, 64 },
327 [I915_OA_FORMAT_A12_B8_C8
] = { 2, 128 },
328 [I915_OA_FORMAT_A32u40_A4u32_B8_C8
] = { 5, 256 },
329 [I915_OA_FORMAT_C4_B8
] = { 7, 64 },
332 #define SAMPLE_OA_REPORT (1<<0)
335 * struct perf_open_properties - for validated properties given to open a stream
336 * @sample_flags: `DRM_I915_PERF_PROP_SAMPLE_*` properties are tracked as flags
337 * @single_context: Whether a single or all gpu contexts should be monitored
338 * @ctx_handle: A gem ctx handle for use with @single_context
339 * @metrics_set: An ID for an OA unit metric set advertised via sysfs
340 * @oa_format: An OA unit HW report format
341 * @oa_periodic: Whether to enable periodic OA unit sampling
342 * @oa_period_exponent: The OA unit sampling period is derived from this
344 * As read_properties_unlocked() enumerates and validates the properties given
345 * to open a stream of metrics the configuration is built up in the structure
346 * which starts out zero initialized.
348 struct perf_open_properties
{
351 u64 single_context
:1;
354 /* OA sampling state */
358 int oa_period_exponent
;
361 static void free_oa_config(struct drm_i915_private
*dev_priv
,
362 struct i915_oa_config
*oa_config
)
364 if (!PTR_ERR(oa_config
->flex_regs
))
365 kfree(oa_config
->flex_regs
);
366 if (!PTR_ERR(oa_config
->b_counter_regs
))
367 kfree(oa_config
->b_counter_regs
);
368 if (!PTR_ERR(oa_config
->mux_regs
))
369 kfree(oa_config
->mux_regs
);
373 static void put_oa_config(struct drm_i915_private
*dev_priv
,
374 struct i915_oa_config
*oa_config
)
376 if (!atomic_dec_and_test(&oa_config
->ref_count
))
379 free_oa_config(dev_priv
, oa_config
);
382 static int get_oa_config(struct drm_i915_private
*dev_priv
,
384 struct i915_oa_config
**out_config
)
388 if (metrics_set
== 1) {
389 *out_config
= &dev_priv
->perf
.oa
.test_config
;
390 atomic_inc(&dev_priv
->perf
.oa
.test_config
.ref_count
);
394 ret
= mutex_lock_interruptible(&dev_priv
->perf
.metrics_lock
);
398 *out_config
= idr_find(&dev_priv
->perf
.metrics_idr
, metrics_set
);
402 atomic_inc(&(*out_config
)->ref_count
);
404 mutex_unlock(&dev_priv
->perf
.metrics_lock
);
409 static u32
gen8_oa_hw_tail_read(struct drm_i915_private
*dev_priv
)
411 return I915_READ(GEN8_OATAILPTR
) & GEN8_OATAILPTR_MASK
;
414 static u32
gen7_oa_hw_tail_read(struct drm_i915_private
*dev_priv
)
416 u32 oastatus1
= I915_READ(GEN7_OASTATUS1
);
418 return oastatus1
& GEN7_OASTATUS1_TAIL_MASK
;
422 * oa_buffer_check_unlocked - check for data and update tail ptr state
423 * @dev_priv: i915 device instance
425 * This is either called via fops (for blocking reads in user ctx) or the poll
426 * check hrtimer (atomic ctx) to check the OA buffer tail pointer and check
427 * if there is data available for userspace to read.
429 * This function is central to providing a workaround for the OA unit tail
430 * pointer having a race with respect to what data is visible to the CPU.
431 * It is responsible for reading tail pointers from the hardware and giving
432 * the pointers time to 'age' before they are made available for reading.
433 * (See description of OA_TAIL_MARGIN_NSEC above for further details.)
435 * Besides returning true when there is data available to read() this function
436 * also has the side effect of updating the oa_buffer.tails[], .aging_timestamp
437 * and .aged_tail_idx state used for reading.
439 * Note: It's safe to read OA config state here unlocked, assuming that this is
440 * only called while the stream is enabled, while the global OA configuration
443 * Returns: %true if the OA buffer contains data, else %false
445 static bool oa_buffer_check_unlocked(struct drm_i915_private
*dev_priv
)
447 int report_size
= dev_priv
->perf
.oa
.oa_buffer
.format_size
;
449 unsigned int aged_idx
;
450 u32 head
, hw_tail
, aged_tail
, aging_tail
;
453 /* We have to consider the (unlikely) possibility that read() errors
454 * could result in an OA buffer reset which might reset the head,
455 * tails[] and aged_tail state.
457 spin_lock_irqsave(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
, flags
);
459 /* NB: The head we observe here might effectively be a little out of
460 * date (between head and tails[aged_idx].offset if there is currently
461 * a read() in progress.
463 head
= dev_priv
->perf
.oa
.oa_buffer
.head
;
465 aged_idx
= dev_priv
->perf
.oa
.oa_buffer
.aged_tail_idx
;
466 aged_tail
= dev_priv
->perf
.oa
.oa_buffer
.tails
[aged_idx
].offset
;
467 aging_tail
= dev_priv
->perf
.oa
.oa_buffer
.tails
[!aged_idx
].offset
;
469 hw_tail
= dev_priv
->perf
.oa
.ops
.oa_hw_tail_read(dev_priv
);
471 /* The tail pointer increases in 64 byte increments,
472 * not in report_size steps...
474 hw_tail
&= ~(report_size
- 1);
476 now
= ktime_get_mono_fast_ns();
478 /* Update the aged tail
480 * Flip the tail pointer available for read()s once the aging tail is
481 * old enough to trust that the corresponding data will be visible to
484 * Do this before updating the aging pointer in case we may be able to
485 * immediately start aging a new pointer too (if new data has become
486 * available) without needing to wait for a later hrtimer callback.
488 if (aging_tail
!= INVALID_TAIL_PTR
&&
489 ((now
- dev_priv
->perf
.oa
.oa_buffer
.aging_timestamp
) >
490 OA_TAIL_MARGIN_NSEC
)) {
493 dev_priv
->perf
.oa
.oa_buffer
.aged_tail_idx
= aged_idx
;
495 aged_tail
= aging_tail
;
497 /* Mark that we need a new pointer to start aging... */
498 dev_priv
->perf
.oa
.oa_buffer
.tails
[!aged_idx
].offset
= INVALID_TAIL_PTR
;
499 aging_tail
= INVALID_TAIL_PTR
;
502 /* Update the aging tail
504 * We throttle aging tail updates until we have a new tail that
505 * represents >= one report more data than is already available for
506 * reading. This ensures there will be enough data for a successful
507 * read once this new pointer has aged and ensures we will give the new
508 * pointer time to age.
510 if (aging_tail
== INVALID_TAIL_PTR
&&
511 (aged_tail
== INVALID_TAIL_PTR
||
512 OA_TAKEN(hw_tail
, aged_tail
) >= report_size
)) {
513 struct i915_vma
*vma
= dev_priv
->perf
.oa
.oa_buffer
.vma
;
514 u32 gtt_offset
= i915_ggtt_offset(vma
);
516 /* Be paranoid and do a bounds check on the pointer read back
517 * from hardware, just in case some spurious hardware condition
518 * could put the tail out of bounds...
520 if (hw_tail
>= gtt_offset
&&
521 hw_tail
< (gtt_offset
+ OA_BUFFER_SIZE
)) {
522 dev_priv
->perf
.oa
.oa_buffer
.tails
[!aged_idx
].offset
=
523 aging_tail
= hw_tail
;
524 dev_priv
->perf
.oa
.oa_buffer
.aging_timestamp
= now
;
526 DRM_ERROR("Ignoring spurious out of range OA buffer tail pointer = %u\n",
531 spin_unlock_irqrestore(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
, flags
);
533 return aged_tail
== INVALID_TAIL_PTR
?
534 false : OA_TAKEN(aged_tail
, head
) >= report_size
;
538 * append_oa_status - Appends a status record to a userspace read() buffer.
539 * @stream: An i915-perf stream opened for OA metrics
540 * @buf: destination buffer given by userspace
541 * @count: the number of bytes userspace wants to read
542 * @offset: (inout): the current position for writing into @buf
543 * @type: The kind of status to report to userspace
545 * Writes a status record (such as `DRM_I915_PERF_RECORD_OA_REPORT_LOST`)
546 * into the userspace read() buffer.
548 * The @buf @offset will only be updated on success.
550 * Returns: 0 on success, negative error code on failure.
552 static int append_oa_status(struct i915_perf_stream
*stream
,
556 enum drm_i915_perf_record_type type
)
558 struct drm_i915_perf_record_header header
= { type
, 0, sizeof(header
) };
560 if ((count
- *offset
) < header
.size
)
563 if (copy_to_user(buf
+ *offset
, &header
, sizeof(header
)))
566 (*offset
) += header
.size
;
572 * append_oa_sample - Copies single OA report into userspace read() buffer.
573 * @stream: An i915-perf stream opened for OA metrics
574 * @buf: destination buffer given by userspace
575 * @count: the number of bytes userspace wants to read
576 * @offset: (inout): the current position for writing into @buf
577 * @report: A single OA report to (optionally) include as part of the sample
579 * The contents of a sample are configured through `DRM_I915_PERF_PROP_SAMPLE_*`
580 * properties when opening a stream, tracked as `stream->sample_flags`. This
581 * function copies the requested components of a single sample to the given
584 * The @buf @offset will only be updated on success.
586 * Returns: 0 on success, negative error code on failure.
588 static int append_oa_sample(struct i915_perf_stream
*stream
,
594 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
595 int report_size
= dev_priv
->perf
.oa
.oa_buffer
.format_size
;
596 struct drm_i915_perf_record_header header
;
597 u32 sample_flags
= stream
->sample_flags
;
599 header
.type
= DRM_I915_PERF_RECORD_SAMPLE
;
601 header
.size
= stream
->sample_size
;
603 if ((count
- *offset
) < header
.size
)
607 if (copy_to_user(buf
, &header
, sizeof(header
)))
609 buf
+= sizeof(header
);
611 if (sample_flags
& SAMPLE_OA_REPORT
) {
612 if (copy_to_user(buf
, report
, report_size
))
616 (*offset
) += header
.size
;
622 * Copies all buffered OA reports into userspace read() buffer.
623 * @stream: An i915-perf stream opened for OA metrics
624 * @buf: destination buffer given by userspace
625 * @count: the number of bytes userspace wants to read
626 * @offset: (inout): the current position for writing into @buf
628 * Notably any error condition resulting in a short read (-%ENOSPC or
629 * -%EFAULT) will be returned even though one or more records may
630 * have been successfully copied. In this case it's up to the caller
631 * to decide if the error should be squashed before returning to
634 * Note: reports are consumed from the head, and appended to the
635 * tail, so the tail chases the head?... If you think that's mad
636 * and back-to-front you're not alone, but this follows the
637 * Gen PRM naming convention.
639 * Returns: 0 on success, negative error code on failure.
641 static int gen8_append_oa_reports(struct i915_perf_stream
*stream
,
646 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
647 int report_size
= dev_priv
->perf
.oa
.oa_buffer
.format_size
;
648 u8
*oa_buf_base
= dev_priv
->perf
.oa
.oa_buffer
.vaddr
;
649 u32 gtt_offset
= i915_ggtt_offset(dev_priv
->perf
.oa
.oa_buffer
.vma
);
650 u32 mask
= (OA_BUFFER_SIZE
- 1);
651 size_t start_offset
= *offset
;
653 unsigned int aged_tail_idx
;
658 if (WARN_ON(!stream
->enabled
))
661 spin_lock_irqsave(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
, flags
);
663 head
= dev_priv
->perf
.oa
.oa_buffer
.head
;
664 aged_tail_idx
= dev_priv
->perf
.oa
.oa_buffer
.aged_tail_idx
;
665 tail
= dev_priv
->perf
.oa
.oa_buffer
.tails
[aged_tail_idx
].offset
;
667 spin_unlock_irqrestore(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
, flags
);
670 * An invalid tail pointer here means we're still waiting for the poll
671 * hrtimer callback to give us a pointer
673 if (tail
== INVALID_TAIL_PTR
)
677 * NB: oa_buffer.head/tail include the gtt_offset which we don't want
678 * while indexing relative to oa_buf_base.
684 * An out of bounds or misaligned head or tail pointer implies a driver
685 * bug since we validate + align the tail pointers we read from the
686 * hardware and we are in full control of the head pointer which should
687 * only be incremented by multiples of the report size (notably also
688 * all a power of two).
690 if (WARN_ONCE(head
> OA_BUFFER_SIZE
|| head
% report_size
||
691 tail
> OA_BUFFER_SIZE
|| tail
% report_size
,
692 "Inconsistent OA buffer pointers: head = %u, tail = %u\n",
698 (taken
= OA_TAKEN(tail
, head
));
699 head
= (head
+ report_size
) & mask
) {
700 u8
*report
= oa_buf_base
+ head
;
701 u32
*report32
= (void *)report
;
706 * All the report sizes factor neatly into the buffer
707 * size so we never expect to see a report split
708 * between the beginning and end of the buffer.
710 * Given the initial alignment check a misalignment
711 * here would imply a driver bug that would result
714 if (WARN_ON((OA_BUFFER_SIZE
- head
) < report_size
)) {
715 DRM_ERROR("Spurious OA head ptr: non-integral report offset\n");
720 * The reason field includes flags identifying what
721 * triggered this specific report (mostly timer
722 * triggered or e.g. due to a context switch).
724 * This field is never expected to be zero so we can
725 * check that the report isn't invalid before copying
728 reason
= ((report32
[0] >> OAREPORT_REASON_SHIFT
) &
729 OAREPORT_REASON_MASK
);
731 if (__ratelimit(&dev_priv
->perf
.oa
.spurious_report_rs
))
732 DRM_NOTE("Skipping spurious, invalid OA report\n");
737 * XXX: Just keep the lower 21 bits for now since I'm not
738 * entirely sure if the HW touches any of the higher bits in
741 ctx_id
= report32
[2] & 0x1fffff;
744 * Squash whatever is in the CTX_ID field if it's marked as
745 * invalid to be sure we avoid false-positive, single-context
748 * Note: that we don't clear the valid_ctx_bit so userspace can
749 * understand that the ID has been squashed by the kernel.
751 if (!(report32
[0] & dev_priv
->perf
.oa
.gen8_valid_ctx_bit
))
752 ctx_id
= report32
[2] = INVALID_CTX_ID
;
755 * NB: For Gen 8 the OA unit no longer supports clock gating
756 * off for a specific context and the kernel can't securely
757 * stop the counters from updating as system-wide / global
760 * Automatic reports now include a context ID so reports can be
761 * filtered on the cpu but it's not worth trying to
762 * automatically subtract/hide counter progress for other
763 * contexts while filtering since we can't stop userspace
764 * issuing MI_REPORT_PERF_COUNT commands which would still
765 * provide a side-band view of the real values.
767 * To allow userspace (such as Mesa/GL_INTEL_performance_query)
768 * to normalize counters for a single filtered context then it
769 * needs be forwarded bookend context-switch reports so that it
770 * can track switches in between MI_REPORT_PERF_COUNT commands
771 * and can itself subtract/ignore the progress of counters
772 * associated with other contexts. Note that the hardware
773 * automatically triggers reports when switching to a new
774 * context which are tagged with the ID of the newly active
775 * context. To avoid the complexity (and likely fragility) of
776 * reading ahead while parsing reports to try and minimize
777 * forwarding redundant context switch reports (i.e. between
778 * other, unrelated contexts) we simply elect to forward them
781 * We don't rely solely on the reason field to identify context
782 * switches since it's not-uncommon for periodic samples to
783 * identify a switch before any 'context switch' report.
785 if (!dev_priv
->perf
.oa
.exclusive_stream
->ctx
||
786 dev_priv
->perf
.oa
.specific_ctx_id
== ctx_id
||
787 (dev_priv
->perf
.oa
.oa_buffer
.last_ctx_id
==
788 dev_priv
->perf
.oa
.specific_ctx_id
) ||
789 reason
& OAREPORT_REASON_CTX_SWITCH
) {
792 * While filtering for a single context we avoid
793 * leaking the IDs of other contexts.
795 if (dev_priv
->perf
.oa
.exclusive_stream
->ctx
&&
796 dev_priv
->perf
.oa
.specific_ctx_id
!= ctx_id
) {
797 report32
[2] = INVALID_CTX_ID
;
800 ret
= append_oa_sample(stream
, buf
, count
, offset
,
805 dev_priv
->perf
.oa
.oa_buffer
.last_ctx_id
= ctx_id
;
809 * The above reason field sanity check is based on
810 * the assumption that the OA buffer is initially
811 * zeroed and we reset the field after copying so the
812 * check is still meaningful once old reports start
818 if (start_offset
!= *offset
) {
819 spin_lock_irqsave(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
, flags
);
822 * We removed the gtt_offset for the copy loop above, indexing
823 * relative to oa_buf_base so put back here...
827 I915_WRITE(GEN8_OAHEADPTR
, head
& GEN8_OAHEADPTR_MASK
);
828 dev_priv
->perf
.oa
.oa_buffer
.head
= head
;
830 spin_unlock_irqrestore(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
, flags
);
837 * gen8_oa_read - copy status records then buffered OA reports
838 * @stream: An i915-perf stream opened for OA metrics
839 * @buf: destination buffer given by userspace
840 * @count: the number of bytes userspace wants to read
841 * @offset: (inout): the current position for writing into @buf
843 * Checks OA unit status registers and if necessary appends corresponding
844 * status records for userspace (such as for a buffer full condition) and then
845 * initiate appending any buffered OA reports.
847 * Updates @offset according to the number of bytes successfully copied into
848 * the userspace buffer.
850 * NB: some data may be successfully copied to the userspace buffer
851 * even if an error is returned, and this is reflected in the
854 * Returns: zero on success or a negative error code
856 static int gen8_oa_read(struct i915_perf_stream
*stream
,
861 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
865 if (WARN_ON(!dev_priv
->perf
.oa
.oa_buffer
.vaddr
))
868 oastatus
= I915_READ(GEN8_OASTATUS
);
871 * We treat OABUFFER_OVERFLOW as a significant error:
873 * Although theoretically we could handle this more gracefully
874 * sometimes, some Gens don't correctly suppress certain
875 * automatically triggered reports in this condition and so we
876 * have to assume that old reports are now being trampled
879 * Considering how we don't currently give userspace control
880 * over the OA buffer size and always configure a large 16MB
881 * buffer, then a buffer overflow does anyway likely indicate
882 * that something has gone quite badly wrong.
884 if (oastatus
& GEN8_OASTATUS_OABUFFER_OVERFLOW
) {
885 ret
= append_oa_status(stream
, buf
, count
, offset
,
886 DRM_I915_PERF_RECORD_OA_BUFFER_LOST
);
890 DRM_DEBUG("OA buffer overflow (exponent = %d): force restart\n",
891 dev_priv
->perf
.oa
.period_exponent
);
893 dev_priv
->perf
.oa
.ops
.oa_disable(dev_priv
);
894 dev_priv
->perf
.oa
.ops
.oa_enable(dev_priv
);
897 * Note: .oa_enable() is expected to re-init the oabuffer and
898 * reset GEN8_OASTATUS for us
900 oastatus
= I915_READ(GEN8_OASTATUS
);
903 if (oastatus
& GEN8_OASTATUS_REPORT_LOST
) {
904 ret
= append_oa_status(stream
, buf
, count
, offset
,
905 DRM_I915_PERF_RECORD_OA_REPORT_LOST
);
908 I915_WRITE(GEN8_OASTATUS
,
909 oastatus
& ~GEN8_OASTATUS_REPORT_LOST
);
912 return gen8_append_oa_reports(stream
, buf
, count
, offset
);
916 * Copies all buffered OA reports into userspace read() buffer.
917 * @stream: An i915-perf stream opened for OA metrics
918 * @buf: destination buffer given by userspace
919 * @count: the number of bytes userspace wants to read
920 * @offset: (inout): the current position for writing into @buf
922 * Notably any error condition resulting in a short read (-%ENOSPC or
923 * -%EFAULT) will be returned even though one or more records may
924 * have been successfully copied. In this case it's up to the caller
925 * to decide if the error should be squashed before returning to
928 * Note: reports are consumed from the head, and appended to the
929 * tail, so the tail chases the head?... If you think that's mad
930 * and back-to-front you're not alone, but this follows the
931 * Gen PRM naming convention.
933 * Returns: 0 on success, negative error code on failure.
935 static int gen7_append_oa_reports(struct i915_perf_stream
*stream
,
940 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
941 int report_size
= dev_priv
->perf
.oa
.oa_buffer
.format_size
;
942 u8
*oa_buf_base
= dev_priv
->perf
.oa
.oa_buffer
.vaddr
;
943 u32 gtt_offset
= i915_ggtt_offset(dev_priv
->perf
.oa
.oa_buffer
.vma
);
944 u32 mask
= (OA_BUFFER_SIZE
- 1);
945 size_t start_offset
= *offset
;
947 unsigned int aged_tail_idx
;
952 if (WARN_ON(!stream
->enabled
))
955 spin_lock_irqsave(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
, flags
);
957 head
= dev_priv
->perf
.oa
.oa_buffer
.head
;
958 aged_tail_idx
= dev_priv
->perf
.oa
.oa_buffer
.aged_tail_idx
;
959 tail
= dev_priv
->perf
.oa
.oa_buffer
.tails
[aged_tail_idx
].offset
;
961 spin_unlock_irqrestore(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
, flags
);
963 /* An invalid tail pointer here means we're still waiting for the poll
964 * hrtimer callback to give us a pointer
966 if (tail
== INVALID_TAIL_PTR
)
969 /* NB: oa_buffer.head/tail include the gtt_offset which we don't want
970 * while indexing relative to oa_buf_base.
975 /* An out of bounds or misaligned head or tail pointer implies a driver
976 * bug since we validate + align the tail pointers we read from the
977 * hardware and we are in full control of the head pointer which should
978 * only be incremented by multiples of the report size (notably also
979 * all a power of two).
981 if (WARN_ONCE(head
> OA_BUFFER_SIZE
|| head
% report_size
||
982 tail
> OA_BUFFER_SIZE
|| tail
% report_size
,
983 "Inconsistent OA buffer pointers: head = %u, tail = %u\n",
989 (taken
= OA_TAKEN(tail
, head
));
990 head
= (head
+ report_size
) & mask
) {
991 u8
*report
= oa_buf_base
+ head
;
992 u32
*report32
= (void *)report
;
994 /* All the report sizes factor neatly into the buffer
995 * size so we never expect to see a report split
996 * between the beginning and end of the buffer.
998 * Given the initial alignment check a misalignment
999 * here would imply a driver bug that would result
1002 if (WARN_ON((OA_BUFFER_SIZE
- head
) < report_size
)) {
1003 DRM_ERROR("Spurious OA head ptr: non-integral report offset\n");
1007 /* The report-ID field for periodic samples includes
1008 * some undocumented flags related to what triggered
1009 * the report and is never expected to be zero so we
1010 * can check that the report isn't invalid before
1011 * copying it to userspace...
1013 if (report32
[0] == 0) {
1014 if (__ratelimit(&dev_priv
->perf
.oa
.spurious_report_rs
))
1015 DRM_NOTE("Skipping spurious, invalid OA report\n");
1019 ret
= append_oa_sample(stream
, buf
, count
, offset
, report
);
1023 /* The above report-id field sanity check is based on
1024 * the assumption that the OA buffer is initially
1025 * zeroed and we reset the field after copying so the
1026 * check is still meaningful once old reports start
1027 * being overwritten.
1032 if (start_offset
!= *offset
) {
1033 spin_lock_irqsave(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
, flags
);
1035 /* We removed the gtt_offset for the copy loop above, indexing
1036 * relative to oa_buf_base so put back here...
1040 I915_WRITE(GEN7_OASTATUS2
,
1041 ((head
& GEN7_OASTATUS2_HEAD_MASK
) |
1042 OA_MEM_SELECT_GGTT
));
1043 dev_priv
->perf
.oa
.oa_buffer
.head
= head
;
1045 spin_unlock_irqrestore(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
, flags
);
1052 * gen7_oa_read - copy status records then buffered OA reports
1053 * @stream: An i915-perf stream opened for OA metrics
1054 * @buf: destination buffer given by userspace
1055 * @count: the number of bytes userspace wants to read
1056 * @offset: (inout): the current position for writing into @buf
1058 * Checks Gen 7 specific OA unit status registers and if necessary appends
1059 * corresponding status records for userspace (such as for a buffer full
1060 * condition) and then initiate appending any buffered OA reports.
1062 * Updates @offset according to the number of bytes successfully copied into
1063 * the userspace buffer.
1065 * Returns: zero on success or a negative error code
1067 static int gen7_oa_read(struct i915_perf_stream
*stream
,
1072 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
1076 if (WARN_ON(!dev_priv
->perf
.oa
.oa_buffer
.vaddr
))
1079 oastatus1
= I915_READ(GEN7_OASTATUS1
);
1081 /* XXX: On Haswell we don't have a safe way to clear oastatus1
1082 * bits while the OA unit is enabled (while the tail pointer
1083 * may be updated asynchronously) so we ignore status bits
1084 * that have already been reported to userspace.
1086 oastatus1
&= ~dev_priv
->perf
.oa
.gen7_latched_oastatus1
;
1088 /* We treat OABUFFER_OVERFLOW as a significant error:
1090 * - The status can be interpreted to mean that the buffer is
1091 * currently full (with a higher precedence than OA_TAKEN()
1092 * which will start to report a near-empty buffer after an
1093 * overflow) but it's awkward that we can't clear the status
1094 * on Haswell, so without a reset we won't be able to catch
1097 * - Since it also implies the HW has started overwriting old
1098 * reports it may also affect our sanity checks for invalid
1099 * reports when copying to userspace that assume new reports
1100 * are being written to cleared memory.
1102 * - In the future we may want to introduce a flight recorder
1103 * mode where the driver will automatically maintain a safe
1104 * guard band between head/tail, avoiding this overflow
1105 * condition, but we avoid the added driver complexity for
1108 if (unlikely(oastatus1
& GEN7_OASTATUS1_OABUFFER_OVERFLOW
)) {
1109 ret
= append_oa_status(stream
, buf
, count
, offset
,
1110 DRM_I915_PERF_RECORD_OA_BUFFER_LOST
);
1114 DRM_DEBUG("OA buffer overflow (exponent = %d): force restart\n",
1115 dev_priv
->perf
.oa
.period_exponent
);
1117 dev_priv
->perf
.oa
.ops
.oa_disable(dev_priv
);
1118 dev_priv
->perf
.oa
.ops
.oa_enable(dev_priv
);
1120 oastatus1
= I915_READ(GEN7_OASTATUS1
);
1123 if (unlikely(oastatus1
& GEN7_OASTATUS1_REPORT_LOST
)) {
1124 ret
= append_oa_status(stream
, buf
, count
, offset
,
1125 DRM_I915_PERF_RECORD_OA_REPORT_LOST
);
1128 dev_priv
->perf
.oa
.gen7_latched_oastatus1
|=
1129 GEN7_OASTATUS1_REPORT_LOST
;
1132 return gen7_append_oa_reports(stream
, buf
, count
, offset
);
1136 * i915_oa_wait_unlocked - handles blocking IO until OA data available
1137 * @stream: An i915-perf stream opened for OA metrics
1139 * Called when userspace tries to read() from a blocking stream FD opened
1140 * for OA metrics. It waits until the hrtimer callback finds a non-empty
1141 * OA buffer and wakes us.
1143 * Note: it's acceptable to have this return with some false positives
1144 * since any subsequent read handling will return -EAGAIN if there isn't
1145 * really data ready for userspace yet.
1147 * Returns: zero on success or a negative error code
1149 static int i915_oa_wait_unlocked(struct i915_perf_stream
*stream
)
1151 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
1153 /* We would wait indefinitely if periodic sampling is not enabled */
1154 if (!dev_priv
->perf
.oa
.periodic
)
1157 return wait_event_interruptible(dev_priv
->perf
.oa
.poll_wq
,
1158 oa_buffer_check_unlocked(dev_priv
));
1162 * i915_oa_poll_wait - call poll_wait() for an OA stream poll()
1163 * @stream: An i915-perf stream opened for OA metrics
1164 * @file: An i915 perf stream file
1165 * @wait: poll() state table
1167 * For handling userspace polling on an i915 perf stream opened for OA metrics,
1168 * this starts a poll_wait with the wait queue that our hrtimer callback wakes
1169 * when it sees data ready to read in the circular OA buffer.
1171 static void i915_oa_poll_wait(struct i915_perf_stream
*stream
,
1175 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
1177 poll_wait(file
, &dev_priv
->perf
.oa
.poll_wq
, wait
);
1181 * i915_oa_read - just calls through to &i915_oa_ops->read
1182 * @stream: An i915-perf stream opened for OA metrics
1183 * @buf: destination buffer given by userspace
1184 * @count: the number of bytes userspace wants to read
1185 * @offset: (inout): the current position for writing into @buf
1187 * Updates @offset according to the number of bytes successfully copied into
1188 * the userspace buffer.
1190 * Returns: zero on success or a negative error code
1192 static int i915_oa_read(struct i915_perf_stream
*stream
,
1197 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
1199 return dev_priv
->perf
.oa
.ops
.read(stream
, buf
, count
, offset
);
1203 * oa_get_render_ctx_id - determine and hold ctx hw id
1204 * @stream: An i915-perf stream opened for OA metrics
1206 * Determine the render context hw id, and ensure it remains fixed for the
1207 * lifetime of the stream. This ensures that we don't have to worry about
1208 * updating the context ID in OACONTROL on the fly.
1210 * Returns: zero on success or a negative error code
1212 static int oa_get_render_ctx_id(struct i915_perf_stream
*stream
)
1214 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
1216 if (i915
.enable_execlists
)
1217 dev_priv
->perf
.oa
.specific_ctx_id
= stream
->ctx
->hw_id
;
1219 struct intel_engine_cs
*engine
= dev_priv
->engine
[RCS
];
1220 struct intel_ring
*ring
;
1223 ret
= i915_mutex_lock_interruptible(&dev_priv
->drm
);
1228 * As the ID is the gtt offset of the context's vma we
1229 * pin the vma to ensure the ID remains fixed.
1231 * NB: implied RCS engine...
1233 ring
= engine
->context_pin(engine
, stream
->ctx
);
1234 mutex_unlock(&dev_priv
->drm
.struct_mutex
);
1236 return PTR_ERR(ring
);
1240 * Explicitly track the ID (instead of calling
1241 * i915_ggtt_offset() on the fly) considering the difference
1242 * with gen8+ and execlists
1244 dev_priv
->perf
.oa
.specific_ctx_id
=
1245 i915_ggtt_offset(stream
->ctx
->engine
[engine
->id
].state
);
1252 * oa_put_render_ctx_id - counterpart to oa_get_render_ctx_id releases hold
1253 * @stream: An i915-perf stream opened for OA metrics
1255 * In case anything needed doing to ensure the context HW ID would remain valid
1256 * for the lifetime of the stream, then that can be undone here.
1258 static void oa_put_render_ctx_id(struct i915_perf_stream
*stream
)
1260 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
1262 if (i915
.enable_execlists
) {
1263 dev_priv
->perf
.oa
.specific_ctx_id
= INVALID_CTX_ID
;
1265 struct intel_engine_cs
*engine
= dev_priv
->engine
[RCS
];
1267 mutex_lock(&dev_priv
->drm
.struct_mutex
);
1269 dev_priv
->perf
.oa
.specific_ctx_id
= INVALID_CTX_ID
;
1270 engine
->context_unpin(engine
, stream
->ctx
);
1272 mutex_unlock(&dev_priv
->drm
.struct_mutex
);
1277 free_oa_buffer(struct drm_i915_private
*i915
)
1279 mutex_lock(&i915
->drm
.struct_mutex
);
1281 i915_gem_object_unpin_map(i915
->perf
.oa
.oa_buffer
.vma
->obj
);
1282 i915_vma_unpin(i915
->perf
.oa
.oa_buffer
.vma
);
1283 i915_gem_object_put(i915
->perf
.oa
.oa_buffer
.vma
->obj
);
1285 i915
->perf
.oa
.oa_buffer
.vma
= NULL
;
1286 i915
->perf
.oa
.oa_buffer
.vaddr
= NULL
;
1288 mutex_unlock(&i915
->drm
.struct_mutex
);
1291 static void i915_oa_stream_destroy(struct i915_perf_stream
*stream
)
1293 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
1295 BUG_ON(stream
!= dev_priv
->perf
.oa
.exclusive_stream
);
1298 * Unset exclusive_stream first, it will be checked while disabling
1299 * the metric set on gen8+.
1301 mutex_lock(&dev_priv
->drm
.struct_mutex
);
1302 dev_priv
->perf
.oa
.exclusive_stream
= NULL
;
1303 mutex_unlock(&dev_priv
->drm
.struct_mutex
);
1305 dev_priv
->perf
.oa
.ops
.disable_metric_set(dev_priv
);
1307 free_oa_buffer(dev_priv
);
1309 intel_uncore_forcewake_put(dev_priv
, FORCEWAKE_ALL
);
1310 intel_runtime_pm_put(dev_priv
);
1313 oa_put_render_ctx_id(stream
);
1315 put_oa_config(dev_priv
, stream
->oa_config
);
1317 if (dev_priv
->perf
.oa
.spurious_report_rs
.missed
) {
1318 DRM_NOTE("%d spurious OA report notices suppressed due to ratelimiting\n",
1319 dev_priv
->perf
.oa
.spurious_report_rs
.missed
);
1323 static void gen7_init_oa_buffer(struct drm_i915_private
*dev_priv
)
1325 u32 gtt_offset
= i915_ggtt_offset(dev_priv
->perf
.oa
.oa_buffer
.vma
);
1326 unsigned long flags
;
1328 spin_lock_irqsave(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
, flags
);
1330 /* Pre-DevBDW: OABUFFER must be set with counters off,
1331 * before OASTATUS1, but after OASTATUS2
1333 I915_WRITE(GEN7_OASTATUS2
, gtt_offset
| OA_MEM_SELECT_GGTT
); /* head */
1334 dev_priv
->perf
.oa
.oa_buffer
.head
= gtt_offset
;
1336 I915_WRITE(GEN7_OABUFFER
, gtt_offset
);
1338 I915_WRITE(GEN7_OASTATUS1
, gtt_offset
| OABUFFER_SIZE_16M
); /* tail */
1340 /* Mark that we need updated tail pointers to read from... */
1341 dev_priv
->perf
.oa
.oa_buffer
.tails
[0].offset
= INVALID_TAIL_PTR
;
1342 dev_priv
->perf
.oa
.oa_buffer
.tails
[1].offset
= INVALID_TAIL_PTR
;
1344 spin_unlock_irqrestore(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
, flags
);
1346 /* On Haswell we have to track which OASTATUS1 flags we've
1347 * already seen since they can't be cleared while periodic
1348 * sampling is enabled.
1350 dev_priv
->perf
.oa
.gen7_latched_oastatus1
= 0;
1352 /* NB: although the OA buffer will initially be allocated
1353 * zeroed via shmfs (and so this memset is redundant when
1354 * first allocating), we may re-init the OA buffer, either
1355 * when re-enabling a stream or in error/reset paths.
1357 * The reason we clear the buffer for each re-init is for the
1358 * sanity check in gen7_append_oa_reports() that looks at the
1359 * report-id field to make sure it's non-zero which relies on
1360 * the assumption that new reports are being written to zeroed
1363 memset(dev_priv
->perf
.oa
.oa_buffer
.vaddr
, 0, OA_BUFFER_SIZE
);
1365 /* Maybe make ->pollin per-stream state if we support multiple
1366 * concurrent streams in the future.
1368 dev_priv
->perf
.oa
.pollin
= false;
1371 static void gen8_init_oa_buffer(struct drm_i915_private
*dev_priv
)
1373 u32 gtt_offset
= i915_ggtt_offset(dev_priv
->perf
.oa
.oa_buffer
.vma
);
1374 unsigned long flags
;
1376 spin_lock_irqsave(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
, flags
);
1378 I915_WRITE(GEN8_OASTATUS
, 0);
1379 I915_WRITE(GEN8_OAHEADPTR
, gtt_offset
);
1380 dev_priv
->perf
.oa
.oa_buffer
.head
= gtt_offset
;
1382 I915_WRITE(GEN8_OABUFFER_UDW
, 0);
1387 * "This MMIO must be set before the OATAILPTR
1388 * register and after the OAHEADPTR register. This is
1389 * to enable proper functionality of the overflow
1392 I915_WRITE(GEN8_OABUFFER
, gtt_offset
|
1393 OABUFFER_SIZE_16M
| OA_MEM_SELECT_GGTT
);
1394 I915_WRITE(GEN8_OATAILPTR
, gtt_offset
& GEN8_OATAILPTR_MASK
);
1396 /* Mark that we need updated tail pointers to read from... */
1397 dev_priv
->perf
.oa
.oa_buffer
.tails
[0].offset
= INVALID_TAIL_PTR
;
1398 dev_priv
->perf
.oa
.oa_buffer
.tails
[1].offset
= INVALID_TAIL_PTR
;
1401 * Reset state used to recognise context switches, affecting which
1402 * reports we will forward to userspace while filtering for a single
1405 dev_priv
->perf
.oa
.oa_buffer
.last_ctx_id
= INVALID_CTX_ID
;
1407 spin_unlock_irqrestore(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
, flags
);
1410 * NB: although the OA buffer will initially be allocated
1411 * zeroed via shmfs (and so this memset is redundant when
1412 * first allocating), we may re-init the OA buffer, either
1413 * when re-enabling a stream or in error/reset paths.
1415 * The reason we clear the buffer for each re-init is for the
1416 * sanity check in gen8_append_oa_reports() that looks at the
1417 * reason field to make sure it's non-zero which relies on
1418 * the assumption that new reports are being written to zeroed
1421 memset(dev_priv
->perf
.oa
.oa_buffer
.vaddr
, 0, OA_BUFFER_SIZE
);
1424 * Maybe make ->pollin per-stream state if we support multiple
1425 * concurrent streams in the future.
1427 dev_priv
->perf
.oa
.pollin
= false;
1430 static int alloc_oa_buffer(struct drm_i915_private
*dev_priv
)
1432 struct drm_i915_gem_object
*bo
;
1433 struct i915_vma
*vma
;
1436 if (WARN_ON(dev_priv
->perf
.oa
.oa_buffer
.vma
))
1439 ret
= i915_mutex_lock_interruptible(&dev_priv
->drm
);
1443 BUILD_BUG_ON_NOT_POWER_OF_2(OA_BUFFER_SIZE
);
1444 BUILD_BUG_ON(OA_BUFFER_SIZE
< SZ_128K
|| OA_BUFFER_SIZE
> SZ_16M
);
1446 bo
= i915_gem_object_create(dev_priv
, OA_BUFFER_SIZE
);
1448 DRM_ERROR("Failed to allocate OA buffer\n");
1453 ret
= i915_gem_object_set_cache_level(bo
, I915_CACHE_LLC
);
1457 /* PreHSW required 512K alignment, HSW requires 16M */
1458 vma
= i915_gem_object_ggtt_pin(bo
, NULL
, 0, SZ_16M
, 0);
1463 dev_priv
->perf
.oa
.oa_buffer
.vma
= vma
;
1465 dev_priv
->perf
.oa
.oa_buffer
.vaddr
=
1466 i915_gem_object_pin_map(bo
, I915_MAP_WB
);
1467 if (IS_ERR(dev_priv
->perf
.oa
.oa_buffer
.vaddr
)) {
1468 ret
= PTR_ERR(dev_priv
->perf
.oa
.oa_buffer
.vaddr
);
1472 dev_priv
->perf
.oa
.ops
.init_oa_buffer(dev_priv
);
1474 DRM_DEBUG_DRIVER("OA Buffer initialized, gtt offset = 0x%x, vaddr = %p\n",
1475 i915_ggtt_offset(dev_priv
->perf
.oa
.oa_buffer
.vma
),
1476 dev_priv
->perf
.oa
.oa_buffer
.vaddr
);
1481 __i915_vma_unpin(vma
);
1484 i915_gem_object_put(bo
);
1486 dev_priv
->perf
.oa
.oa_buffer
.vaddr
= NULL
;
1487 dev_priv
->perf
.oa
.oa_buffer
.vma
= NULL
;
1490 mutex_unlock(&dev_priv
->drm
.struct_mutex
);
1494 static void config_oa_regs(struct drm_i915_private
*dev_priv
,
1495 const struct i915_oa_reg
*regs
,
1500 for (i
= 0; i
< n_regs
; i
++) {
1501 const struct i915_oa_reg
*reg
= regs
+ i
;
1503 I915_WRITE(reg
->addr
, reg
->value
);
1507 static int hsw_enable_metric_set(struct drm_i915_private
*dev_priv
,
1508 const struct i915_oa_config
*oa_config
)
1512 * OA unit is using “crclk” for its functionality. When trunk
1513 * level clock gating takes place, OA clock would be gated,
1514 * unable to count the events from non-render clock domain.
1515 * Render clock gating must be disabled when OA is enabled to
1516 * count the events from non-render domain. Unit level clock
1517 * gating for RCS should also be disabled.
1519 I915_WRITE(GEN7_MISCCPCTL
, (I915_READ(GEN7_MISCCPCTL
) &
1520 ~GEN7_DOP_CLOCK_GATE_ENABLE
));
1521 I915_WRITE(GEN6_UCGCTL1
, (I915_READ(GEN6_UCGCTL1
) |
1522 GEN6_CSUNIT_CLOCK_GATE_DISABLE
));
1524 config_oa_regs(dev_priv
, oa_config
->mux_regs
, oa_config
->mux_regs_len
);
1526 /* It apparently takes a fairly long time for a new MUX
1527 * configuration to be be applied after these register writes.
1528 * This delay duration was derived empirically based on the
1529 * render_basic config but hopefully it covers the maximum
1530 * configuration latency.
1532 * As a fallback, the checks in _append_oa_reports() to skip
1533 * invalid OA reports do also seem to work to discard reports
1534 * generated before this config has completed - albeit not
1537 * Unfortunately this is essentially a magic number, since we
1538 * don't currently know of a reliable mechanism for predicting
1539 * how long the MUX config will take to apply and besides
1540 * seeing invalid reports we don't know of a reliable way to
1541 * explicitly check that the MUX config has landed.
1543 * It's even possible we've miss characterized the underlying
1544 * problem - it just seems like the simplest explanation why
1545 * a delay at this location would mitigate any invalid reports.
1547 usleep_range(15000, 20000);
1549 config_oa_regs(dev_priv
, oa_config
->b_counter_regs
,
1550 oa_config
->b_counter_regs_len
);
1555 static void hsw_disable_metric_set(struct drm_i915_private
*dev_priv
)
1557 I915_WRITE(GEN6_UCGCTL1
, (I915_READ(GEN6_UCGCTL1
) &
1558 ~GEN6_CSUNIT_CLOCK_GATE_DISABLE
));
1559 I915_WRITE(GEN7_MISCCPCTL
, (I915_READ(GEN7_MISCCPCTL
) |
1560 GEN7_DOP_CLOCK_GATE_ENABLE
));
1562 I915_WRITE(GDT_CHICKEN_BITS
, (I915_READ(GDT_CHICKEN_BITS
) &
1567 * NB: It must always remain pointer safe to run this even if the OA unit
1568 * has been disabled.
1570 * It's fine to put out-of-date values into these per-context registers
1571 * in the case that the OA unit has been disabled.
1573 static void gen8_update_reg_state_unlocked(struct i915_gem_context
*ctx
,
1575 const struct i915_oa_config
*oa_config
)
1577 struct drm_i915_private
*dev_priv
= ctx
->i915
;
1578 u32 ctx_oactxctrl
= dev_priv
->perf
.oa
.ctx_oactxctrl_offset
;
1579 u32 ctx_flexeu0
= dev_priv
->perf
.oa
.ctx_flexeu0_offset
;
1580 /* The MMIO offsets for Flex EU registers aren't contiguous */
1582 i915_mmio_reg_offset(EU_PERF_CNTL0
),
1583 i915_mmio_reg_offset(EU_PERF_CNTL1
),
1584 i915_mmio_reg_offset(EU_PERF_CNTL2
),
1585 i915_mmio_reg_offset(EU_PERF_CNTL3
),
1586 i915_mmio_reg_offset(EU_PERF_CNTL4
),
1587 i915_mmio_reg_offset(EU_PERF_CNTL5
),
1588 i915_mmio_reg_offset(EU_PERF_CNTL6
),
1592 reg_state
[ctx_oactxctrl
] = i915_mmio_reg_offset(GEN8_OACTXCONTROL
);
1593 reg_state
[ctx_oactxctrl
+1] = (dev_priv
->perf
.oa
.period_exponent
<<
1594 GEN8_OA_TIMER_PERIOD_SHIFT
) |
1595 (dev_priv
->perf
.oa
.periodic
?
1596 GEN8_OA_TIMER_ENABLE
: 0) |
1597 GEN8_OA_COUNTER_RESUME
;
1599 for (i
= 0; i
< ARRAY_SIZE(flex_mmio
); i
++) {
1600 u32 state_offset
= ctx_flexeu0
+ i
* 2;
1601 u32 mmio
= flex_mmio
[i
];
1604 * This arbitrary default will select the 'EU FPU0 Pipeline
1605 * Active' event. In the future it's anticipated that there
1606 * will be an explicit 'No Event' we can select, but not yet...
1613 for (j
= 0; j
< oa_config
->flex_regs_len
; j
++) {
1614 if (i915_mmio_reg_offset(oa_config
->flex_regs
[j
].addr
) == mmio
) {
1615 value
= oa_config
->flex_regs
[j
].value
;
1621 reg_state
[state_offset
] = mmio
;
1622 reg_state
[state_offset
+1] = value
;
1627 * Same as gen8_update_reg_state_unlocked only through the batchbuffer. This
1628 * is only used by the kernel context.
1630 static int gen8_emit_oa_config(struct drm_i915_gem_request
*req
,
1631 const struct i915_oa_config
*oa_config
)
1633 struct drm_i915_private
*dev_priv
= req
->i915
;
1634 /* The MMIO offsets for Flex EU registers aren't contiguous */
1636 i915_mmio_reg_offset(EU_PERF_CNTL0
),
1637 i915_mmio_reg_offset(EU_PERF_CNTL1
),
1638 i915_mmio_reg_offset(EU_PERF_CNTL2
),
1639 i915_mmio_reg_offset(EU_PERF_CNTL3
),
1640 i915_mmio_reg_offset(EU_PERF_CNTL4
),
1641 i915_mmio_reg_offset(EU_PERF_CNTL5
),
1642 i915_mmio_reg_offset(EU_PERF_CNTL6
),
1647 cs
= intel_ring_begin(req
, ARRAY_SIZE(flex_mmio
) * 2 + 4);
1651 *cs
++ = MI_LOAD_REGISTER_IMM(ARRAY_SIZE(flex_mmio
) + 1);
1653 *cs
++ = i915_mmio_reg_offset(GEN8_OACTXCONTROL
);
1654 *cs
++ = (dev_priv
->perf
.oa
.period_exponent
<< GEN8_OA_TIMER_PERIOD_SHIFT
) |
1655 (dev_priv
->perf
.oa
.periodic
? GEN8_OA_TIMER_ENABLE
: 0) |
1656 GEN8_OA_COUNTER_RESUME
;
1658 for (i
= 0; i
< ARRAY_SIZE(flex_mmio
); i
++) {
1659 u32 mmio
= flex_mmio
[i
];
1662 * This arbitrary default will select the 'EU FPU0 Pipeline
1663 * Active' event. In the future it's anticipated that there
1664 * will be an explicit 'No Event' we can select, but not
1672 for (j
= 0; j
< oa_config
->flex_regs_len
; j
++) {
1673 if (i915_mmio_reg_offset(oa_config
->flex_regs
[j
].addr
) == mmio
) {
1674 value
= oa_config
->flex_regs
[j
].value
;
1685 intel_ring_advance(req
, cs
);
1690 static int gen8_switch_to_updated_kernel_context(struct drm_i915_private
*dev_priv
,
1691 const struct i915_oa_config
*oa_config
)
1693 struct intel_engine_cs
*engine
= dev_priv
->engine
[RCS
];
1694 struct i915_gem_timeline
*timeline
;
1695 struct drm_i915_gem_request
*req
;
1698 lockdep_assert_held(&dev_priv
->drm
.struct_mutex
);
1700 i915_gem_retire_requests(dev_priv
);
1702 req
= i915_gem_request_alloc(engine
, dev_priv
->kernel_context
);
1704 return PTR_ERR(req
);
1706 ret
= gen8_emit_oa_config(req
, oa_config
);
1708 i915_add_request(req
);
1712 /* Queue this switch after all other activity */
1713 list_for_each_entry(timeline
, &dev_priv
->gt
.timelines
, link
) {
1714 struct drm_i915_gem_request
*prev
;
1715 struct intel_timeline
*tl
;
1717 tl
= &timeline
->engine
[engine
->id
];
1718 prev
= i915_gem_active_raw(&tl
->last_request
,
1719 &dev_priv
->drm
.struct_mutex
);
1721 i915_sw_fence_await_sw_fence_gfp(&req
->submit
,
1726 ret
= i915_switch_context(req
);
1727 i915_add_request(req
);
1733 * Manages updating the per-context aspects of the OA stream
1734 * configuration across all contexts.
1736 * The awkward consideration here is that OACTXCONTROL controls the
1737 * exponent for periodic sampling which is primarily used for system
1738 * wide profiling where we'd like a consistent sampling period even in
1739 * the face of context switches.
1741 * Our approach of updating the register state context (as opposed to
1742 * say using a workaround batch buffer) ensures that the hardware
1743 * won't automatically reload an out-of-date timer exponent even
1744 * transiently before a WA BB could be parsed.
1746 * This function needs to:
1747 * - Ensure the currently running context's per-context OA state is
1749 * - Ensure that all existing contexts will have the correct per-context
1750 * OA state if they are scheduled for use.
1751 * - Ensure any new contexts will be initialized with the correct
1752 * per-context OA state.
1754 * Note: it's only the RCS/Render context that has any OA state.
1756 static int gen8_configure_all_contexts(struct drm_i915_private
*dev_priv
,
1757 const struct i915_oa_config
*oa_config
,
1760 struct i915_gem_context
*ctx
;
1762 unsigned int wait_flags
= I915_WAIT_LOCKED
;
1764 if (interruptible
) {
1765 ret
= i915_mutex_lock_interruptible(&dev_priv
->drm
);
1769 wait_flags
|= I915_WAIT_INTERRUPTIBLE
;
1771 mutex_lock(&dev_priv
->drm
.struct_mutex
);
1774 /* Switch away from any user context. */
1775 ret
= gen8_switch_to_updated_kernel_context(dev_priv
, oa_config
);
1780 * The OA register config is setup through the context image. This image
1781 * might be written to by the GPU on context switch (in particular on
1782 * lite-restore). This means we can't safely update a context's image,
1783 * if this context is scheduled/submitted to run on the GPU.
1785 * We could emit the OA register config through the batch buffer but
1786 * this might leave small interval of time where the OA unit is
1787 * configured at an invalid sampling period.
1789 * So far the best way to work around this issue seems to be draining
1790 * the GPU from any submitted work.
1792 ret
= i915_gem_wait_for_idle(dev_priv
, wait_flags
);
1796 /* Update all contexts now that we've stalled the submission. */
1797 list_for_each_entry(ctx
, &dev_priv
->contexts
.list
, link
) {
1798 struct intel_context
*ce
= &ctx
->engine
[RCS
];
1801 /* OA settings will be set upon first use */
1805 regs
= i915_gem_object_pin_map(ce
->state
->obj
, I915_MAP_WB
);
1807 ret
= PTR_ERR(regs
);
1811 ce
->state
->obj
->mm
.dirty
= true;
1812 regs
+= LRC_STATE_PN
* PAGE_SIZE
/ sizeof(*regs
);
1814 gen8_update_reg_state_unlocked(ctx
, regs
, oa_config
);
1816 i915_gem_object_unpin_map(ce
->state
->obj
);
1820 mutex_unlock(&dev_priv
->drm
.struct_mutex
);
1825 static int gen8_enable_metric_set(struct drm_i915_private
*dev_priv
,
1826 const struct i915_oa_config
*oa_config
)
1831 * We disable slice/unslice clock ratio change reports on SKL since
1832 * they are too noisy. The HW generates a lot of redundant reports
1833 * where the ratio hasn't really changed causing a lot of redundant
1834 * work to processes and increasing the chances we'll hit buffer
1837 * Although we don't currently use the 'disable overrun' OABUFFER
1838 * feature it's worth noting that clock ratio reports have to be
1839 * disabled before considering to use that feature since the HW doesn't
1840 * correctly block these reports.
1842 * Currently none of the high-level metrics we have depend on knowing
1843 * this ratio to normalize.
1845 * Note: This register is not power context saved and restored, but
1846 * that's OK considering that we disable RC6 while the OA unit is
1849 * The _INCLUDE_CLK_RATIO bit allows the slice/unslice frequency to
1850 * be read back from automatically triggered reports, as part of the
1853 if (IS_SKYLAKE(dev_priv
) || IS_BROXTON(dev_priv
) ||
1854 IS_KABYLAKE(dev_priv
) || IS_GEMINILAKE(dev_priv
)) {
1855 I915_WRITE(GEN8_OA_DEBUG
,
1856 _MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS
|
1857 GEN9_OA_DEBUG_INCLUDE_CLK_RATIO
));
1861 * Update all contexts prior writing the mux configurations as we need
1862 * to make sure all slices/subslices are ON before writing to NOA
1865 ret
= gen8_configure_all_contexts(dev_priv
, oa_config
, true);
1869 config_oa_regs(dev_priv
, oa_config
->mux_regs
, oa_config
->mux_regs_len
);
1871 config_oa_regs(dev_priv
, oa_config
->b_counter_regs
,
1872 oa_config
->b_counter_regs_len
);
1877 static void gen8_disable_metric_set(struct drm_i915_private
*dev_priv
)
1879 /* Reset all contexts' slices/subslices configurations. */
1880 gen8_configure_all_contexts(dev_priv
, NULL
, false);
1882 I915_WRITE(GDT_CHICKEN_BITS
, (I915_READ(GDT_CHICKEN_BITS
) &
1887 static void gen7_oa_enable(struct drm_i915_private
*dev_priv
)
1890 * Reset buf pointers so we don't forward reports from before now.
1892 * Think carefully if considering trying to avoid this, since it
1893 * also ensures status flags and the buffer itself are cleared
1894 * in error paths, and we have checks for invalid reports based
1895 * on the assumption that certain fields are written to zeroed
1896 * memory which this helps maintains.
1898 gen7_init_oa_buffer(dev_priv
);
1900 if (dev_priv
->perf
.oa
.exclusive_stream
->enabled
) {
1901 struct i915_gem_context
*ctx
=
1902 dev_priv
->perf
.oa
.exclusive_stream
->ctx
;
1903 u32 ctx_id
= dev_priv
->perf
.oa
.specific_ctx_id
;
1905 bool periodic
= dev_priv
->perf
.oa
.periodic
;
1906 u32 period_exponent
= dev_priv
->perf
.oa
.period_exponent
;
1907 u32 report_format
= dev_priv
->perf
.oa
.oa_buffer
.format
;
1909 I915_WRITE(GEN7_OACONTROL
,
1910 (ctx_id
& GEN7_OACONTROL_CTX_MASK
) |
1912 GEN7_OACONTROL_TIMER_PERIOD_SHIFT
) |
1913 (periodic
? GEN7_OACONTROL_TIMER_ENABLE
: 0) |
1914 (report_format
<< GEN7_OACONTROL_FORMAT_SHIFT
) |
1915 (ctx
? GEN7_OACONTROL_PER_CTX_ENABLE
: 0) |
1916 GEN7_OACONTROL_ENABLE
);
1918 I915_WRITE(GEN7_OACONTROL
, 0);
1921 static void gen8_oa_enable(struct drm_i915_private
*dev_priv
)
1923 u32 report_format
= dev_priv
->perf
.oa
.oa_buffer
.format
;
1926 * Reset buf pointers so we don't forward reports from before now.
1928 * Think carefully if considering trying to avoid this, since it
1929 * also ensures status flags and the buffer itself are cleared
1930 * in error paths, and we have checks for invalid reports based
1931 * on the assumption that certain fields are written to zeroed
1932 * memory which this helps maintains.
1934 gen8_init_oa_buffer(dev_priv
);
1937 * Note: we don't rely on the hardware to perform single context
1938 * filtering and instead filter on the cpu based on the context-id
1941 I915_WRITE(GEN8_OACONTROL
, (report_format
<<
1942 GEN8_OA_REPORT_FORMAT_SHIFT
) |
1943 GEN8_OA_COUNTER_ENABLE
);
1947 * i915_oa_stream_enable - handle `I915_PERF_IOCTL_ENABLE` for OA stream
1948 * @stream: An i915 perf stream opened for OA metrics
1950 * [Re]enables hardware periodic sampling according to the period configured
1951 * when opening the stream. This also starts a hrtimer that will periodically
1952 * check for data in the circular OA buffer for notifying userspace (e.g.
1953 * during a read() or poll()).
1955 static void i915_oa_stream_enable(struct i915_perf_stream
*stream
)
1957 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
1959 dev_priv
->perf
.oa
.ops
.oa_enable(dev_priv
);
1961 if (dev_priv
->perf
.oa
.periodic
)
1962 hrtimer_start(&dev_priv
->perf
.oa
.poll_check_timer
,
1963 ns_to_ktime(POLL_PERIOD
),
1964 HRTIMER_MODE_REL_PINNED
);
1967 static void gen7_oa_disable(struct drm_i915_private
*dev_priv
)
1969 I915_WRITE(GEN7_OACONTROL
, 0);
1972 static void gen8_oa_disable(struct drm_i915_private
*dev_priv
)
1974 I915_WRITE(GEN8_OACONTROL
, 0);
1978 * i915_oa_stream_disable - handle `I915_PERF_IOCTL_DISABLE` for OA stream
1979 * @stream: An i915 perf stream opened for OA metrics
1981 * Stops the OA unit from periodically writing counter reports into the
1982 * circular OA buffer. This also stops the hrtimer that periodically checks for
1983 * data in the circular OA buffer, for notifying userspace.
1985 static void i915_oa_stream_disable(struct i915_perf_stream
*stream
)
1987 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
1989 dev_priv
->perf
.oa
.ops
.oa_disable(dev_priv
);
1991 if (dev_priv
->perf
.oa
.periodic
)
1992 hrtimer_cancel(&dev_priv
->perf
.oa
.poll_check_timer
);
1995 static const struct i915_perf_stream_ops i915_oa_stream_ops
= {
1996 .destroy
= i915_oa_stream_destroy
,
1997 .enable
= i915_oa_stream_enable
,
1998 .disable
= i915_oa_stream_disable
,
1999 .wait_unlocked
= i915_oa_wait_unlocked
,
2000 .poll_wait
= i915_oa_poll_wait
,
2001 .read
= i915_oa_read
,
2005 * i915_oa_stream_init - validate combined props for OA stream and init
2006 * @stream: An i915 perf stream
2007 * @param: The open parameters passed to `DRM_I915_PERF_OPEN`
2008 * @props: The property state that configures stream (individually validated)
2010 * While read_properties_unlocked() validates properties in isolation it
2011 * doesn't ensure that the combination necessarily makes sense.
2013 * At this point it has been determined that userspace wants a stream of
2014 * OA metrics, but still we need to further validate the combined
2015 * properties are OK.
2017 * If the configuration makes sense then we can allocate memory for
2018 * a circular OA buffer and apply the requested metric set configuration.
2020 * Returns: zero on success or a negative error code.
2022 static int i915_oa_stream_init(struct i915_perf_stream
*stream
,
2023 struct drm_i915_perf_open_param
*param
,
2024 struct perf_open_properties
*props
)
2026 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
2030 /* If the sysfs metrics/ directory wasn't registered for some
2031 * reason then don't let userspace try their luck with config
2034 if (!dev_priv
->perf
.metrics_kobj
) {
2035 DRM_DEBUG("OA metrics weren't advertised via sysfs\n");
2039 if (!(props
->sample_flags
& SAMPLE_OA_REPORT
)) {
2040 DRM_DEBUG("Only OA report sampling supported\n");
2044 if (!dev_priv
->perf
.oa
.ops
.init_oa_buffer
) {
2045 DRM_DEBUG("OA unit not supported\n");
2049 /* To avoid the complexity of having to accurately filter
2050 * counter reports and marshal to the appropriate client
2051 * we currently only allow exclusive access
2053 if (dev_priv
->perf
.oa
.exclusive_stream
) {
2054 DRM_DEBUG("OA unit already in use\n");
2058 if (!props
->oa_format
) {
2059 DRM_DEBUG("OA report format not specified\n");
2063 /* We set up some ratelimit state to potentially throttle any _NOTES
2064 * about spurious, invalid OA reports which we don't forward to
2067 * The initialization is associated with opening the stream (not driver
2068 * init) considering we print a _NOTE about any throttling when closing
2069 * the stream instead of waiting until driver _fini which no one would
2072 * Using the same limiting factors as printk_ratelimit()
2074 ratelimit_state_init(&dev_priv
->perf
.oa
.spurious_report_rs
,
2076 /* Since we use a DRM_NOTE for spurious reports it would be
2077 * inconsistent to let __ratelimit() automatically print a warning for
2080 ratelimit_set_flags(&dev_priv
->perf
.oa
.spurious_report_rs
,
2081 RATELIMIT_MSG_ON_RELEASE
);
2083 stream
->sample_size
= sizeof(struct drm_i915_perf_record_header
);
2085 format_size
= dev_priv
->perf
.oa
.oa_formats
[props
->oa_format
].size
;
2087 stream
->sample_flags
|= SAMPLE_OA_REPORT
;
2088 stream
->sample_size
+= format_size
;
2090 dev_priv
->perf
.oa
.oa_buffer
.format_size
= format_size
;
2091 if (WARN_ON(dev_priv
->perf
.oa
.oa_buffer
.format_size
== 0))
2094 dev_priv
->perf
.oa
.oa_buffer
.format
=
2095 dev_priv
->perf
.oa
.oa_formats
[props
->oa_format
].format
;
2097 dev_priv
->perf
.oa
.periodic
= props
->oa_periodic
;
2098 if (dev_priv
->perf
.oa
.periodic
)
2099 dev_priv
->perf
.oa
.period_exponent
= props
->oa_period_exponent
;
2102 ret
= oa_get_render_ctx_id(stream
);
2107 ret
= get_oa_config(dev_priv
, props
->metrics_set
, &stream
->oa_config
);
2111 /* PRM - observability performance counters:
2113 * OACONTROL, performance counter enable, note:
2115 * "When this bit is set, in order to have coherent counts,
2116 * RC6 power state and trunk clock gating must be disabled.
2117 * This can be achieved by programming MMIO registers as
2118 * 0xA094=0 and 0xA090[31]=1"
2120 * In our case we are expecting that taking pm + FORCEWAKE
2121 * references will effectively disable RC6.
2123 intel_runtime_pm_get(dev_priv
);
2124 intel_uncore_forcewake_get(dev_priv
, FORCEWAKE_ALL
);
2126 ret
= alloc_oa_buffer(dev_priv
);
2128 goto err_oa_buf_alloc
;
2130 ret
= dev_priv
->perf
.oa
.ops
.enable_metric_set(dev_priv
,
2135 stream
->ops
= &i915_oa_stream_ops
;
2137 /* Lock device for exclusive_stream access late because
2138 * enable_metric_set() might lock as well on gen8+.
2140 ret
= i915_mutex_lock_interruptible(&dev_priv
->drm
);
2144 dev_priv
->perf
.oa
.exclusive_stream
= stream
;
2146 mutex_unlock(&dev_priv
->drm
.struct_mutex
);
2151 dev_priv
->perf
.oa
.ops
.disable_metric_set(dev_priv
);
2154 free_oa_buffer(dev_priv
);
2157 put_oa_config(dev_priv
, stream
->oa_config
);
2159 intel_uncore_forcewake_put(dev_priv
, FORCEWAKE_ALL
);
2160 intel_runtime_pm_put(dev_priv
);
2164 oa_put_render_ctx_id(stream
);
2169 void i915_oa_init_reg_state(struct intel_engine_cs
*engine
,
2170 struct i915_gem_context
*ctx
,
2173 struct drm_i915_private
*dev_priv
= engine
->i915
;
2174 struct i915_perf_stream
*stream
= dev_priv
->perf
.oa
.exclusive_stream
;
2176 lockdep_assert_held(&dev_priv
->drm
.struct_mutex
);
2178 if (engine
->id
!= RCS
)
2181 if (!dev_priv
->perf
.initialized
)
2185 gen8_update_reg_state_unlocked(ctx
, reg_state
, stream
->oa_config
);
2189 * i915_perf_read_locked - &i915_perf_stream_ops->read with error normalisation
2190 * @stream: An i915 perf stream
2191 * @file: An i915 perf stream file
2192 * @buf: destination buffer given by userspace
2193 * @count: the number of bytes userspace wants to read
2194 * @ppos: (inout) file seek position (unused)
2196 * Besides wrapping &i915_perf_stream_ops->read this provides a common place to
2197 * ensure that if we've successfully copied any data then reporting that takes
2198 * precedence over any internal error status, so the data isn't lost.
2200 * For example ret will be -ENOSPC whenever there is more buffered data than
2201 * can be copied to userspace, but that's only interesting if we weren't able
2202 * to copy some data because it implies the userspace buffer is too small to
2203 * receive a single record (and we never split records).
2205 * Another case with ret == -EFAULT is more of a grey area since it would seem
2206 * like bad form for userspace to ask us to overrun its buffer, but the user
2209 * http://yarchive.net/comp/linux/partial_reads_writes.html
2211 * Returns: The number of bytes copied or a negative error code on failure.
2213 static ssize_t
i915_perf_read_locked(struct i915_perf_stream
*stream
,
2219 /* Note we keep the offset (aka bytes read) separate from any
2220 * error status so that the final check for whether we return
2221 * the bytes read with a higher precedence than any error (see
2222 * comment below) doesn't need to be handled/duplicated in
2223 * stream->ops->read() implementations.
2226 int ret
= stream
->ops
->read(stream
, buf
, count
, &offset
);
2228 return offset
?: (ret
?: -EAGAIN
);
2232 * i915_perf_read - handles read() FOP for i915 perf stream FDs
2233 * @file: An i915 perf stream file
2234 * @buf: destination buffer given by userspace
2235 * @count: the number of bytes userspace wants to read
2236 * @ppos: (inout) file seek position (unused)
2238 * The entry point for handling a read() on a stream file descriptor from
2239 * userspace. Most of the work is left to the i915_perf_read_locked() and
2240 * &i915_perf_stream_ops->read but to save having stream implementations (of
2241 * which we might have multiple later) we handle blocking read here.
2243 * We can also consistently treat trying to read from a disabled stream
2244 * as an IO error so implementations can assume the stream is enabled
2247 * Returns: The number of bytes copied or a negative error code on failure.
2249 static ssize_t
i915_perf_read(struct file
*file
,
2254 struct i915_perf_stream
*stream
= file
->private_data
;
2255 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
2258 /* To ensure it's handled consistently we simply treat all reads of a
2259 * disabled stream as an error. In particular it might otherwise lead
2260 * to a deadlock for blocking file descriptors...
2262 if (!stream
->enabled
)
2265 if (!(file
->f_flags
& O_NONBLOCK
)) {
2266 /* There's the small chance of false positives from
2267 * stream->ops->wait_unlocked.
2269 * E.g. with single context filtering since we only wait until
2270 * oabuffer has >= 1 report we don't immediately know whether
2271 * any reports really belong to the current context
2274 ret
= stream
->ops
->wait_unlocked(stream
);
2278 mutex_lock(&dev_priv
->perf
.lock
);
2279 ret
= i915_perf_read_locked(stream
, file
,
2281 mutex_unlock(&dev_priv
->perf
.lock
);
2282 } while (ret
== -EAGAIN
);
2284 mutex_lock(&dev_priv
->perf
.lock
);
2285 ret
= i915_perf_read_locked(stream
, file
, buf
, count
, ppos
);
2286 mutex_unlock(&dev_priv
->perf
.lock
);
2289 /* We allow the poll checking to sometimes report false positive POLLIN
2290 * events where we might actually report EAGAIN on read() if there's
2291 * not really any data available. In this situation though we don't
2292 * want to enter a busy loop between poll() reporting a POLLIN event
2293 * and read() returning -EAGAIN. Clearing the oa.pollin state here
2294 * effectively ensures we back off until the next hrtimer callback
2295 * before reporting another POLLIN event.
2297 if (ret
>= 0 || ret
== -EAGAIN
) {
2298 /* Maybe make ->pollin per-stream state if we support multiple
2299 * concurrent streams in the future.
2301 dev_priv
->perf
.oa
.pollin
= false;
2307 static enum hrtimer_restart
oa_poll_check_timer_cb(struct hrtimer
*hrtimer
)
2309 struct drm_i915_private
*dev_priv
=
2310 container_of(hrtimer
, typeof(*dev_priv
),
2311 perf
.oa
.poll_check_timer
);
2313 if (oa_buffer_check_unlocked(dev_priv
)) {
2314 dev_priv
->perf
.oa
.pollin
= true;
2315 wake_up(&dev_priv
->perf
.oa
.poll_wq
);
2318 hrtimer_forward_now(hrtimer
, ns_to_ktime(POLL_PERIOD
));
2320 return HRTIMER_RESTART
;
2324 * i915_perf_poll_locked - poll_wait() with a suitable wait queue for stream
2325 * @dev_priv: i915 device instance
2326 * @stream: An i915 perf stream
2327 * @file: An i915 perf stream file
2328 * @wait: poll() state table
2330 * For handling userspace polling on an i915 perf stream, this calls through to
2331 * &i915_perf_stream_ops->poll_wait to call poll_wait() with a wait queue that
2332 * will be woken for new stream data.
2334 * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize
2335 * with any non-file-operation driver hooks.
2337 * Returns: any poll events that are ready without sleeping
2339 static unsigned int i915_perf_poll_locked(struct drm_i915_private
*dev_priv
,
2340 struct i915_perf_stream
*stream
,
2344 unsigned int events
= 0;
2346 stream
->ops
->poll_wait(stream
, file
, wait
);
2348 /* Note: we don't explicitly check whether there's something to read
2349 * here since this path may be very hot depending on what else
2350 * userspace is polling, or on the timeout in use. We rely solely on
2351 * the hrtimer/oa_poll_check_timer_cb to notify us when there are
2354 if (dev_priv
->perf
.oa
.pollin
)
2361 * i915_perf_poll - call poll_wait() with a suitable wait queue for stream
2362 * @file: An i915 perf stream file
2363 * @wait: poll() state table
2365 * For handling userspace polling on an i915 perf stream, this ensures
2366 * poll_wait() gets called with a wait queue that will be woken for new stream
2369 * Note: Implementation deferred to i915_perf_poll_locked()
2371 * Returns: any poll events that are ready without sleeping
2373 static unsigned int i915_perf_poll(struct file
*file
, poll_table
*wait
)
2375 struct i915_perf_stream
*stream
= file
->private_data
;
2376 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
2379 mutex_lock(&dev_priv
->perf
.lock
);
2380 ret
= i915_perf_poll_locked(dev_priv
, stream
, file
, wait
);
2381 mutex_unlock(&dev_priv
->perf
.lock
);
2387 * i915_perf_enable_locked - handle `I915_PERF_IOCTL_ENABLE` ioctl
2388 * @stream: A disabled i915 perf stream
2390 * [Re]enables the associated capture of data for this stream.
2392 * If a stream was previously enabled then there's currently no intention
2393 * to provide userspace any guarantee about the preservation of previously
2396 static void i915_perf_enable_locked(struct i915_perf_stream
*stream
)
2398 if (stream
->enabled
)
2401 /* Allow stream->ops->enable() to refer to this */
2402 stream
->enabled
= true;
2404 if (stream
->ops
->enable
)
2405 stream
->ops
->enable(stream
);
2409 * i915_perf_disable_locked - handle `I915_PERF_IOCTL_DISABLE` ioctl
2410 * @stream: An enabled i915 perf stream
2412 * Disables the associated capture of data for this stream.
2414 * The intention is that disabling an re-enabling a stream will ideally be
2415 * cheaper than destroying and re-opening a stream with the same configuration,
2416 * though there are no formal guarantees about what state or buffered data
2417 * must be retained between disabling and re-enabling a stream.
2419 * Note: while a stream is disabled it's considered an error for userspace
2420 * to attempt to read from the stream (-EIO).
2422 static void i915_perf_disable_locked(struct i915_perf_stream
*stream
)
2424 if (!stream
->enabled
)
2427 /* Allow stream->ops->disable() to refer to this */
2428 stream
->enabled
= false;
2430 if (stream
->ops
->disable
)
2431 stream
->ops
->disable(stream
);
2435 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
2436 * @stream: An i915 perf stream
2437 * @cmd: the ioctl request
2438 * @arg: the ioctl data
2440 * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize
2441 * with any non-file-operation driver hooks.
2443 * Returns: zero on success or a negative error code. Returns -EINVAL for
2444 * an unknown ioctl request.
2446 static long i915_perf_ioctl_locked(struct i915_perf_stream
*stream
,
2451 case I915_PERF_IOCTL_ENABLE
:
2452 i915_perf_enable_locked(stream
);
2454 case I915_PERF_IOCTL_DISABLE
:
2455 i915_perf_disable_locked(stream
);
2463 * i915_perf_ioctl - support ioctl() usage with i915 perf stream FDs
2464 * @file: An i915 perf stream file
2465 * @cmd: the ioctl request
2466 * @arg: the ioctl data
2468 * Implementation deferred to i915_perf_ioctl_locked().
2470 * Returns: zero on success or a negative error code. Returns -EINVAL for
2471 * an unknown ioctl request.
2473 static long i915_perf_ioctl(struct file
*file
,
2477 struct i915_perf_stream
*stream
= file
->private_data
;
2478 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
2481 mutex_lock(&dev_priv
->perf
.lock
);
2482 ret
= i915_perf_ioctl_locked(stream
, cmd
, arg
);
2483 mutex_unlock(&dev_priv
->perf
.lock
);
2489 * i915_perf_destroy_locked - destroy an i915 perf stream
2490 * @stream: An i915 perf stream
2492 * Frees all resources associated with the given i915 perf @stream, disabling
2493 * any associated data capture in the process.
2495 * Note: The &drm_i915_private->perf.lock mutex has been taken to serialize
2496 * with any non-file-operation driver hooks.
2498 static void i915_perf_destroy_locked(struct i915_perf_stream
*stream
)
2500 if (stream
->enabled
)
2501 i915_perf_disable_locked(stream
);
2503 if (stream
->ops
->destroy
)
2504 stream
->ops
->destroy(stream
);
2506 list_del(&stream
->link
);
2509 i915_gem_context_put(stream
->ctx
);
2515 * i915_perf_release - handles userspace close() of a stream file
2516 * @inode: anonymous inode associated with file
2517 * @file: An i915 perf stream file
2519 * Cleans up any resources associated with an open i915 perf stream file.
2521 * NB: close() can't really fail from the userspace point of view.
2523 * Returns: zero on success or a negative error code.
2525 static int i915_perf_release(struct inode
*inode
, struct file
*file
)
2527 struct i915_perf_stream
*stream
= file
->private_data
;
2528 struct drm_i915_private
*dev_priv
= stream
->dev_priv
;
2530 mutex_lock(&dev_priv
->perf
.lock
);
2531 i915_perf_destroy_locked(stream
);
2532 mutex_unlock(&dev_priv
->perf
.lock
);
2538 static const struct file_operations fops
= {
2539 .owner
= THIS_MODULE
,
2540 .llseek
= no_llseek
,
2541 .release
= i915_perf_release
,
2542 .poll
= i915_perf_poll
,
2543 .read
= i915_perf_read
,
2544 .unlocked_ioctl
= i915_perf_ioctl
,
2549 * i915_perf_open_ioctl_locked - DRM ioctl() for userspace to open a stream FD
2550 * @dev_priv: i915 device instance
2551 * @param: The open parameters passed to 'DRM_I915_PERF_OPEN`
2552 * @props: individually validated u64 property value pairs
2555 * See i915_perf_ioctl_open() for interface details.
2557 * Implements further stream config validation and stream initialization on
2558 * behalf of i915_perf_open_ioctl() with the &drm_i915_private->perf.lock mutex
2559 * taken to serialize with any non-file-operation driver hooks.
2561 * Note: at this point the @props have only been validated in isolation and
2562 * it's still necessary to validate that the combination of properties makes
2565 * In the case where userspace is interested in OA unit metrics then further
2566 * config validation and stream initialization details will be handled by
2567 * i915_oa_stream_init(). The code here should only validate config state that
2568 * will be relevant to all stream types / backends.
2570 * Returns: zero on success or a negative error code.
2573 i915_perf_open_ioctl_locked(struct drm_i915_private
*dev_priv
,
2574 struct drm_i915_perf_open_param
*param
,
2575 struct perf_open_properties
*props
,
2576 struct drm_file
*file
)
2578 struct i915_gem_context
*specific_ctx
= NULL
;
2579 struct i915_perf_stream
*stream
= NULL
;
2580 unsigned long f_flags
= 0;
2581 bool privileged_op
= true;
2585 if (props
->single_context
) {
2586 u32 ctx_handle
= props
->ctx_handle
;
2587 struct drm_i915_file_private
*file_priv
= file
->driver_priv
;
2589 specific_ctx
= i915_gem_context_lookup(file_priv
, ctx_handle
);
2590 if (!specific_ctx
) {
2591 DRM_DEBUG("Failed to look up context with ID %u for opening perf stream\n",
2599 * On Haswell the OA unit supports clock gating off for a specific
2600 * context and in this mode there's no visibility of metrics for the
2601 * rest of the system, which we consider acceptable for a
2602 * non-privileged client.
2604 * For Gen8+ the OA unit no longer supports clock gating off for a
2605 * specific context and the kernel can't securely stop the counters
2606 * from updating as system-wide / global values. Even though we can
2607 * filter reports based on the included context ID we can't block
2608 * clients from seeing the raw / global counter values via
2609 * MI_REPORT_PERF_COUNT commands and so consider it a privileged op to
2610 * enable the OA unit by default.
2612 if (IS_HASWELL(dev_priv
) && specific_ctx
)
2613 privileged_op
= false;
2615 /* Similar to perf's kernel.perf_paranoid_cpu sysctl option
2616 * we check a dev.i915.perf_stream_paranoid sysctl option
2617 * to determine if it's ok to access system wide OA counters
2618 * without CAP_SYS_ADMIN privileges.
2620 if (privileged_op
&&
2621 i915_perf_stream_paranoid
&& !capable(CAP_SYS_ADMIN
)) {
2622 DRM_DEBUG("Insufficient privileges to open system-wide i915 perf stream\n");
2627 stream
= kzalloc(sizeof(*stream
), GFP_KERNEL
);
2633 stream
->dev_priv
= dev_priv
;
2634 stream
->ctx
= specific_ctx
;
2636 ret
= i915_oa_stream_init(stream
, param
, props
);
2640 /* we avoid simply assigning stream->sample_flags = props->sample_flags
2641 * to have _stream_init check the combination of sample flags more
2642 * thoroughly, but still this is the expected result at this point.
2644 if (WARN_ON(stream
->sample_flags
!= props
->sample_flags
)) {
2649 list_add(&stream
->link
, &dev_priv
->perf
.streams
);
2651 if (param
->flags
& I915_PERF_FLAG_FD_CLOEXEC
)
2652 f_flags
|= O_CLOEXEC
;
2653 if (param
->flags
& I915_PERF_FLAG_FD_NONBLOCK
)
2654 f_flags
|= O_NONBLOCK
;
2656 stream_fd
= anon_inode_getfd("[i915_perf]", &fops
, stream
, f_flags
);
2657 if (stream_fd
< 0) {
2662 if (!(param
->flags
& I915_PERF_FLAG_DISABLED
))
2663 i915_perf_enable_locked(stream
);
2668 list_del(&stream
->link
);
2670 if (stream
->ops
->destroy
)
2671 stream
->ops
->destroy(stream
);
2676 i915_gem_context_put(specific_ctx
);
2681 static u64
oa_exponent_to_ns(struct drm_i915_private
*dev_priv
, int exponent
)
2683 return div_u64(1000000000ULL * (2ULL << exponent
),
2684 dev_priv
->perf
.oa
.timestamp_frequency
);
2688 * read_properties_unlocked - validate + copy userspace stream open properties
2689 * @dev_priv: i915 device instance
2690 * @uprops: The array of u64 key value pairs given by userspace
2691 * @n_props: The number of key value pairs expected in @uprops
2692 * @props: The stream configuration built up while validating properties
2694 * Note this function only validates properties in isolation it doesn't
2695 * validate that the combination of properties makes sense or that all
2696 * properties necessary for a particular kind of stream have been set.
2698 * Note that there currently aren't any ordering requirements for properties so
2699 * we shouldn't validate or assume anything about ordering here. This doesn't
2700 * rule out defining new properties with ordering requirements in the future.
2702 static int read_properties_unlocked(struct drm_i915_private
*dev_priv
,
2705 struct perf_open_properties
*props
)
2707 u64 __user
*uprop
= uprops
;
2710 memset(props
, 0, sizeof(struct perf_open_properties
));
2713 DRM_DEBUG("No i915 perf properties given\n");
2717 /* Considering that ID = 0 is reserved and assuming that we don't
2718 * (currently) expect any configurations to ever specify duplicate
2719 * values for a particular property ID then the last _PROP_MAX value is
2720 * one greater than the maximum number of properties we expect to get
2723 if (n_props
>= DRM_I915_PERF_PROP_MAX
) {
2724 DRM_DEBUG("More i915 perf properties specified than exist\n");
2728 for (i
= 0; i
< n_props
; i
++) {
2729 u64 oa_period
, oa_freq_hz
;
2733 ret
= get_user(id
, uprop
);
2737 ret
= get_user(value
, uprop
+ 1);
2741 if (id
== 0 || id
>= DRM_I915_PERF_PROP_MAX
) {
2742 DRM_DEBUG("Unknown i915 perf property ID\n");
2746 switch ((enum drm_i915_perf_property_id
)id
) {
2747 case DRM_I915_PERF_PROP_CTX_HANDLE
:
2748 props
->single_context
= 1;
2749 props
->ctx_handle
= value
;
2751 case DRM_I915_PERF_PROP_SAMPLE_OA
:
2752 props
->sample_flags
|= SAMPLE_OA_REPORT
;
2754 case DRM_I915_PERF_PROP_OA_METRICS_SET
:
2756 DRM_DEBUG("Unknown OA metric set ID\n");
2759 props
->metrics_set
= value
;
2761 case DRM_I915_PERF_PROP_OA_FORMAT
:
2762 if (value
== 0 || value
>= I915_OA_FORMAT_MAX
) {
2763 DRM_DEBUG("Out-of-range OA report format %llu\n",
2767 if (!dev_priv
->perf
.oa
.oa_formats
[value
].size
) {
2768 DRM_DEBUG("Unsupported OA report format %llu\n",
2772 props
->oa_format
= value
;
2774 case DRM_I915_PERF_PROP_OA_EXPONENT
:
2775 if (value
> OA_EXPONENT_MAX
) {
2776 DRM_DEBUG("OA timer exponent too high (> %u)\n",
2781 /* Theoretically we can program the OA unit to sample
2782 * e.g. every 160ns for HSW, 167ns for BDW/SKL or 104ns
2783 * for BXT. We don't allow such high sampling
2784 * frequencies by default unless root.
2787 BUILD_BUG_ON(sizeof(oa_period
) != 8);
2788 oa_period
= oa_exponent_to_ns(dev_priv
, value
);
2790 /* This check is primarily to ensure that oa_period <=
2791 * UINT32_MAX (before passing to do_div which only
2792 * accepts a u32 denominator), but we can also skip
2793 * checking anything < 1Hz which implicitly can't be
2794 * limited via an integer oa_max_sample_rate.
2796 if (oa_period
<= NSEC_PER_SEC
) {
2797 u64 tmp
= NSEC_PER_SEC
;
2798 do_div(tmp
, oa_period
);
2803 if (oa_freq_hz
> i915_oa_max_sample_rate
&&
2804 !capable(CAP_SYS_ADMIN
)) {
2805 DRM_DEBUG("OA exponent would exceed the max sampling frequency (sysctl dev.i915.oa_max_sample_rate) %uHz without root privileges\n",
2806 i915_oa_max_sample_rate
);
2810 props
->oa_periodic
= true;
2811 props
->oa_period_exponent
= value
;
2813 case DRM_I915_PERF_PROP_MAX
:
2825 * i915_perf_open_ioctl - DRM ioctl() for userspace to open a stream FD
2827 * @data: ioctl data copied from userspace (unvalidated)
2830 * Validates the stream open parameters given by userspace including flags
2831 * and an array of u64 key, value pair properties.
2833 * Very little is assumed up front about the nature of the stream being
2834 * opened (for instance we don't assume it's for periodic OA unit metrics). An
2835 * i915-perf stream is expected to be a suitable interface for other forms of
2836 * buffered data written by the GPU besides periodic OA metrics.
2838 * Note we copy the properties from userspace outside of the i915 perf
2839 * mutex to avoid an awkward lockdep with mmap_sem.
2841 * Most of the implementation details are handled by
2842 * i915_perf_open_ioctl_locked() after taking the &drm_i915_private->perf.lock
2843 * mutex for serializing with any non-file-operation driver hooks.
2845 * Return: A newly opened i915 Perf stream file descriptor or negative
2846 * error code on failure.
2848 int i915_perf_open_ioctl(struct drm_device
*dev
, void *data
,
2849 struct drm_file
*file
)
2851 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
2852 struct drm_i915_perf_open_param
*param
= data
;
2853 struct perf_open_properties props
;
2854 u32 known_open_flags
;
2857 if (!dev_priv
->perf
.initialized
) {
2858 DRM_DEBUG("i915 perf interface not available for this system\n");
2862 known_open_flags
= I915_PERF_FLAG_FD_CLOEXEC
|
2863 I915_PERF_FLAG_FD_NONBLOCK
|
2864 I915_PERF_FLAG_DISABLED
;
2865 if (param
->flags
& ~known_open_flags
) {
2866 DRM_DEBUG("Unknown drm_i915_perf_open_param flag\n");
2870 ret
= read_properties_unlocked(dev_priv
,
2871 u64_to_user_ptr(param
->properties_ptr
),
2872 param
->num_properties
,
2877 mutex_lock(&dev_priv
->perf
.lock
);
2878 ret
= i915_perf_open_ioctl_locked(dev_priv
, param
, &props
, file
);
2879 mutex_unlock(&dev_priv
->perf
.lock
);
2885 * i915_perf_register - exposes i915-perf to userspace
2886 * @dev_priv: i915 device instance
2888 * In particular OA metric sets are advertised under a sysfs metrics/
2889 * directory allowing userspace to enumerate valid IDs that can be
2890 * used to open an i915-perf stream.
2892 void i915_perf_register(struct drm_i915_private
*dev_priv
)
2896 if (!dev_priv
->perf
.initialized
)
2899 /* To be sure we're synchronized with an attempted
2900 * i915_perf_open_ioctl(); considering that we register after
2901 * being exposed to userspace.
2903 mutex_lock(&dev_priv
->perf
.lock
);
2905 dev_priv
->perf
.metrics_kobj
=
2906 kobject_create_and_add("metrics",
2907 &dev_priv
->drm
.primary
->kdev
->kobj
);
2908 if (!dev_priv
->perf
.metrics_kobj
)
2911 memset(&dev_priv
->perf
.oa
.test_config
, 0,
2912 sizeof(dev_priv
->perf
.oa
.test_config
));
2914 if (IS_HASWELL(dev_priv
)) {
2915 i915_perf_load_test_config_hsw(dev_priv
);
2916 } else if (IS_BROADWELL(dev_priv
)) {
2917 i915_perf_load_test_config_bdw(dev_priv
);
2918 } else if (IS_CHERRYVIEW(dev_priv
)) {
2919 i915_perf_load_test_config_chv(dev_priv
);
2920 } else if (IS_SKYLAKE(dev_priv
)) {
2921 if (IS_SKL_GT2(dev_priv
))
2922 i915_perf_load_test_config_sklgt2(dev_priv
);
2923 else if (IS_SKL_GT3(dev_priv
))
2924 i915_perf_load_test_config_sklgt3(dev_priv
);
2925 else if (IS_SKL_GT4(dev_priv
))
2926 i915_perf_load_test_config_sklgt4(dev_priv
);
2927 } else if (IS_BROXTON(dev_priv
)) {
2928 i915_perf_load_test_config_bxt(dev_priv
);
2929 } else if (IS_KABYLAKE(dev_priv
)) {
2930 if (IS_KBL_GT2(dev_priv
))
2931 i915_perf_load_test_config_kblgt2(dev_priv
);
2932 else if (IS_KBL_GT3(dev_priv
))
2933 i915_perf_load_test_config_kblgt3(dev_priv
);
2934 } else if (IS_GEMINILAKE(dev_priv
)) {
2935 i915_perf_load_test_config_glk(dev_priv
);
2938 if (dev_priv
->perf
.oa
.test_config
.id
== 0)
2941 ret
= sysfs_create_group(dev_priv
->perf
.metrics_kobj
,
2942 &dev_priv
->perf
.oa
.test_config
.sysfs_metric
);
2946 atomic_set(&dev_priv
->perf
.oa
.test_config
.ref_count
, 1);
2951 kobject_put(dev_priv
->perf
.metrics_kobj
);
2952 dev_priv
->perf
.metrics_kobj
= NULL
;
2955 mutex_unlock(&dev_priv
->perf
.lock
);
2959 * i915_perf_unregister - hide i915-perf from userspace
2960 * @dev_priv: i915 device instance
2962 * i915-perf state cleanup is split up into an 'unregister' and
2963 * 'deinit' phase where the interface is first hidden from
2964 * userspace by i915_perf_unregister() before cleaning up
2965 * remaining state in i915_perf_fini().
2967 void i915_perf_unregister(struct drm_i915_private
*dev_priv
)
2969 if (!dev_priv
->perf
.metrics_kobj
)
2972 sysfs_remove_group(dev_priv
->perf
.metrics_kobj
,
2973 &dev_priv
->perf
.oa
.test_config
.sysfs_metric
);
2975 kobject_put(dev_priv
->perf
.metrics_kobj
);
2976 dev_priv
->perf
.metrics_kobj
= NULL
;
2979 static bool gen8_is_valid_flex_addr(struct drm_i915_private
*dev_priv
, u32 addr
)
2981 static const i915_reg_t flex_eu_regs
[] = {
2992 for (i
= 0; i
< ARRAY_SIZE(flex_eu_regs
); i
++) {
2993 if (flex_eu_regs
[i
].reg
== addr
)
2999 static bool gen7_is_valid_b_counter_addr(struct drm_i915_private
*dev_priv
, u32 addr
)
3001 return (addr
>= OASTARTTRIG1
.reg
&& addr
<= OASTARTTRIG8
.reg
) ||
3002 (addr
>= OAREPORTTRIG1
.reg
&& addr
<= OAREPORTTRIG8
.reg
) ||
3003 (addr
>= OACEC0_0
.reg
&& addr
<= OACEC7_1
.reg
);
3006 static bool gen7_is_valid_mux_addr(struct drm_i915_private
*dev_priv
, u32 addr
)
3008 return addr
== HALF_SLICE_CHICKEN2
.reg
||
3009 (addr
>= MICRO_BP0_0
.reg
&& addr
<= NOA_WRITE
.reg
) ||
3010 (addr
>= OA_PERFCNT1_LO
.reg
&& addr
<= OA_PERFCNT2_HI
.reg
) ||
3011 (addr
>= OA_PERFMATRIX_LO
.reg
&& addr
<= OA_PERFMATRIX_HI
.reg
);
3014 static bool gen8_is_valid_mux_addr(struct drm_i915_private
*dev_priv
, u32 addr
)
3016 return gen7_is_valid_mux_addr(dev_priv
, addr
) ||
3017 addr
== WAIT_FOR_RC6_EXIT
.reg
||
3018 (addr
>= RPM_CONFIG0
.reg
&& addr
<= NOA_CONFIG(8).reg
);
3021 static bool hsw_is_valid_mux_addr(struct drm_i915_private
*dev_priv
, u32 addr
)
3023 return gen7_is_valid_mux_addr(dev_priv
, addr
) ||
3024 (addr
>= 0x25100 && addr
<= 0x2FF90) ||
3028 static bool chv_is_valid_mux_addr(struct drm_i915_private
*dev_priv
, u32 addr
)
3030 return gen7_is_valid_mux_addr(dev_priv
, addr
) ||
3031 (addr
>= 0x182300 && addr
<= 0x1823A4);
3034 static uint32_t mask_reg_value(u32 reg
, u32 val
)
3036 /* HALF_SLICE_CHICKEN2 is programmed with a the
3037 * WaDisableSTUnitPowerOptimization workaround. Make sure the value
3038 * programmed by userspace doesn't change this.
3040 if (HALF_SLICE_CHICKEN2
.reg
== reg
)
3041 val
= val
& ~_MASKED_BIT_ENABLE(GEN8_ST_PO_DISABLE
);
3043 /* WAIT_FOR_RC6_EXIT has only one bit fullfilling the function
3044 * indicated by its name and a bunch of selection fields used by OA
3047 if (WAIT_FOR_RC6_EXIT
.reg
== reg
)
3048 val
= val
& ~_MASKED_BIT_ENABLE(HSW_WAIT_FOR_RC6_EXIT_ENABLE
);
3053 static struct i915_oa_reg
*alloc_oa_regs(struct drm_i915_private
*dev_priv
,
3054 bool (*is_valid
)(struct drm_i915_private
*dev_priv
, u32 addr
),
3058 struct i915_oa_reg
*oa_regs
;
3065 if (!access_ok(VERIFY_READ
, regs
, n_regs
* sizeof(u32
) * 2))
3066 return ERR_PTR(-EFAULT
);
3068 /* No is_valid function means we're not allowing any register to be programmed. */
3069 GEM_BUG_ON(!is_valid
);
3071 return ERR_PTR(-EINVAL
);
3073 oa_regs
= kmalloc_array(n_regs
, sizeof(*oa_regs
), GFP_KERNEL
);
3075 return ERR_PTR(-ENOMEM
);
3077 for (i
= 0; i
< n_regs
; i
++) {
3080 err
= get_user(addr
, regs
);
3084 if (!is_valid(dev_priv
, addr
)) {
3085 DRM_DEBUG("Invalid oa_reg address: %X\n", addr
);
3090 err
= get_user(value
, regs
+ 1);
3094 oa_regs
[i
].addr
= _MMIO(addr
);
3095 oa_regs
[i
].value
= mask_reg_value(addr
, value
);
3104 return ERR_PTR(err
);
3107 static ssize_t
show_dynamic_id(struct device
*dev
,
3108 struct device_attribute
*attr
,
3111 struct i915_oa_config
*oa_config
=
3112 container_of(attr
, typeof(*oa_config
), sysfs_metric_id
);
3114 return sprintf(buf
, "%d\n", oa_config
->id
);
3117 static int create_dynamic_oa_sysfs_entry(struct drm_i915_private
*dev_priv
,
3118 struct i915_oa_config
*oa_config
)
3120 oa_config
->sysfs_metric_id
.attr
.name
= "id";
3121 oa_config
->sysfs_metric_id
.attr
.mode
= S_IRUGO
;
3122 oa_config
->sysfs_metric_id
.show
= show_dynamic_id
;
3123 oa_config
->sysfs_metric_id
.store
= NULL
;
3125 oa_config
->attrs
[0] = &oa_config
->sysfs_metric_id
.attr
;
3126 oa_config
->attrs
[1] = NULL
;
3128 oa_config
->sysfs_metric
.name
= oa_config
->uuid
;
3129 oa_config
->sysfs_metric
.attrs
= oa_config
->attrs
;
3131 return sysfs_create_group(dev_priv
->perf
.metrics_kobj
,
3132 &oa_config
->sysfs_metric
);
3136 * i915_perf_add_config_ioctl - DRM ioctl() for userspace to add a new OA config
3138 * @data: ioctl data (pointer to struct drm_i915_perf_oa_config) copied from
3139 * userspace (unvalidated)
3142 * Validates the submitted OA register to be saved into a new OA config that
3143 * can then be used for programming the OA unit and its NOA network.
3145 * Returns: A new allocated config number to be used with the perf open ioctl
3146 * or a negative error code on failure.
3148 int i915_perf_add_config_ioctl(struct drm_device
*dev
, void *data
,
3149 struct drm_file
*file
)
3151 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
3152 struct drm_i915_perf_oa_config
*args
= data
;
3153 struct i915_oa_config
*oa_config
, *tmp
;
3156 if (!dev_priv
->perf
.initialized
) {
3157 DRM_DEBUG("i915 perf interface not available for this system\n");
3161 if (!dev_priv
->perf
.metrics_kobj
) {
3162 DRM_DEBUG("OA metrics weren't advertised via sysfs\n");
3166 if (i915_perf_stream_paranoid
&& !capable(CAP_SYS_ADMIN
)) {
3167 DRM_DEBUG("Insufficient privileges to add i915 OA config\n");
3171 if ((!args
->mux_regs_ptr
|| !args
->n_mux_regs
) &&
3172 (!args
->boolean_regs_ptr
|| !args
->n_boolean_regs
) &&
3173 (!args
->flex_regs_ptr
|| !args
->n_flex_regs
)) {
3174 DRM_DEBUG("No OA registers given\n");
3178 oa_config
= kzalloc(sizeof(*oa_config
), GFP_KERNEL
);
3180 DRM_DEBUG("Failed to allocate memory for the OA config\n");
3184 atomic_set(&oa_config
->ref_count
, 1);
3186 if (!uuid_is_valid(args
->uuid
)) {
3187 DRM_DEBUG("Invalid uuid format for OA config\n");
3192 /* Last character in oa_config->uuid will be 0 because oa_config is
3195 memcpy(oa_config
->uuid
, args
->uuid
, sizeof(args
->uuid
));
3197 oa_config
->mux_regs_len
= args
->n_mux_regs
;
3198 oa_config
->mux_regs
=
3199 alloc_oa_regs(dev_priv
,
3200 dev_priv
->perf
.oa
.ops
.is_valid_mux_reg
,
3201 u64_to_user_ptr(args
->mux_regs_ptr
),
3204 if (IS_ERR(oa_config
->mux_regs
)) {
3205 DRM_DEBUG("Failed to create OA config for mux_regs\n");
3206 err
= PTR_ERR(oa_config
->mux_regs
);
3210 oa_config
->b_counter_regs_len
= args
->n_boolean_regs
;
3211 oa_config
->b_counter_regs
=
3212 alloc_oa_regs(dev_priv
,
3213 dev_priv
->perf
.oa
.ops
.is_valid_b_counter_reg
,
3214 u64_to_user_ptr(args
->boolean_regs_ptr
),
3215 args
->n_boolean_regs
);
3217 if (IS_ERR(oa_config
->b_counter_regs
)) {
3218 DRM_DEBUG("Failed to create OA config for b_counter_regs\n");
3219 err
= PTR_ERR(oa_config
->b_counter_regs
);
3223 if (INTEL_GEN(dev_priv
) < 8) {
3224 if (args
->n_flex_regs
!= 0) {
3229 oa_config
->flex_regs_len
= args
->n_flex_regs
;
3230 oa_config
->flex_regs
=
3231 alloc_oa_regs(dev_priv
,
3232 dev_priv
->perf
.oa
.ops
.is_valid_flex_reg
,
3233 u64_to_user_ptr(args
->flex_regs_ptr
),
3236 if (IS_ERR(oa_config
->flex_regs
)) {
3237 DRM_DEBUG("Failed to create OA config for flex_regs\n");
3238 err
= PTR_ERR(oa_config
->flex_regs
);
3243 err
= mutex_lock_interruptible(&dev_priv
->perf
.metrics_lock
);
3247 /* We shouldn't have too many configs, so this iteration shouldn't be
3250 idr_for_each_entry(&dev_priv
->perf
.metrics_idr
, tmp
, id
) {
3251 if (!strcmp(tmp
->uuid
, oa_config
->uuid
)) {
3252 DRM_DEBUG("OA config already exists with this uuid\n");
3258 err
= create_dynamic_oa_sysfs_entry(dev_priv
, oa_config
);
3260 DRM_DEBUG("Failed to create sysfs entry for OA config\n");
3264 /* Config id 0 is invalid, id 1 for kernel stored test config. */
3265 oa_config
->id
= idr_alloc(&dev_priv
->perf
.metrics_idr
,
3268 if (oa_config
->id
< 0) {
3269 DRM_DEBUG("Failed to create sysfs entry for OA config\n");
3270 err
= oa_config
->id
;
3274 mutex_unlock(&dev_priv
->perf
.metrics_lock
);
3276 return oa_config
->id
;
3279 mutex_unlock(&dev_priv
->perf
.metrics_lock
);
3281 put_oa_config(dev_priv
, oa_config
);
3282 DRM_DEBUG("Failed to add new OA config\n");
3287 * i915_perf_remove_config_ioctl - DRM ioctl() for userspace to remove an OA config
3289 * @data: ioctl data (pointer to u64 integer) copied from userspace
3292 * Configs can be removed while being used, the will stop appearing in sysfs
3293 * and their content will be freed when the stream using the config is closed.
3295 * Returns: 0 on success or a negative error code on failure.
3297 int i915_perf_remove_config_ioctl(struct drm_device
*dev
, void *data
,
3298 struct drm_file
*file
)
3300 struct drm_i915_private
*dev_priv
= dev
->dev_private
;
3302 struct i915_oa_config
*oa_config
;
3305 if (!dev_priv
->perf
.initialized
) {
3306 DRM_DEBUG("i915 perf interface not available for this system\n");
3310 if (i915_perf_stream_paranoid
&& !capable(CAP_SYS_ADMIN
)) {
3311 DRM_DEBUG("Insufficient privileges to remove i915 OA config\n");
3315 ret
= mutex_lock_interruptible(&dev_priv
->perf
.metrics_lock
);
3319 oa_config
= idr_find(&dev_priv
->perf
.metrics_idr
, *arg
);
3321 DRM_DEBUG("Failed to remove unknown OA config\n");
3326 GEM_BUG_ON(*arg
!= oa_config
->id
);
3328 sysfs_remove_group(dev_priv
->perf
.metrics_kobj
,
3329 &oa_config
->sysfs_metric
);
3331 idr_remove(&dev_priv
->perf
.metrics_idr
, *arg
);
3332 put_oa_config(dev_priv
, oa_config
);
3335 mutex_unlock(&dev_priv
->perf
.metrics_lock
);
3340 static struct ctl_table oa_table
[] = {
3342 .procname
= "perf_stream_paranoid",
3343 .data
= &i915_perf_stream_paranoid
,
3344 .maxlen
= sizeof(i915_perf_stream_paranoid
),
3346 .proc_handler
= proc_dointvec_minmax
,
3351 .procname
= "oa_max_sample_rate",
3352 .data
= &i915_oa_max_sample_rate
,
3353 .maxlen
= sizeof(i915_oa_max_sample_rate
),
3355 .proc_handler
= proc_dointvec_minmax
,
3357 .extra2
= &oa_sample_rate_hard_limit
,
3362 static struct ctl_table i915_root
[] = {
3372 static struct ctl_table dev_root
[] = {
3383 * i915_perf_init - initialize i915-perf state on module load
3384 * @dev_priv: i915 device instance
3386 * Initializes i915-perf state without exposing anything to userspace.
3388 * Note: i915-perf initialization is split into an 'init' and 'register'
3389 * phase with the i915_perf_register() exposing state to userspace.
3391 void i915_perf_init(struct drm_i915_private
*dev_priv
)
3393 dev_priv
->perf
.oa
.timestamp_frequency
= 0;
3395 if (IS_HASWELL(dev_priv
)) {
3396 dev_priv
->perf
.oa
.ops
.is_valid_b_counter_reg
=
3397 gen7_is_valid_b_counter_addr
;
3398 dev_priv
->perf
.oa
.ops
.is_valid_mux_reg
=
3399 hsw_is_valid_mux_addr
;
3400 dev_priv
->perf
.oa
.ops
.is_valid_flex_reg
= NULL
;
3401 dev_priv
->perf
.oa
.ops
.init_oa_buffer
= gen7_init_oa_buffer
;
3402 dev_priv
->perf
.oa
.ops
.enable_metric_set
= hsw_enable_metric_set
;
3403 dev_priv
->perf
.oa
.ops
.disable_metric_set
= hsw_disable_metric_set
;
3404 dev_priv
->perf
.oa
.ops
.oa_enable
= gen7_oa_enable
;
3405 dev_priv
->perf
.oa
.ops
.oa_disable
= gen7_oa_disable
;
3406 dev_priv
->perf
.oa
.ops
.read
= gen7_oa_read
;
3407 dev_priv
->perf
.oa
.ops
.oa_hw_tail_read
=
3408 gen7_oa_hw_tail_read
;
3410 dev_priv
->perf
.oa
.timestamp_frequency
= 12500000;
3412 dev_priv
->perf
.oa
.oa_formats
= hsw_oa_formats
;
3413 } else if (i915
.enable_execlists
) {
3414 /* Note: that although we could theoretically also support the
3415 * legacy ringbuffer mode on BDW (and earlier iterations of
3416 * this driver, before upstreaming did this) it didn't seem
3417 * worth the complexity to maintain now that BDW+ enable
3418 * execlist mode by default.
3420 dev_priv
->perf
.oa
.ops
.is_valid_b_counter_reg
=
3421 gen7_is_valid_b_counter_addr
;
3422 dev_priv
->perf
.oa
.ops
.is_valid_mux_reg
=
3423 gen8_is_valid_mux_addr
;
3424 dev_priv
->perf
.oa
.ops
.is_valid_flex_reg
=
3425 gen8_is_valid_flex_addr
;
3427 dev_priv
->perf
.oa
.ops
.init_oa_buffer
= gen8_init_oa_buffer
;
3428 dev_priv
->perf
.oa
.ops
.enable_metric_set
= gen8_enable_metric_set
;
3429 dev_priv
->perf
.oa
.ops
.disable_metric_set
= gen8_disable_metric_set
;
3430 dev_priv
->perf
.oa
.ops
.oa_enable
= gen8_oa_enable
;
3431 dev_priv
->perf
.oa
.ops
.oa_disable
= gen8_oa_disable
;
3432 dev_priv
->perf
.oa
.ops
.read
= gen8_oa_read
;
3433 dev_priv
->perf
.oa
.ops
.oa_hw_tail_read
= gen8_oa_hw_tail_read
;
3435 dev_priv
->perf
.oa
.oa_formats
= gen8_plus_oa_formats
;
3437 if (IS_GEN8(dev_priv
)) {
3438 dev_priv
->perf
.oa
.ctx_oactxctrl_offset
= 0x120;
3439 dev_priv
->perf
.oa
.ctx_flexeu0_offset
= 0x2ce;
3441 dev_priv
->perf
.oa
.timestamp_frequency
= 12500000;
3443 dev_priv
->perf
.oa
.gen8_valid_ctx_bit
= (1<<25);
3444 if (IS_CHERRYVIEW(dev_priv
)) {
3445 dev_priv
->perf
.oa
.ops
.is_valid_mux_reg
=
3446 chv_is_valid_mux_addr
;
3448 } else if (IS_GEN9(dev_priv
)) {
3449 dev_priv
->perf
.oa
.ctx_oactxctrl_offset
= 0x128;
3450 dev_priv
->perf
.oa
.ctx_flexeu0_offset
= 0x3de;
3452 dev_priv
->perf
.oa
.gen8_valid_ctx_bit
= (1<<16);
3454 switch (dev_priv
->info
.platform
) {
3456 case INTEL_GEMINILAKE
:
3457 dev_priv
->perf
.oa
.timestamp_frequency
= 19200000;
3460 case INTEL_KABYLAKE
:
3461 dev_priv
->perf
.oa
.timestamp_frequency
= 12000000;
3464 /* Leave timestamp_frequency to 0 so we can
3465 * detect unsupported platforms.
3472 if (dev_priv
->perf
.oa
.timestamp_frequency
) {
3473 hrtimer_init(&dev_priv
->perf
.oa
.poll_check_timer
,
3474 CLOCK_MONOTONIC
, HRTIMER_MODE_REL
);
3475 dev_priv
->perf
.oa
.poll_check_timer
.function
= oa_poll_check_timer_cb
;
3476 init_waitqueue_head(&dev_priv
->perf
.oa
.poll_wq
);
3478 INIT_LIST_HEAD(&dev_priv
->perf
.streams
);
3479 mutex_init(&dev_priv
->perf
.lock
);
3480 spin_lock_init(&dev_priv
->perf
.oa
.oa_buffer
.ptr_lock
);
3482 oa_sample_rate_hard_limit
=
3483 dev_priv
->perf
.oa
.timestamp_frequency
/ 2;
3484 dev_priv
->perf
.sysctl_header
= register_sysctl_table(dev_root
);
3486 mutex_init(&dev_priv
->perf
.metrics_lock
);
3487 idr_init(&dev_priv
->perf
.metrics_idr
);
3489 dev_priv
->perf
.initialized
= true;
3493 static int destroy_config(int id
, void *p
, void *data
)
3495 struct drm_i915_private
*dev_priv
= data
;
3496 struct i915_oa_config
*oa_config
= p
;
3498 put_oa_config(dev_priv
, oa_config
);
3504 * i915_perf_fini - Counter part to i915_perf_init()
3505 * @dev_priv: i915 device instance
3507 void i915_perf_fini(struct drm_i915_private
*dev_priv
)
3509 if (!dev_priv
->perf
.initialized
)
3512 idr_for_each(&dev_priv
->perf
.metrics_idr
, destroy_config
, dev_priv
);
3513 idr_destroy(&dev_priv
->perf
.metrics_idr
);
3515 unregister_sysctl_table(dev_priv
->perf
.sysctl_header
);
3517 memset(&dev_priv
->perf
.oa
.ops
, 0, sizeof(dev_priv
->perf
.oa
.ops
));
3519 dev_priv
->perf
.initialized
= false;