]>
Commit | Line | Data |
---|---|---|
eec688e1 RB |
1 | /* |
2 | * Copyright © 2015-2016 Intel Corporation | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING | |
20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
21 | * IN THE SOFTWARE. | |
22 | * | |
23 | * Authors: | |
24 | * Robert Bragg <robert@sixbynine.org> | |
25 | */ | |
26 | ||
27 | #include <linux/anon_inodes.h> | |
d7965152 | 28 | #include <linux/sizes.h> |
eec688e1 RB |
29 | |
30 | #include "i915_drv.h" | |
d7965152 RB |
31 | #include "i915_oa_hsw.h" |
32 | ||
33 | /* HW requires this to be a power of two, between 128k and 16M, though driver | |
34 | * is currently generally designed assuming the largest 16M size is used such | |
35 | * that the overflow cases are unlikely in normal operation. | |
36 | */ | |
37 | #define OA_BUFFER_SIZE SZ_16M | |
38 | ||
39 | #define OA_TAKEN(tail, head) ((tail - head) & (OA_BUFFER_SIZE - 1)) | |
40 | ||
41 | /* There's a HW race condition between OA unit tail pointer register updates and | |
42 | * writes to memory whereby the tail pointer can sometimes get ahead of what's | |
43 | * been written out to the OA buffer so far. | |
44 | * | |
45 | * Although this can be observed explicitly by checking for a zeroed report-id | |
46 | * field in tail reports, it seems preferable to account for this earlier e.g. | |
47 | * as part of the _oa_buffer_is_empty checks to minimize -EAGAIN polling cycles | |
48 | * in this situation. | |
49 | * | |
50 | * To give time for the most recent reports to land before they may be copied to | |
51 | * userspace, the driver operates as if the tail pointer effectively lags behind | |
52 | * the HW tail pointer by 'tail_margin' bytes. The margin in bytes is calculated | |
53 | * based on this constant in nanoseconds, the current OA sampling exponent | |
54 | * and current report size. | |
55 | * | |
56 | * There is also a fallback check while reading to simply skip over reports with | |
57 | * a zeroed report-id. | |
58 | */ | |
59 | #define OA_TAIL_MARGIN_NSEC 100000ULL | |
60 | ||
61 | /* frequency for checking whether the OA unit has written new reports to the | |
62 | * circular OA buffer... | |
63 | */ | |
64 | #define POLL_FREQUENCY 200 | |
65 | #define POLL_PERIOD (NSEC_PER_SEC / POLL_FREQUENCY) | |
66 | ||
67 | /* The maximum exponent the hardware accepts is 63 (essentially it selects one | |
68 | * of the 64bit timestamp bits to trigger reports from) but there's currently | |
69 | * no known use case for sampling as infrequently as once per 47 thousand years. | |
70 | * | |
71 | * Since the timestamps included in OA reports are only 32bits it seems | |
72 | * reasonable to limit the OA exponent where it's still possible to account for | |
73 | * overflow in OA report timestamps. | |
74 | */ | |
75 | #define OA_EXPONENT_MAX 31 | |
76 | ||
77 | #define INVALID_CTX_ID 0xffffffff | |
78 | ||
79 | ||
80 | /* XXX: beware if future OA HW adds new report formats that the current | |
81 | * code assumes all reports have a power-of-two size and ~(size - 1) can | |
82 | * be used as a mask to align the OA tail pointer. | |
83 | */ | |
84 | static struct i915_oa_format hsw_oa_formats[I915_OA_FORMAT_MAX] = { | |
85 | [I915_OA_FORMAT_A13] = { 0, 64 }, | |
86 | [I915_OA_FORMAT_A29] = { 1, 128 }, | |
87 | [I915_OA_FORMAT_A13_B8_C8] = { 2, 128 }, | |
88 | /* A29_B8_C8 Disallowed as 192 bytes doesn't factor into buffer size */ | |
89 | [I915_OA_FORMAT_B4_C8] = { 4, 64 }, | |
90 | [I915_OA_FORMAT_A45_B8_C8] = { 5, 256 }, | |
91 | [I915_OA_FORMAT_B4_C8_A16] = { 6, 128 }, | |
92 | [I915_OA_FORMAT_C4_B8] = { 7, 64 }, | |
93 | }; | |
94 | ||
95 | #define SAMPLE_OA_REPORT (1<<0) | |
eec688e1 RB |
96 | |
97 | struct perf_open_properties { | |
98 | u32 sample_flags; | |
99 | ||
100 | u64 single_context:1; | |
101 | u64 ctx_handle; | |
d7965152 RB |
102 | |
103 | /* OA sampling state */ | |
104 | int metrics_set; | |
105 | int oa_format; | |
106 | bool oa_periodic; | |
107 | int oa_period_exponent; | |
108 | }; | |
109 | ||
110 | /* NB: This is either called via fops or the poll check hrtimer (atomic ctx) | |
111 | * | |
112 | * It's safe to read OA config state here unlocked, assuming that this is only | |
113 | * called while the stream is enabled, while the global OA configuration can't | |
114 | * be modified. | |
115 | * | |
116 | * Note: we don't lock around the head/tail reads even though there's the slim | |
117 | * possibility of read() fop errors forcing a re-init of the OA buffer | |
118 | * pointers. A race here could result in a false positive !empty status which | |
119 | * is acceptable. | |
120 | */ | |
121 | static bool gen7_oa_buffer_is_empty_fop_unlocked(struct drm_i915_private *dev_priv) | |
122 | { | |
123 | int report_size = dev_priv->perf.oa.oa_buffer.format_size; | |
124 | u32 oastatus2 = I915_READ(GEN7_OASTATUS2); | |
125 | u32 oastatus1 = I915_READ(GEN7_OASTATUS1); | |
126 | u32 head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; | |
127 | u32 tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; | |
128 | ||
129 | return OA_TAKEN(tail, head) < | |
130 | dev_priv->perf.oa.tail_margin + report_size; | |
131 | } | |
132 | ||
133 | /** | |
134 | * Appends a status record to a userspace read() buffer. | |
135 | */ | |
136 | static int append_oa_status(struct i915_perf_stream *stream, | |
137 | char __user *buf, | |
138 | size_t count, | |
139 | size_t *offset, | |
140 | enum drm_i915_perf_record_type type) | |
141 | { | |
142 | struct drm_i915_perf_record_header header = { type, 0, sizeof(header) }; | |
143 | ||
144 | if ((count - *offset) < header.size) | |
145 | return -ENOSPC; | |
146 | ||
147 | if (copy_to_user(buf + *offset, &header, sizeof(header))) | |
148 | return -EFAULT; | |
149 | ||
150 | (*offset) += header.size; | |
151 | ||
152 | return 0; | |
153 | } | |
154 | ||
155 | /** | |
156 | * Copies single OA report into userspace read() buffer. | |
157 | */ | |
158 | static int append_oa_sample(struct i915_perf_stream *stream, | |
159 | char __user *buf, | |
160 | size_t count, | |
161 | size_t *offset, | |
162 | const u8 *report) | |
163 | { | |
164 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
165 | int report_size = dev_priv->perf.oa.oa_buffer.format_size; | |
166 | struct drm_i915_perf_record_header header; | |
167 | u32 sample_flags = stream->sample_flags; | |
168 | ||
169 | header.type = DRM_I915_PERF_RECORD_SAMPLE; | |
170 | header.pad = 0; | |
171 | header.size = stream->sample_size; | |
172 | ||
173 | if ((count - *offset) < header.size) | |
174 | return -ENOSPC; | |
175 | ||
176 | buf += *offset; | |
177 | if (copy_to_user(buf, &header, sizeof(header))) | |
178 | return -EFAULT; | |
179 | buf += sizeof(header); | |
180 | ||
181 | if (sample_flags & SAMPLE_OA_REPORT) { | |
182 | if (copy_to_user(buf, report, report_size)) | |
183 | return -EFAULT; | |
184 | } | |
185 | ||
186 | (*offset) += header.size; | |
187 | ||
188 | return 0; | |
189 | } | |
190 | ||
191 | /** | |
192 | * Copies all buffered OA reports into userspace read() buffer. | |
193 | * @stream: An i915-perf stream opened for OA metrics | |
194 | * @buf: destination buffer given by userspace | |
195 | * @count: the number of bytes userspace wants to read | |
196 | * @offset: (inout): the current position for writing into @buf | |
197 | * @head_ptr: (inout): the current oa buffer cpu read position | |
198 | * @tail: the current oa buffer gpu write position | |
199 | * | |
200 | * Returns 0 on success, negative error code on failure. | |
201 | * | |
202 | * Notably any error condition resulting in a short read (-ENOSPC or | |
203 | * -EFAULT) will be returned even though one or more records may | |
204 | * have been successfully copied. In this case it's up to the caller | |
205 | * to decide if the error should be squashed before returning to | |
206 | * userspace. | |
207 | * | |
208 | * Note: reports are consumed from the head, and appended to the | |
209 | * tail, so the head chases the tail?... If you think that's mad | |
210 | * and back-to-front you're not alone, but this follows the | |
211 | * Gen PRM naming convention. | |
212 | */ | |
213 | static int gen7_append_oa_reports(struct i915_perf_stream *stream, | |
214 | char __user *buf, | |
215 | size_t count, | |
216 | size_t *offset, | |
217 | u32 *head_ptr, | |
218 | u32 tail) | |
219 | { | |
220 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
221 | int report_size = dev_priv->perf.oa.oa_buffer.format_size; | |
222 | u8 *oa_buf_base = dev_priv->perf.oa.oa_buffer.vaddr; | |
223 | int tail_margin = dev_priv->perf.oa.tail_margin; | |
224 | u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); | |
225 | u32 mask = (OA_BUFFER_SIZE - 1); | |
226 | u32 head; | |
227 | u32 taken; | |
228 | int ret = 0; | |
229 | ||
230 | if (WARN_ON(!stream->enabled)) | |
231 | return -EIO; | |
232 | ||
233 | head = *head_ptr - gtt_offset; | |
234 | tail -= gtt_offset; | |
235 | ||
236 | /* The OA unit is expected to wrap the tail pointer according to the OA | |
237 | * buffer size and since we should never write a misaligned head | |
238 | * pointer we don't expect to read one back either... | |
239 | */ | |
240 | if (tail > OA_BUFFER_SIZE || head > OA_BUFFER_SIZE || | |
241 | head % report_size) { | |
242 | DRM_ERROR("Inconsistent OA buffer pointer (head = %u, tail = %u): force restart\n", | |
243 | head, tail); | |
244 | dev_priv->perf.oa.ops.oa_disable(dev_priv); | |
245 | dev_priv->perf.oa.ops.oa_enable(dev_priv); | |
246 | *head_ptr = I915_READ(GEN7_OASTATUS2) & | |
247 | GEN7_OASTATUS2_HEAD_MASK; | |
248 | return -EIO; | |
249 | } | |
250 | ||
251 | ||
252 | /* The tail pointer increases in 64 byte increments, not in report_size | |
253 | * steps... | |
254 | */ | |
255 | tail &= ~(report_size - 1); | |
256 | ||
257 | /* Move the tail pointer back by the current tail_margin to account for | |
258 | * the possibility that the latest reports may not have really landed | |
259 | * in memory yet... | |
260 | */ | |
261 | ||
262 | if (OA_TAKEN(tail, head) < report_size + tail_margin) | |
263 | return -EAGAIN; | |
264 | ||
265 | tail -= tail_margin; | |
266 | tail &= mask; | |
267 | ||
268 | for (/* none */; | |
269 | (taken = OA_TAKEN(tail, head)); | |
270 | head = (head + report_size) & mask) { | |
271 | u8 *report = oa_buf_base + head; | |
272 | u32 *report32 = (void *)report; | |
273 | ||
274 | /* All the report sizes factor neatly into the buffer | |
275 | * size so we never expect to see a report split | |
276 | * between the beginning and end of the buffer. | |
277 | * | |
278 | * Given the initial alignment check a misalignment | |
279 | * here would imply a driver bug that would result | |
280 | * in an overrun. | |
281 | */ | |
282 | if (WARN_ON((OA_BUFFER_SIZE - head) < report_size)) { | |
283 | DRM_ERROR("Spurious OA head ptr: non-integral report offset\n"); | |
284 | break; | |
285 | } | |
286 | ||
287 | /* The report-ID field for periodic samples includes | |
288 | * some undocumented flags related to what triggered | |
289 | * the report and is never expected to be zero so we | |
290 | * can check that the report isn't invalid before | |
291 | * copying it to userspace... | |
292 | */ | |
293 | if (report32[0] == 0) { | |
294 | DRM_ERROR("Skipping spurious, invalid OA report\n"); | |
295 | continue; | |
296 | } | |
297 | ||
298 | ret = append_oa_sample(stream, buf, count, offset, report); | |
299 | if (ret) | |
300 | break; | |
301 | ||
302 | /* The above report-id field sanity check is based on | |
303 | * the assumption that the OA buffer is initially | |
304 | * zeroed and we reset the field after copying so the | |
305 | * check is still meaningful once old reports start | |
306 | * being overwritten. | |
307 | */ | |
308 | report32[0] = 0; | |
309 | } | |
310 | ||
311 | *head_ptr = gtt_offset + head; | |
312 | ||
313 | return ret; | |
314 | } | |
315 | ||
316 | static int gen7_oa_read(struct i915_perf_stream *stream, | |
317 | char __user *buf, | |
318 | size_t count, | |
319 | size_t *offset) | |
320 | { | |
321 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
322 | int report_size = dev_priv->perf.oa.oa_buffer.format_size; | |
323 | u32 oastatus2; | |
324 | u32 oastatus1; | |
325 | u32 head; | |
326 | u32 tail; | |
327 | int ret; | |
328 | ||
329 | if (WARN_ON(!dev_priv->perf.oa.oa_buffer.vaddr)) | |
330 | return -EIO; | |
331 | ||
332 | oastatus2 = I915_READ(GEN7_OASTATUS2); | |
333 | oastatus1 = I915_READ(GEN7_OASTATUS1); | |
334 | ||
335 | head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; | |
336 | tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; | |
337 | ||
338 | /* XXX: On Haswell we don't have a safe way to clear oastatus1 | |
339 | * bits while the OA unit is enabled (while the tail pointer | |
340 | * may be updated asynchronously) so we ignore status bits | |
341 | * that have already been reported to userspace. | |
342 | */ | |
343 | oastatus1 &= ~dev_priv->perf.oa.gen7_latched_oastatus1; | |
344 | ||
345 | /* We treat OABUFFER_OVERFLOW as a significant error: | |
346 | * | |
347 | * - The status can be interpreted to mean that the buffer is | |
348 | * currently full (with a higher precedence than OA_TAKEN() | |
349 | * which will start to report a near-empty buffer after an | |
350 | * overflow) but it's awkward that we can't clear the status | |
351 | * on Haswell, so without a reset we won't be able to catch | |
352 | * the state again. | |
353 | * | |
354 | * - Since it also implies the HW has started overwriting old | |
355 | * reports it may also affect our sanity checks for invalid | |
356 | * reports when copying to userspace that assume new reports | |
357 | * are being written to cleared memory. | |
358 | * | |
359 | * - In the future we may want to introduce a flight recorder | |
360 | * mode where the driver will automatically maintain a safe | |
361 | * guard band between head/tail, avoiding this overflow | |
362 | * condition, but we avoid the added driver complexity for | |
363 | * now. | |
364 | */ | |
365 | if (unlikely(oastatus1 & GEN7_OASTATUS1_OABUFFER_OVERFLOW)) { | |
366 | ret = append_oa_status(stream, buf, count, offset, | |
367 | DRM_I915_PERF_RECORD_OA_BUFFER_LOST); | |
368 | if (ret) | |
369 | return ret; | |
370 | ||
371 | DRM_ERROR("OA buffer overflow: force restart\n"); | |
372 | ||
373 | dev_priv->perf.oa.ops.oa_disable(dev_priv); | |
374 | dev_priv->perf.oa.ops.oa_enable(dev_priv); | |
375 | ||
376 | oastatus2 = I915_READ(GEN7_OASTATUS2); | |
377 | oastatus1 = I915_READ(GEN7_OASTATUS1); | |
378 | ||
379 | head = oastatus2 & GEN7_OASTATUS2_HEAD_MASK; | |
380 | tail = oastatus1 & GEN7_OASTATUS1_TAIL_MASK; | |
381 | } | |
382 | ||
383 | if (unlikely(oastatus1 & GEN7_OASTATUS1_REPORT_LOST)) { | |
384 | ret = append_oa_status(stream, buf, count, offset, | |
385 | DRM_I915_PERF_RECORD_OA_REPORT_LOST); | |
386 | if (ret) | |
387 | return ret; | |
388 | dev_priv->perf.oa.gen7_latched_oastatus1 |= | |
389 | GEN7_OASTATUS1_REPORT_LOST; | |
390 | } | |
391 | ||
392 | ret = gen7_append_oa_reports(stream, buf, count, offset, | |
393 | &head, tail); | |
394 | ||
395 | /* All the report sizes are a power of two and the | |
396 | * head should always be incremented by some multiple | |
397 | * of the report size. | |
398 | * | |
399 | * A warning here, but notably if we later read back a | |
400 | * misaligned pointer we will treat that as a bug since | |
401 | * it could lead to a buffer overrun. | |
402 | */ | |
403 | WARN_ONCE(head & (report_size - 1), | |
404 | "i915: Writing misaligned OA head pointer"); | |
405 | ||
406 | /* Note: we update the head pointer here even if an error | |
407 | * was returned since the error may represent a short read | |
408 | * where some some reports were successfully copied. | |
409 | */ | |
410 | I915_WRITE(GEN7_OASTATUS2, | |
411 | ((head & GEN7_OASTATUS2_HEAD_MASK) | | |
412 | OA_MEM_SELECT_GGTT)); | |
413 | ||
414 | return ret; | |
415 | } | |
416 | ||
417 | static int i915_oa_wait_unlocked(struct i915_perf_stream *stream) | |
418 | { | |
419 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
420 | ||
421 | /* We would wait indefinitely if periodic sampling is not enabled */ | |
422 | if (!dev_priv->perf.oa.periodic) | |
423 | return -EIO; | |
424 | ||
425 | /* Note: the oa_buffer_is_empty() condition is ok to run unlocked as it | |
426 | * just performs mmio reads of the OA buffer head + tail pointers and | |
427 | * it's assumed we're handling some operation that implies the stream | |
428 | * can't be destroyed until completion (such as a read()) that ensures | |
429 | * the device + OA buffer can't disappear | |
430 | */ | |
431 | return wait_event_interruptible(dev_priv->perf.oa.poll_wq, | |
432 | !dev_priv->perf.oa.ops.oa_buffer_is_empty(dev_priv)); | |
433 | } | |
434 | ||
435 | static void i915_oa_poll_wait(struct i915_perf_stream *stream, | |
436 | struct file *file, | |
437 | poll_table *wait) | |
438 | { | |
439 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
440 | ||
441 | poll_wait(file, &dev_priv->perf.oa.poll_wq, wait); | |
442 | } | |
443 | ||
444 | static int i915_oa_read(struct i915_perf_stream *stream, | |
445 | char __user *buf, | |
446 | size_t count, | |
447 | size_t *offset) | |
448 | { | |
449 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
450 | ||
451 | return dev_priv->perf.oa.ops.read(stream, buf, count, offset); | |
452 | } | |
453 | ||
454 | /* Determine the render context hw id, and ensure it remains fixed for the | |
455 | * lifetime of the stream. This ensures that we don't have to worry about | |
456 | * updating the context ID in OACONTROL on the fly. | |
457 | */ | |
458 | static int oa_get_render_ctx_id(struct i915_perf_stream *stream) | |
459 | { | |
460 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
461 | struct i915_vma *vma; | |
462 | int ret; | |
463 | ||
464 | ret = i915_mutex_lock_interruptible(&dev_priv->drm); | |
465 | if (ret) | |
466 | return ret; | |
467 | ||
468 | /* As the ID is the gtt offset of the context's vma we pin | |
469 | * the vma to ensure the ID remains fixed. | |
470 | * | |
471 | * NB: implied RCS engine... | |
472 | */ | |
473 | vma = i915_gem_context_pin_legacy(stream->ctx, 0); | |
474 | if (IS_ERR(vma)) { | |
475 | ret = PTR_ERR(vma); | |
476 | goto unlock; | |
477 | } | |
478 | ||
479 | dev_priv->perf.oa.pinned_rcs_vma = vma; | |
480 | ||
481 | /* Explicitly track the ID (instead of calling i915_ggtt_offset() | |
482 | * on the fly) considering the difference with gen8+ and | |
483 | * execlists | |
484 | */ | |
485 | dev_priv->perf.oa.specific_ctx_id = i915_ggtt_offset(vma); | |
486 | ||
487 | unlock: | |
488 | mutex_unlock(&dev_priv->drm.struct_mutex); | |
489 | ||
490 | return ret; | |
491 | } | |
492 | ||
493 | static void oa_put_render_ctx_id(struct i915_perf_stream *stream) | |
494 | { | |
495 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
496 | ||
497 | mutex_lock(&dev_priv->drm.struct_mutex); | |
498 | ||
499 | i915_vma_unpin(dev_priv->perf.oa.pinned_rcs_vma); | |
500 | dev_priv->perf.oa.pinned_rcs_vma = NULL; | |
501 | ||
502 | dev_priv->perf.oa.specific_ctx_id = INVALID_CTX_ID; | |
503 | ||
504 | mutex_unlock(&dev_priv->drm.struct_mutex); | |
505 | } | |
506 | ||
507 | static void | |
508 | free_oa_buffer(struct drm_i915_private *i915) | |
509 | { | |
510 | mutex_lock(&i915->drm.struct_mutex); | |
511 | ||
512 | i915_gem_object_unpin_map(i915->perf.oa.oa_buffer.vma->obj); | |
513 | i915_vma_unpin(i915->perf.oa.oa_buffer.vma); | |
514 | i915_gem_object_put(i915->perf.oa.oa_buffer.vma->obj); | |
515 | ||
516 | i915->perf.oa.oa_buffer.vma = NULL; | |
517 | i915->perf.oa.oa_buffer.vaddr = NULL; | |
518 | ||
519 | mutex_unlock(&i915->drm.struct_mutex); | |
520 | } | |
521 | ||
522 | static void i915_oa_stream_destroy(struct i915_perf_stream *stream) | |
523 | { | |
524 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
525 | ||
526 | BUG_ON(stream != dev_priv->perf.oa.exclusive_stream); | |
527 | ||
528 | dev_priv->perf.oa.ops.disable_metric_set(dev_priv); | |
529 | ||
530 | free_oa_buffer(dev_priv); | |
531 | ||
532 | intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); | |
533 | intel_runtime_pm_put(dev_priv); | |
534 | ||
535 | if (stream->ctx) | |
536 | oa_put_render_ctx_id(stream); | |
537 | ||
538 | dev_priv->perf.oa.exclusive_stream = NULL; | |
539 | } | |
540 | ||
541 | static void gen7_init_oa_buffer(struct drm_i915_private *dev_priv) | |
542 | { | |
543 | u32 gtt_offset = i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma); | |
544 | ||
545 | /* Pre-DevBDW: OABUFFER must be set with counters off, | |
546 | * before OASTATUS1, but after OASTATUS2 | |
547 | */ | |
548 | I915_WRITE(GEN7_OASTATUS2, gtt_offset | OA_MEM_SELECT_GGTT); /* head */ | |
549 | I915_WRITE(GEN7_OABUFFER, gtt_offset); | |
550 | I915_WRITE(GEN7_OASTATUS1, gtt_offset | OABUFFER_SIZE_16M); /* tail */ | |
551 | ||
552 | /* On Haswell we have to track which OASTATUS1 flags we've | |
553 | * already seen since they can't be cleared while periodic | |
554 | * sampling is enabled. | |
555 | */ | |
556 | dev_priv->perf.oa.gen7_latched_oastatus1 = 0; | |
557 | ||
558 | /* NB: although the OA buffer will initially be allocated | |
559 | * zeroed via shmfs (and so this memset is redundant when | |
560 | * first allocating), we may re-init the OA buffer, either | |
561 | * when re-enabling a stream or in error/reset paths. | |
562 | * | |
563 | * The reason we clear the buffer for each re-init is for the | |
564 | * sanity check in gen7_append_oa_reports() that looks at the | |
565 | * report-id field to make sure it's non-zero which relies on | |
566 | * the assumption that new reports are being written to zeroed | |
567 | * memory... | |
568 | */ | |
569 | memset(dev_priv->perf.oa.oa_buffer.vaddr, 0, OA_BUFFER_SIZE); | |
570 | ||
571 | /* Maybe make ->pollin per-stream state if we support multiple | |
572 | * concurrent streams in the future. | |
573 | */ | |
574 | dev_priv->perf.oa.pollin = false; | |
575 | } | |
576 | ||
577 | static int alloc_oa_buffer(struct drm_i915_private *dev_priv) | |
578 | { | |
579 | struct drm_i915_gem_object *bo; | |
580 | struct i915_vma *vma; | |
581 | int ret; | |
582 | ||
583 | if (WARN_ON(dev_priv->perf.oa.oa_buffer.vma)) | |
584 | return -ENODEV; | |
585 | ||
586 | ret = i915_mutex_lock_interruptible(&dev_priv->drm); | |
587 | if (ret) | |
588 | return ret; | |
589 | ||
590 | BUILD_BUG_ON_NOT_POWER_OF_2(OA_BUFFER_SIZE); | |
591 | BUILD_BUG_ON(OA_BUFFER_SIZE < SZ_128K || OA_BUFFER_SIZE > SZ_16M); | |
592 | ||
593 | bo = i915_gem_object_create(&dev_priv->drm, OA_BUFFER_SIZE); | |
594 | if (IS_ERR(bo)) { | |
595 | DRM_ERROR("Failed to allocate OA buffer\n"); | |
596 | ret = PTR_ERR(bo); | |
597 | goto unlock; | |
598 | } | |
599 | ||
600 | ret = i915_gem_object_set_cache_level(bo, I915_CACHE_LLC); | |
601 | if (ret) | |
602 | goto err_unref; | |
603 | ||
604 | /* PreHSW required 512K alignment, HSW requires 16M */ | |
605 | vma = i915_gem_object_ggtt_pin(bo, NULL, 0, SZ_16M, 0); | |
606 | if (IS_ERR(vma)) { | |
607 | ret = PTR_ERR(vma); | |
608 | goto err_unref; | |
609 | } | |
610 | dev_priv->perf.oa.oa_buffer.vma = vma; | |
611 | ||
612 | dev_priv->perf.oa.oa_buffer.vaddr = | |
613 | i915_gem_object_pin_map(bo, I915_MAP_WB); | |
614 | if (IS_ERR(dev_priv->perf.oa.oa_buffer.vaddr)) { | |
615 | ret = PTR_ERR(dev_priv->perf.oa.oa_buffer.vaddr); | |
616 | goto err_unpin; | |
617 | } | |
618 | ||
619 | dev_priv->perf.oa.ops.init_oa_buffer(dev_priv); | |
620 | ||
621 | DRM_DEBUG_DRIVER("OA Buffer initialized, gtt offset = 0x%x, vaddr = %p\n", | |
622 | i915_ggtt_offset(dev_priv->perf.oa.oa_buffer.vma), | |
623 | dev_priv->perf.oa.oa_buffer.vaddr); | |
624 | ||
625 | goto unlock; | |
626 | ||
627 | err_unpin: | |
628 | __i915_vma_unpin(vma); | |
629 | ||
630 | err_unref: | |
631 | i915_gem_object_put(bo); | |
632 | ||
633 | dev_priv->perf.oa.oa_buffer.vaddr = NULL; | |
634 | dev_priv->perf.oa.oa_buffer.vma = NULL; | |
635 | ||
636 | unlock: | |
637 | mutex_unlock(&dev_priv->drm.struct_mutex); | |
638 | return ret; | |
639 | } | |
640 | ||
641 | static void config_oa_regs(struct drm_i915_private *dev_priv, | |
642 | const struct i915_oa_reg *regs, | |
643 | int n_regs) | |
644 | { | |
645 | int i; | |
646 | ||
647 | for (i = 0; i < n_regs; i++) { | |
648 | const struct i915_oa_reg *reg = regs + i; | |
649 | ||
650 | I915_WRITE(reg->addr, reg->value); | |
651 | } | |
652 | } | |
653 | ||
654 | static int hsw_enable_metric_set(struct drm_i915_private *dev_priv) | |
655 | { | |
656 | int ret = i915_oa_select_metric_set_hsw(dev_priv); | |
657 | ||
658 | if (ret) | |
659 | return ret; | |
660 | ||
661 | I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) | | |
662 | GT_NOA_ENABLE)); | |
663 | ||
664 | /* PRM: | |
665 | * | |
666 | * OA unit is using “crclk” for its functionality. When trunk | |
667 | * level clock gating takes place, OA clock would be gated, | |
668 | * unable to count the events from non-render clock domain. | |
669 | * Render clock gating must be disabled when OA is enabled to | |
670 | * count the events from non-render domain. Unit level clock | |
671 | * gating for RCS should also be disabled. | |
672 | */ | |
673 | I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) & | |
674 | ~GEN7_DOP_CLOCK_GATE_ENABLE)); | |
675 | I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) | | |
676 | GEN6_CSUNIT_CLOCK_GATE_DISABLE)); | |
677 | ||
678 | config_oa_regs(dev_priv, dev_priv->perf.oa.mux_regs, | |
679 | dev_priv->perf.oa.mux_regs_len); | |
680 | ||
681 | /* It apparently takes a fairly long time for a new MUX | |
682 | * configuration to be be applied after these register writes. | |
683 | * This delay duration was derived empirically based on the | |
684 | * render_basic config but hopefully it covers the maximum | |
685 | * configuration latency. | |
686 | * | |
687 | * As a fallback, the checks in _append_oa_reports() to skip | |
688 | * invalid OA reports do also seem to work to discard reports | |
689 | * generated before this config has completed - albeit not | |
690 | * silently. | |
691 | * | |
692 | * Unfortunately this is essentially a magic number, since we | |
693 | * don't currently know of a reliable mechanism for predicting | |
694 | * how long the MUX config will take to apply and besides | |
695 | * seeing invalid reports we don't know of a reliable way to | |
696 | * explicitly check that the MUX config has landed. | |
697 | * | |
698 | * It's even possible we've miss characterized the underlying | |
699 | * problem - it just seems like the simplest explanation why | |
700 | * a delay at this location would mitigate any invalid reports. | |
701 | */ | |
702 | usleep_range(15000, 20000); | |
703 | ||
704 | config_oa_regs(dev_priv, dev_priv->perf.oa.b_counter_regs, | |
705 | dev_priv->perf.oa.b_counter_regs_len); | |
706 | ||
707 | return 0; | |
708 | } | |
709 | ||
710 | static void hsw_disable_metric_set(struct drm_i915_private *dev_priv) | |
711 | { | |
712 | I915_WRITE(GEN6_UCGCTL1, (I915_READ(GEN6_UCGCTL1) & | |
713 | ~GEN6_CSUNIT_CLOCK_GATE_DISABLE)); | |
714 | I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) | | |
715 | GEN7_DOP_CLOCK_GATE_ENABLE)); | |
716 | ||
717 | I915_WRITE(GDT_CHICKEN_BITS, (I915_READ(GDT_CHICKEN_BITS) & | |
718 | ~GT_NOA_ENABLE)); | |
719 | } | |
720 | ||
721 | static void gen7_update_oacontrol_locked(struct drm_i915_private *dev_priv) | |
722 | { | |
723 | assert_spin_locked(&dev_priv->perf.hook_lock); | |
724 | ||
725 | if (dev_priv->perf.oa.exclusive_stream->enabled) { | |
726 | struct i915_gem_context *ctx = | |
727 | dev_priv->perf.oa.exclusive_stream->ctx; | |
728 | u32 ctx_id = dev_priv->perf.oa.specific_ctx_id; | |
729 | ||
730 | bool periodic = dev_priv->perf.oa.periodic; | |
731 | u32 period_exponent = dev_priv->perf.oa.period_exponent; | |
732 | u32 report_format = dev_priv->perf.oa.oa_buffer.format; | |
733 | ||
734 | I915_WRITE(GEN7_OACONTROL, | |
735 | (ctx_id & GEN7_OACONTROL_CTX_MASK) | | |
736 | (period_exponent << | |
737 | GEN7_OACONTROL_TIMER_PERIOD_SHIFT) | | |
738 | (periodic ? GEN7_OACONTROL_TIMER_ENABLE : 0) | | |
739 | (report_format << GEN7_OACONTROL_FORMAT_SHIFT) | | |
740 | (ctx ? GEN7_OACONTROL_PER_CTX_ENABLE : 0) | | |
741 | GEN7_OACONTROL_ENABLE); | |
742 | } else | |
743 | I915_WRITE(GEN7_OACONTROL, 0); | |
744 | } | |
745 | ||
746 | static void gen7_oa_enable(struct drm_i915_private *dev_priv) | |
747 | { | |
748 | unsigned long flags; | |
749 | ||
750 | /* Reset buf pointers so we don't forward reports from before now. | |
751 | * | |
752 | * Think carefully if considering trying to avoid this, since it | |
753 | * also ensures status flags and the buffer itself are cleared | |
754 | * in error paths, and we have checks for invalid reports based | |
755 | * on the assumption that certain fields are written to zeroed | |
756 | * memory which this helps maintains. | |
757 | */ | |
758 | gen7_init_oa_buffer(dev_priv); | |
759 | ||
760 | spin_lock_irqsave(&dev_priv->perf.hook_lock, flags); | |
761 | gen7_update_oacontrol_locked(dev_priv); | |
762 | spin_unlock_irqrestore(&dev_priv->perf.hook_lock, flags); | |
763 | } | |
764 | ||
765 | static void i915_oa_stream_enable(struct i915_perf_stream *stream) | |
766 | { | |
767 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
768 | ||
769 | dev_priv->perf.oa.ops.oa_enable(dev_priv); | |
770 | ||
771 | if (dev_priv->perf.oa.periodic) | |
772 | hrtimer_start(&dev_priv->perf.oa.poll_check_timer, | |
773 | ns_to_ktime(POLL_PERIOD), | |
774 | HRTIMER_MODE_REL_PINNED); | |
775 | } | |
776 | ||
777 | static void gen7_oa_disable(struct drm_i915_private *dev_priv) | |
778 | { | |
779 | I915_WRITE(GEN7_OACONTROL, 0); | |
780 | } | |
781 | ||
782 | static void i915_oa_stream_disable(struct i915_perf_stream *stream) | |
783 | { | |
784 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
785 | ||
786 | dev_priv->perf.oa.ops.oa_disable(dev_priv); | |
787 | ||
788 | if (dev_priv->perf.oa.periodic) | |
789 | hrtimer_cancel(&dev_priv->perf.oa.poll_check_timer); | |
790 | } | |
791 | ||
792 | static u64 oa_exponent_to_ns(struct drm_i915_private *dev_priv, int exponent) | |
793 | { | |
794 | return 1000000000ULL * (2ULL << exponent) / | |
795 | dev_priv->perf.oa.timestamp_frequency; | |
796 | } | |
797 | ||
798 | static const struct i915_perf_stream_ops i915_oa_stream_ops = { | |
799 | .destroy = i915_oa_stream_destroy, | |
800 | .enable = i915_oa_stream_enable, | |
801 | .disable = i915_oa_stream_disable, | |
802 | .wait_unlocked = i915_oa_wait_unlocked, | |
803 | .poll_wait = i915_oa_poll_wait, | |
804 | .read = i915_oa_read, | |
eec688e1 RB |
805 | }; |
806 | ||
d7965152 RB |
807 | static int i915_oa_stream_init(struct i915_perf_stream *stream, |
808 | struct drm_i915_perf_open_param *param, | |
809 | struct perf_open_properties *props) | |
810 | { | |
811 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
812 | int format_size; | |
813 | int ret; | |
814 | ||
442b8c06 RB |
815 | /* If the sysfs metrics/ directory wasn't registered for some |
816 | * reason then don't let userspace try their luck with config | |
817 | * IDs | |
818 | */ | |
819 | if (!dev_priv->perf.metrics_kobj) { | |
820 | DRM_ERROR("OA metrics weren't advertised via sysfs\n"); | |
821 | return -EINVAL; | |
822 | } | |
823 | ||
d7965152 RB |
824 | if (!(props->sample_flags & SAMPLE_OA_REPORT)) { |
825 | DRM_ERROR("Only OA report sampling supported\n"); | |
826 | return -EINVAL; | |
827 | } | |
828 | ||
829 | if (!dev_priv->perf.oa.ops.init_oa_buffer) { | |
830 | DRM_ERROR("OA unit not supported\n"); | |
831 | return -ENODEV; | |
832 | } | |
833 | ||
834 | /* To avoid the complexity of having to accurately filter | |
835 | * counter reports and marshal to the appropriate client | |
836 | * we currently only allow exclusive access | |
837 | */ | |
838 | if (dev_priv->perf.oa.exclusive_stream) { | |
839 | DRM_ERROR("OA unit already in use\n"); | |
840 | return -EBUSY; | |
841 | } | |
842 | ||
843 | if (!props->metrics_set) { | |
844 | DRM_ERROR("OA metric set not specified\n"); | |
845 | return -EINVAL; | |
846 | } | |
847 | ||
848 | if (!props->oa_format) { | |
849 | DRM_ERROR("OA report format not specified\n"); | |
850 | return -EINVAL; | |
851 | } | |
852 | ||
853 | stream->sample_size = sizeof(struct drm_i915_perf_record_header); | |
854 | ||
855 | format_size = dev_priv->perf.oa.oa_formats[props->oa_format].size; | |
856 | ||
857 | stream->sample_flags |= SAMPLE_OA_REPORT; | |
858 | stream->sample_size += format_size; | |
859 | ||
860 | dev_priv->perf.oa.oa_buffer.format_size = format_size; | |
861 | if (WARN_ON(dev_priv->perf.oa.oa_buffer.format_size == 0)) | |
862 | return -EINVAL; | |
863 | ||
864 | dev_priv->perf.oa.oa_buffer.format = | |
865 | dev_priv->perf.oa.oa_formats[props->oa_format].format; | |
866 | ||
867 | dev_priv->perf.oa.metrics_set = props->metrics_set; | |
868 | ||
869 | dev_priv->perf.oa.periodic = props->oa_periodic; | |
870 | if (dev_priv->perf.oa.periodic) { | |
871 | u64 period_ns = oa_exponent_to_ns(dev_priv, | |
872 | props->oa_period_exponent); | |
873 | ||
874 | dev_priv->perf.oa.period_exponent = props->oa_period_exponent; | |
875 | ||
876 | /* See comment for OA_TAIL_MARGIN_NSEC for details | |
877 | * about this tail_margin... | |
878 | */ | |
879 | dev_priv->perf.oa.tail_margin = | |
880 | ((OA_TAIL_MARGIN_NSEC / period_ns) + 1) * format_size; | |
881 | } | |
882 | ||
883 | if (stream->ctx) { | |
884 | ret = oa_get_render_ctx_id(stream); | |
885 | if (ret) | |
886 | return ret; | |
887 | } | |
888 | ||
889 | ret = alloc_oa_buffer(dev_priv); | |
890 | if (ret) | |
891 | goto err_oa_buf_alloc; | |
892 | ||
893 | /* PRM - observability performance counters: | |
894 | * | |
895 | * OACONTROL, performance counter enable, note: | |
896 | * | |
897 | * "When this bit is set, in order to have coherent counts, | |
898 | * RC6 power state and trunk clock gating must be disabled. | |
899 | * This can be achieved by programming MMIO registers as | |
900 | * 0xA094=0 and 0xA090[31]=1" | |
901 | * | |
902 | * In our case we are expecting that taking pm + FORCEWAKE | |
903 | * references will effectively disable RC6. | |
904 | */ | |
905 | intel_runtime_pm_get(dev_priv); | |
906 | intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL); | |
907 | ||
908 | ret = dev_priv->perf.oa.ops.enable_metric_set(dev_priv); | |
909 | if (ret) | |
910 | goto err_enable; | |
911 | ||
912 | stream->ops = &i915_oa_stream_ops; | |
913 | ||
914 | dev_priv->perf.oa.exclusive_stream = stream; | |
915 | ||
916 | return 0; | |
917 | ||
918 | err_enable: | |
919 | intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL); | |
920 | intel_runtime_pm_put(dev_priv); | |
921 | free_oa_buffer(dev_priv); | |
922 | ||
923 | err_oa_buf_alloc: | |
924 | if (stream->ctx) | |
925 | oa_put_render_ctx_id(stream); | |
926 | ||
927 | return ret; | |
928 | } | |
929 | ||
eec688e1 RB |
930 | static ssize_t i915_perf_read_locked(struct i915_perf_stream *stream, |
931 | struct file *file, | |
932 | char __user *buf, | |
933 | size_t count, | |
934 | loff_t *ppos) | |
935 | { | |
936 | /* Note we keep the offset (aka bytes read) separate from any | |
937 | * error status so that the final check for whether we return | |
938 | * the bytes read with a higher precedence than any error (see | |
939 | * comment below) doesn't need to be handled/duplicated in | |
940 | * stream->ops->read() implementations. | |
941 | */ | |
942 | size_t offset = 0; | |
943 | int ret = stream->ops->read(stream, buf, count, &offset); | |
944 | ||
945 | /* If we've successfully copied any data then reporting that | |
946 | * takes precedence over any internal error status, so the | |
947 | * data isn't lost. | |
948 | * | |
949 | * For example ret will be -ENOSPC whenever there is more | |
950 | * buffered data than can be copied to userspace, but that's | |
951 | * only interesting if we weren't able to copy some data | |
952 | * because it implies the userspace buffer is too small to | |
953 | * receive a single record (and we never split records). | |
954 | * | |
955 | * Another case with ret == -EFAULT is more of a grey area | |
956 | * since it would seem like bad form for userspace to ask us | |
957 | * to overrun its buffer, but the user knows best: | |
958 | * | |
959 | * http://yarchive.net/comp/linux/partial_reads_writes.html | |
960 | */ | |
961 | return offset ?: (ret ?: -EAGAIN); | |
962 | } | |
963 | ||
964 | static ssize_t i915_perf_read(struct file *file, | |
965 | char __user *buf, | |
966 | size_t count, | |
967 | loff_t *ppos) | |
968 | { | |
969 | struct i915_perf_stream *stream = file->private_data; | |
970 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
971 | ssize_t ret; | |
972 | ||
d7965152 RB |
973 | /* To ensure it's handled consistently we simply treat all reads of a |
974 | * disabled stream as an error. In particular it might otherwise lead | |
975 | * to a deadlock for blocking file descriptors... | |
976 | */ | |
977 | if (!stream->enabled) | |
978 | return -EIO; | |
979 | ||
eec688e1 | 980 | if (!(file->f_flags & O_NONBLOCK)) { |
d7965152 RB |
981 | /* There's the small chance of false positives from |
982 | * stream->ops->wait_unlocked. | |
983 | * | |
984 | * E.g. with single context filtering since we only wait until | |
985 | * oabuffer has >= 1 report we don't immediately know whether | |
986 | * any reports really belong to the current context | |
eec688e1 RB |
987 | */ |
988 | do { | |
989 | ret = stream->ops->wait_unlocked(stream); | |
990 | if (ret) | |
991 | return ret; | |
992 | ||
993 | mutex_lock(&dev_priv->perf.lock); | |
994 | ret = i915_perf_read_locked(stream, file, | |
995 | buf, count, ppos); | |
996 | mutex_unlock(&dev_priv->perf.lock); | |
997 | } while (ret == -EAGAIN); | |
998 | } else { | |
999 | mutex_lock(&dev_priv->perf.lock); | |
1000 | ret = i915_perf_read_locked(stream, file, buf, count, ppos); | |
1001 | mutex_unlock(&dev_priv->perf.lock); | |
1002 | } | |
1003 | ||
d7965152 RB |
1004 | if (ret >= 0) { |
1005 | /* Maybe make ->pollin per-stream state if we support multiple | |
1006 | * concurrent streams in the future. | |
1007 | */ | |
1008 | dev_priv->perf.oa.pollin = false; | |
1009 | } | |
1010 | ||
eec688e1 RB |
1011 | return ret; |
1012 | } | |
1013 | ||
d7965152 RB |
1014 | static enum hrtimer_restart oa_poll_check_timer_cb(struct hrtimer *hrtimer) |
1015 | { | |
1016 | struct drm_i915_private *dev_priv = | |
1017 | container_of(hrtimer, typeof(*dev_priv), | |
1018 | perf.oa.poll_check_timer); | |
1019 | ||
1020 | if (!dev_priv->perf.oa.ops.oa_buffer_is_empty(dev_priv)) { | |
1021 | dev_priv->perf.oa.pollin = true; | |
1022 | wake_up(&dev_priv->perf.oa.poll_wq); | |
1023 | } | |
1024 | ||
1025 | hrtimer_forward_now(hrtimer, ns_to_ktime(POLL_PERIOD)); | |
1026 | ||
1027 | return HRTIMER_RESTART; | |
1028 | } | |
1029 | ||
1030 | static unsigned int i915_perf_poll_locked(struct drm_i915_private *dev_priv, | |
1031 | struct i915_perf_stream *stream, | |
eec688e1 RB |
1032 | struct file *file, |
1033 | poll_table *wait) | |
1034 | { | |
d7965152 | 1035 | unsigned int events = 0; |
eec688e1 RB |
1036 | |
1037 | stream->ops->poll_wait(stream, file, wait); | |
1038 | ||
d7965152 RB |
1039 | /* Note: we don't explicitly check whether there's something to read |
1040 | * here since this path may be very hot depending on what else | |
1041 | * userspace is polling, or on the timeout in use. We rely solely on | |
1042 | * the hrtimer/oa_poll_check_timer_cb to notify us when there are | |
1043 | * samples to read. | |
1044 | */ | |
1045 | if (dev_priv->perf.oa.pollin) | |
1046 | events |= POLLIN; | |
eec688e1 | 1047 | |
d7965152 | 1048 | return events; |
eec688e1 RB |
1049 | } |
1050 | ||
1051 | static unsigned int i915_perf_poll(struct file *file, poll_table *wait) | |
1052 | { | |
1053 | struct i915_perf_stream *stream = file->private_data; | |
1054 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
1055 | int ret; | |
1056 | ||
1057 | mutex_lock(&dev_priv->perf.lock); | |
d7965152 | 1058 | ret = i915_perf_poll_locked(dev_priv, stream, file, wait); |
eec688e1 RB |
1059 | mutex_unlock(&dev_priv->perf.lock); |
1060 | ||
1061 | return ret; | |
1062 | } | |
1063 | ||
1064 | static void i915_perf_enable_locked(struct i915_perf_stream *stream) | |
1065 | { | |
1066 | if (stream->enabled) | |
1067 | return; | |
1068 | ||
1069 | /* Allow stream->ops->enable() to refer to this */ | |
1070 | stream->enabled = true; | |
1071 | ||
1072 | if (stream->ops->enable) | |
1073 | stream->ops->enable(stream); | |
1074 | } | |
1075 | ||
1076 | static void i915_perf_disable_locked(struct i915_perf_stream *stream) | |
1077 | { | |
1078 | if (!stream->enabled) | |
1079 | return; | |
1080 | ||
1081 | /* Allow stream->ops->disable() to refer to this */ | |
1082 | stream->enabled = false; | |
1083 | ||
1084 | if (stream->ops->disable) | |
1085 | stream->ops->disable(stream); | |
1086 | } | |
1087 | ||
1088 | static long i915_perf_ioctl_locked(struct i915_perf_stream *stream, | |
1089 | unsigned int cmd, | |
1090 | unsigned long arg) | |
1091 | { | |
1092 | switch (cmd) { | |
1093 | case I915_PERF_IOCTL_ENABLE: | |
1094 | i915_perf_enable_locked(stream); | |
1095 | return 0; | |
1096 | case I915_PERF_IOCTL_DISABLE: | |
1097 | i915_perf_disable_locked(stream); | |
1098 | return 0; | |
1099 | } | |
1100 | ||
1101 | return -EINVAL; | |
1102 | } | |
1103 | ||
1104 | static long i915_perf_ioctl(struct file *file, | |
1105 | unsigned int cmd, | |
1106 | unsigned long arg) | |
1107 | { | |
1108 | struct i915_perf_stream *stream = file->private_data; | |
1109 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
1110 | long ret; | |
1111 | ||
1112 | mutex_lock(&dev_priv->perf.lock); | |
1113 | ret = i915_perf_ioctl_locked(stream, cmd, arg); | |
1114 | mutex_unlock(&dev_priv->perf.lock); | |
1115 | ||
1116 | return ret; | |
1117 | } | |
1118 | ||
1119 | static void i915_perf_destroy_locked(struct i915_perf_stream *stream) | |
1120 | { | |
1121 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
1122 | ||
1123 | if (stream->enabled) | |
1124 | i915_perf_disable_locked(stream); | |
1125 | ||
1126 | if (stream->ops->destroy) | |
1127 | stream->ops->destroy(stream); | |
1128 | ||
1129 | list_del(&stream->link); | |
1130 | ||
1131 | if (stream->ctx) { | |
1132 | mutex_lock(&dev_priv->drm.struct_mutex); | |
1133 | i915_gem_context_put(stream->ctx); | |
1134 | mutex_unlock(&dev_priv->drm.struct_mutex); | |
1135 | } | |
1136 | ||
1137 | kfree(stream); | |
1138 | } | |
1139 | ||
1140 | static int i915_perf_release(struct inode *inode, struct file *file) | |
1141 | { | |
1142 | struct i915_perf_stream *stream = file->private_data; | |
1143 | struct drm_i915_private *dev_priv = stream->dev_priv; | |
1144 | ||
1145 | mutex_lock(&dev_priv->perf.lock); | |
1146 | i915_perf_destroy_locked(stream); | |
1147 | mutex_unlock(&dev_priv->perf.lock); | |
1148 | ||
1149 | return 0; | |
1150 | } | |
1151 | ||
1152 | ||
1153 | static const struct file_operations fops = { | |
1154 | .owner = THIS_MODULE, | |
1155 | .llseek = no_llseek, | |
1156 | .release = i915_perf_release, | |
1157 | .poll = i915_perf_poll, | |
1158 | .read = i915_perf_read, | |
1159 | .unlocked_ioctl = i915_perf_ioctl, | |
1160 | }; | |
1161 | ||
1162 | ||
1163 | static struct i915_gem_context * | |
1164 | lookup_context(struct drm_i915_private *dev_priv, | |
1165 | struct drm_i915_file_private *file_priv, | |
1166 | u32 ctx_user_handle) | |
1167 | { | |
1168 | struct i915_gem_context *ctx; | |
1169 | int ret; | |
1170 | ||
1171 | ret = i915_mutex_lock_interruptible(&dev_priv->drm); | |
1172 | if (ret) | |
1173 | return ERR_PTR(ret); | |
1174 | ||
1175 | ctx = i915_gem_context_lookup(file_priv, ctx_user_handle); | |
1176 | if (!IS_ERR(ctx)) | |
1177 | i915_gem_context_get(ctx); | |
1178 | ||
1179 | mutex_unlock(&dev_priv->drm.struct_mutex); | |
1180 | ||
1181 | return ctx; | |
1182 | } | |
1183 | ||
1184 | static int | |
1185 | i915_perf_open_ioctl_locked(struct drm_i915_private *dev_priv, | |
1186 | struct drm_i915_perf_open_param *param, | |
1187 | struct perf_open_properties *props, | |
1188 | struct drm_file *file) | |
1189 | { | |
1190 | struct i915_gem_context *specific_ctx = NULL; | |
1191 | struct i915_perf_stream *stream = NULL; | |
1192 | unsigned long f_flags = 0; | |
1193 | int stream_fd; | |
1194 | int ret; | |
1195 | ||
1196 | if (props->single_context) { | |
1197 | u32 ctx_handle = props->ctx_handle; | |
1198 | struct drm_i915_file_private *file_priv = file->driver_priv; | |
1199 | ||
1200 | specific_ctx = lookup_context(dev_priv, file_priv, ctx_handle); | |
1201 | if (IS_ERR(specific_ctx)) { | |
1202 | ret = PTR_ERR(specific_ctx); | |
1203 | if (ret != -EINTR) | |
1204 | DRM_ERROR("Failed to look up context with ID %u for opening perf stream\n", | |
1205 | ctx_handle); | |
1206 | goto err; | |
1207 | } | |
1208 | } | |
1209 | ||
1210 | if (!specific_ctx && !capable(CAP_SYS_ADMIN)) { | |
1211 | DRM_ERROR("Insufficient privileges to open system-wide i915 perf stream\n"); | |
1212 | ret = -EACCES; | |
1213 | goto err_ctx; | |
1214 | } | |
1215 | ||
1216 | stream = kzalloc(sizeof(*stream), GFP_KERNEL); | |
1217 | if (!stream) { | |
1218 | ret = -ENOMEM; | |
1219 | goto err_ctx; | |
1220 | } | |
1221 | ||
eec688e1 RB |
1222 | stream->dev_priv = dev_priv; |
1223 | stream->ctx = specific_ctx; | |
1224 | ||
d7965152 RB |
1225 | ret = i915_oa_stream_init(stream, param, props); |
1226 | if (ret) | |
1227 | goto err_alloc; | |
1228 | ||
1229 | /* we avoid simply assigning stream->sample_flags = props->sample_flags | |
1230 | * to have _stream_init check the combination of sample flags more | |
1231 | * thoroughly, but still this is the expected result at this point. | |
eec688e1 | 1232 | */ |
d7965152 RB |
1233 | if (WARN_ON(stream->sample_flags != props->sample_flags)) { |
1234 | ret = -ENODEV; | |
1235 | goto err_alloc; | |
1236 | } | |
eec688e1 RB |
1237 | |
1238 | list_add(&stream->link, &dev_priv->perf.streams); | |
1239 | ||
1240 | if (param->flags & I915_PERF_FLAG_FD_CLOEXEC) | |
1241 | f_flags |= O_CLOEXEC; | |
1242 | if (param->flags & I915_PERF_FLAG_FD_NONBLOCK) | |
1243 | f_flags |= O_NONBLOCK; | |
1244 | ||
1245 | stream_fd = anon_inode_getfd("[i915_perf]", &fops, stream, f_flags); | |
1246 | if (stream_fd < 0) { | |
1247 | ret = stream_fd; | |
1248 | goto err_open; | |
1249 | } | |
1250 | ||
1251 | if (!(param->flags & I915_PERF_FLAG_DISABLED)) | |
1252 | i915_perf_enable_locked(stream); | |
1253 | ||
1254 | return stream_fd; | |
1255 | ||
1256 | err_open: | |
1257 | list_del(&stream->link); | |
1258 | if (stream->ops->destroy) | |
1259 | stream->ops->destroy(stream); | |
1260 | err_alloc: | |
1261 | kfree(stream); | |
1262 | err_ctx: | |
1263 | if (specific_ctx) { | |
1264 | mutex_lock(&dev_priv->drm.struct_mutex); | |
1265 | i915_gem_context_put(specific_ctx); | |
1266 | mutex_unlock(&dev_priv->drm.struct_mutex); | |
1267 | } | |
1268 | err: | |
1269 | return ret; | |
1270 | } | |
1271 | ||
1272 | /* Note we copy the properties from userspace outside of the i915 perf | |
1273 | * mutex to avoid an awkward lockdep with mmap_sem. | |
1274 | * | |
1275 | * Note this function only validates properties in isolation it doesn't | |
1276 | * validate that the combination of properties makes sense or that all | |
1277 | * properties necessary for a particular kind of stream have been set. | |
1278 | */ | |
1279 | static int read_properties_unlocked(struct drm_i915_private *dev_priv, | |
1280 | u64 __user *uprops, | |
1281 | u32 n_props, | |
1282 | struct perf_open_properties *props) | |
1283 | { | |
1284 | u64 __user *uprop = uprops; | |
1285 | int i; | |
1286 | ||
1287 | memset(props, 0, sizeof(struct perf_open_properties)); | |
1288 | ||
1289 | if (!n_props) { | |
1290 | DRM_ERROR("No i915 perf properties given"); | |
1291 | return -EINVAL; | |
1292 | } | |
1293 | ||
1294 | /* Considering that ID = 0 is reserved and assuming that we don't | |
1295 | * (currently) expect any configurations to ever specify duplicate | |
1296 | * values for a particular property ID then the last _PROP_MAX value is | |
1297 | * one greater than the maximum number of properties we expect to get | |
1298 | * from userspace. | |
1299 | */ | |
1300 | if (n_props >= DRM_I915_PERF_PROP_MAX) { | |
1301 | DRM_ERROR("More i915 perf properties specified than exist"); | |
1302 | return -EINVAL; | |
1303 | } | |
1304 | ||
1305 | for (i = 0; i < n_props; i++) { | |
1306 | u64 id, value; | |
1307 | int ret; | |
1308 | ||
1309 | ret = get_user(id, uprop); | |
1310 | if (ret) | |
1311 | return ret; | |
1312 | ||
1313 | ret = get_user(value, uprop + 1); | |
1314 | if (ret) | |
1315 | return ret; | |
1316 | ||
1317 | switch ((enum drm_i915_perf_property_id)id) { | |
1318 | case DRM_I915_PERF_PROP_CTX_HANDLE: | |
1319 | props->single_context = 1; | |
1320 | props->ctx_handle = value; | |
1321 | break; | |
d7965152 RB |
1322 | case DRM_I915_PERF_PROP_SAMPLE_OA: |
1323 | props->sample_flags |= SAMPLE_OA_REPORT; | |
1324 | break; | |
1325 | case DRM_I915_PERF_PROP_OA_METRICS_SET: | |
1326 | if (value == 0 || | |
1327 | value > dev_priv->perf.oa.n_builtin_sets) { | |
1328 | DRM_ERROR("Unknown OA metric set ID"); | |
1329 | return -EINVAL; | |
1330 | } | |
1331 | props->metrics_set = value; | |
1332 | break; | |
1333 | case DRM_I915_PERF_PROP_OA_FORMAT: | |
1334 | if (value == 0 || value >= I915_OA_FORMAT_MAX) { | |
1335 | DRM_ERROR("Invalid OA report format\n"); | |
1336 | return -EINVAL; | |
1337 | } | |
1338 | if (!dev_priv->perf.oa.oa_formats[value].size) { | |
1339 | DRM_ERROR("Invalid OA report format\n"); | |
1340 | return -EINVAL; | |
1341 | } | |
1342 | props->oa_format = value; | |
1343 | break; | |
1344 | case DRM_I915_PERF_PROP_OA_EXPONENT: | |
1345 | if (value > OA_EXPONENT_MAX) { | |
1346 | DRM_ERROR("OA timer exponent too high (> %u)\n", | |
1347 | OA_EXPONENT_MAX); | |
1348 | return -EINVAL; | |
1349 | } | |
1350 | ||
1351 | /* NB: The exponent represents a period as follows: | |
1352 | * | |
1353 | * 80ns * 2^(period_exponent + 1) | |
1354 | * | |
1355 | * Theoretically we can program the OA unit to sample | |
1356 | * every 160ns but don't allow that by default unless | |
1357 | * root. | |
1358 | * | |
1359 | * Referring to perf's | |
1360 | * kernel.perf_event_max_sample_rate for a precedent | |
1361 | * (100000 by default); with an OA exponent of 6 we get | |
1362 | * a period of 10.240 microseconds -just under 100000Hz | |
1363 | */ | |
1364 | if (value < 6 && !capable(CAP_SYS_ADMIN)) { | |
1365 | DRM_ERROR("Minimum OA sampling exponent is 6 without root privileges\n"); | |
1366 | return -EACCES; | |
1367 | } | |
1368 | ||
1369 | props->oa_periodic = true; | |
1370 | props->oa_period_exponent = value; | |
1371 | break; | |
eec688e1 RB |
1372 | default: |
1373 | MISSING_CASE(id); | |
1374 | DRM_ERROR("Unknown i915 perf property ID"); | |
1375 | return -EINVAL; | |
1376 | } | |
1377 | ||
1378 | uprop += 2; | |
1379 | } | |
1380 | ||
1381 | return 0; | |
1382 | } | |
1383 | ||
1384 | int i915_perf_open_ioctl(struct drm_device *dev, void *data, | |
1385 | struct drm_file *file) | |
1386 | { | |
1387 | struct drm_i915_private *dev_priv = dev->dev_private; | |
1388 | struct drm_i915_perf_open_param *param = data; | |
1389 | struct perf_open_properties props; | |
1390 | u32 known_open_flags; | |
1391 | int ret; | |
1392 | ||
1393 | if (!dev_priv->perf.initialized) { | |
1394 | DRM_ERROR("i915 perf interface not available for this system"); | |
1395 | return -ENOTSUPP; | |
1396 | } | |
1397 | ||
1398 | known_open_flags = I915_PERF_FLAG_FD_CLOEXEC | | |
1399 | I915_PERF_FLAG_FD_NONBLOCK | | |
1400 | I915_PERF_FLAG_DISABLED; | |
1401 | if (param->flags & ~known_open_flags) { | |
1402 | DRM_ERROR("Unknown drm_i915_perf_open_param flag\n"); | |
1403 | return -EINVAL; | |
1404 | } | |
1405 | ||
1406 | ret = read_properties_unlocked(dev_priv, | |
1407 | u64_to_user_ptr(param->properties_ptr), | |
1408 | param->num_properties, | |
1409 | &props); | |
1410 | if (ret) | |
1411 | return ret; | |
1412 | ||
1413 | mutex_lock(&dev_priv->perf.lock); | |
1414 | ret = i915_perf_open_ioctl_locked(dev_priv, param, &props, file); | |
1415 | mutex_unlock(&dev_priv->perf.lock); | |
1416 | ||
1417 | return ret; | |
1418 | } | |
1419 | ||
442b8c06 RB |
1420 | void i915_perf_register(struct drm_i915_private *dev_priv) |
1421 | { | |
1422 | if (!IS_HASWELL(dev_priv)) | |
1423 | return; | |
1424 | ||
1425 | if (!dev_priv->perf.initialized) | |
1426 | return; | |
1427 | ||
1428 | /* To be sure we're synchronized with an attempted | |
1429 | * i915_perf_open_ioctl(); considering that we register after | |
1430 | * being exposed to userspace. | |
1431 | */ | |
1432 | mutex_lock(&dev_priv->perf.lock); | |
1433 | ||
1434 | dev_priv->perf.metrics_kobj = | |
1435 | kobject_create_and_add("metrics", | |
1436 | &dev_priv->drm.primary->kdev->kobj); | |
1437 | if (!dev_priv->perf.metrics_kobj) | |
1438 | goto exit; | |
1439 | ||
1440 | if (i915_perf_register_sysfs_hsw(dev_priv)) { | |
1441 | kobject_put(dev_priv->perf.metrics_kobj); | |
1442 | dev_priv->perf.metrics_kobj = NULL; | |
1443 | } | |
1444 | ||
1445 | exit: | |
1446 | mutex_unlock(&dev_priv->perf.lock); | |
1447 | } | |
1448 | ||
1449 | void i915_perf_unregister(struct drm_i915_private *dev_priv) | |
1450 | { | |
1451 | if (!IS_HASWELL(dev_priv)) | |
1452 | return; | |
1453 | ||
1454 | if (!dev_priv->perf.metrics_kobj) | |
1455 | return; | |
1456 | ||
1457 | i915_perf_unregister_sysfs_hsw(dev_priv); | |
1458 | ||
1459 | kobject_put(dev_priv->perf.metrics_kobj); | |
1460 | dev_priv->perf.metrics_kobj = NULL; | |
1461 | } | |
1462 | ||
eec688e1 RB |
1463 | void i915_perf_init(struct drm_i915_private *dev_priv) |
1464 | { | |
d7965152 RB |
1465 | if (!IS_HASWELL(dev_priv)) |
1466 | return; | |
1467 | ||
1468 | hrtimer_init(&dev_priv->perf.oa.poll_check_timer, | |
1469 | CLOCK_MONOTONIC, HRTIMER_MODE_REL); | |
1470 | dev_priv->perf.oa.poll_check_timer.function = oa_poll_check_timer_cb; | |
1471 | init_waitqueue_head(&dev_priv->perf.oa.poll_wq); | |
1472 | ||
eec688e1 RB |
1473 | INIT_LIST_HEAD(&dev_priv->perf.streams); |
1474 | mutex_init(&dev_priv->perf.lock); | |
d7965152 RB |
1475 | spin_lock_init(&dev_priv->perf.hook_lock); |
1476 | ||
1477 | dev_priv->perf.oa.ops.init_oa_buffer = gen7_init_oa_buffer; | |
1478 | dev_priv->perf.oa.ops.enable_metric_set = hsw_enable_metric_set; | |
1479 | dev_priv->perf.oa.ops.disable_metric_set = hsw_disable_metric_set; | |
1480 | dev_priv->perf.oa.ops.oa_enable = gen7_oa_enable; | |
1481 | dev_priv->perf.oa.ops.oa_disable = gen7_oa_disable; | |
1482 | dev_priv->perf.oa.ops.read = gen7_oa_read; | |
1483 | dev_priv->perf.oa.ops.oa_buffer_is_empty = | |
1484 | gen7_oa_buffer_is_empty_fop_unlocked; | |
1485 | ||
1486 | dev_priv->perf.oa.timestamp_frequency = 12500000; | |
1487 | ||
1488 | dev_priv->perf.oa.oa_formats = hsw_oa_formats; | |
1489 | ||
1490 | dev_priv->perf.oa.n_builtin_sets = | |
1491 | i915_oa_n_builtin_metric_sets_hsw; | |
eec688e1 RB |
1492 | |
1493 | dev_priv->perf.initialized = true; | |
1494 | } | |
1495 | ||
1496 | void i915_perf_fini(struct drm_i915_private *dev_priv) | |
1497 | { | |
1498 | if (!dev_priv->perf.initialized) | |
1499 | return; | |
1500 | ||
d7965152 | 1501 | memset(&dev_priv->perf.oa.ops, 0, sizeof(dev_priv->perf.oa.ops)); |
eec688e1 RB |
1502 | dev_priv->perf.initialized = false; |
1503 | } |