]>
Commit | Line | Data |
---|---|---|
8453d674 ZW |
1 | /* |
2 | * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
21 | * SOFTWARE. | |
22 | * | |
23 | * Authors: | |
24 | * Zhiyuan Lv <zhiyuan.lv@intel.com> | |
25 | * Zhi Wang <zhi.a.wang@intel.com> | |
26 | * | |
27 | * Contributors: | |
28 | * Min He <min.he@intel.com> | |
29 | * Bing Niu <bing.niu@intel.com> | |
30 | * Ping Gao <ping.a.gao@intel.com> | |
31 | * Tina Zhang <tina.zhang@intel.com> | |
32 | * | |
33 | */ | |
34 | ||
35 | #include "i915_drv.h" | |
feddf6e8 | 36 | #include "gvt.h" |
8453d674 ZW |
37 | |
38 | #define _EL_OFFSET_STATUS 0x234 | |
39 | #define _EL_OFFSET_STATUS_BUF 0x370 | |
40 | #define _EL_OFFSET_STATUS_PTR 0x3A0 | |
41 | ||
42 | #define execlist_ring_mmio(gvt, ring_id, offset) \ | |
1140f9ed | 43 | (gvt->dev_priv->engine[ring_id]->mmio_base + (offset)) |
8453d674 ZW |
44 | |
45 | #define valid_context(ctx) ((ctx)->valid) | |
46 | #define same_context(a, b) (((a)->context_id == (b)->context_id) && \ | |
47 | ((a)->lrca == (b)->lrca)) | |
48 | ||
49 | static int context_switch_events[] = { | |
50 | [RCS] = RCS_AS_CONTEXT_SWITCH, | |
51 | [BCS] = BCS_AS_CONTEXT_SWITCH, | |
52 | [VCS] = VCS_AS_CONTEXT_SWITCH, | |
53 | [VCS2] = VCS2_AS_CONTEXT_SWITCH, | |
54 | [VECS] = VECS_AS_CONTEXT_SWITCH, | |
55 | }; | |
56 | ||
57 | static int ring_id_to_context_switch_event(int ring_id) | |
58 | { | |
59 | if (WARN_ON(ring_id < RCS && ring_id > | |
60 | ARRAY_SIZE(context_switch_events))) | |
61 | return -EINVAL; | |
62 | ||
63 | return context_switch_events[ring_id]; | |
64 | } | |
65 | ||
66 | static void switch_virtual_execlist_slot(struct intel_vgpu_execlist *execlist) | |
67 | { | |
68 | gvt_dbg_el("[before] running slot %d/context %x pending slot %d\n", | |
69 | execlist->running_slot ? | |
70 | execlist->running_slot->index : -1, | |
71 | execlist->running_context ? | |
72 | execlist->running_context->context_id : 0, | |
73 | execlist->pending_slot ? | |
74 | execlist->pending_slot->index : -1); | |
75 | ||
76 | execlist->running_slot = execlist->pending_slot; | |
77 | execlist->pending_slot = NULL; | |
78 | execlist->running_context = execlist->running_context ? | |
79 | &execlist->running_slot->ctx[0] : NULL; | |
80 | ||
81 | gvt_dbg_el("[after] running slot %d/context %x pending slot %d\n", | |
82 | execlist->running_slot ? | |
83 | execlist->running_slot->index : -1, | |
84 | execlist->running_context ? | |
85 | execlist->running_context->context_id : 0, | |
86 | execlist->pending_slot ? | |
87 | execlist->pending_slot->index : -1); | |
88 | } | |
89 | ||
90 | static void emulate_execlist_status(struct intel_vgpu_execlist *execlist) | |
91 | { | |
92 | struct intel_vgpu_execlist_slot *running = execlist->running_slot; | |
93 | struct intel_vgpu_execlist_slot *pending = execlist->pending_slot; | |
94 | struct execlist_ctx_descriptor_format *desc = execlist->running_context; | |
95 | struct intel_vgpu *vgpu = execlist->vgpu; | |
96 | struct execlist_status_format status; | |
97 | int ring_id = execlist->ring_id; | |
98 | u32 status_reg = execlist_ring_mmio(vgpu->gvt, | |
99 | ring_id, _EL_OFFSET_STATUS); | |
100 | ||
101 | status.ldw = vgpu_vreg(vgpu, status_reg); | |
102 | status.udw = vgpu_vreg(vgpu, status_reg + 4); | |
103 | ||
104 | if (running) { | |
105 | status.current_execlist_pointer = !!running->index; | |
106 | status.execlist_write_pointer = !!!running->index; | |
107 | status.execlist_0_active = status.execlist_0_valid = | |
108 | !!!(running->index); | |
109 | status.execlist_1_active = status.execlist_1_valid = | |
110 | !!(running->index); | |
111 | } else { | |
112 | status.context_id = 0; | |
113 | status.execlist_0_active = status.execlist_0_valid = 0; | |
114 | status.execlist_1_active = status.execlist_1_valid = 0; | |
115 | } | |
116 | ||
117 | status.context_id = desc ? desc->context_id : 0; | |
118 | status.execlist_queue_full = !!(pending); | |
119 | ||
120 | vgpu_vreg(vgpu, status_reg) = status.ldw; | |
121 | vgpu_vreg(vgpu, status_reg + 4) = status.udw; | |
122 | ||
123 | gvt_dbg_el("vgpu%d: status reg offset %x ldw %x udw %x\n", | |
124 | vgpu->id, status_reg, status.ldw, status.udw); | |
125 | } | |
126 | ||
127 | static void emulate_csb_update(struct intel_vgpu_execlist *execlist, | |
128 | struct execlist_context_status_format *status, | |
129 | bool trigger_interrupt_later) | |
130 | { | |
131 | struct intel_vgpu *vgpu = execlist->vgpu; | |
132 | int ring_id = execlist->ring_id; | |
133 | struct execlist_context_status_pointer_format ctx_status_ptr; | |
134 | u32 write_pointer; | |
135 | u32 ctx_status_ptr_reg, ctx_status_buf_reg, offset; | |
136 | ||
137 | ctx_status_ptr_reg = execlist_ring_mmio(vgpu->gvt, ring_id, | |
138 | _EL_OFFSET_STATUS_PTR); | |
139 | ctx_status_buf_reg = execlist_ring_mmio(vgpu->gvt, ring_id, | |
140 | _EL_OFFSET_STATUS_BUF); | |
141 | ||
142 | ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg); | |
143 | ||
144 | write_pointer = ctx_status_ptr.write_ptr; | |
145 | ||
146 | if (write_pointer == 0x7) | |
147 | write_pointer = 0; | |
148 | else { | |
149 | ++write_pointer; | |
150 | write_pointer %= 0x6; | |
151 | } | |
152 | ||
153 | offset = ctx_status_buf_reg + write_pointer * 8; | |
154 | ||
155 | vgpu_vreg(vgpu, offset) = status->ldw; | |
156 | vgpu_vreg(vgpu, offset + 4) = status->udw; | |
157 | ||
158 | ctx_status_ptr.write_ptr = write_pointer; | |
159 | vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw; | |
160 | ||
161 | gvt_dbg_el("vgpu%d: w pointer %u reg %x csb l %x csb h %x\n", | |
162 | vgpu->id, write_pointer, offset, status->ldw, status->udw); | |
163 | ||
164 | if (trigger_interrupt_later) | |
165 | return; | |
166 | ||
167 | intel_vgpu_trigger_virtual_event(vgpu, | |
168 | ring_id_to_context_switch_event(execlist->ring_id)); | |
169 | } | |
170 | ||
28c4c6ca | 171 | static int emulate_execlist_ctx_schedule_out( |
8453d674 ZW |
172 | struct intel_vgpu_execlist *execlist, |
173 | struct execlist_ctx_descriptor_format *ctx) | |
174 | { | |
695fbc08 | 175 | struct intel_vgpu *vgpu = execlist->vgpu; |
8453d674 ZW |
176 | struct intel_vgpu_execlist_slot *running = execlist->running_slot; |
177 | struct intel_vgpu_execlist_slot *pending = execlist->pending_slot; | |
178 | struct execlist_ctx_descriptor_format *ctx0 = &running->ctx[0]; | |
179 | struct execlist_ctx_descriptor_format *ctx1 = &running->ctx[1]; | |
180 | struct execlist_context_status_format status; | |
181 | ||
182 | memset(&status, 0, sizeof(status)); | |
183 | ||
184 | gvt_dbg_el("schedule out context id %x\n", ctx->context_id); | |
185 | ||
186 | if (WARN_ON(!same_context(ctx, execlist->running_context))) { | |
695fbc08 | 187 | gvt_vgpu_err("schedule out context is not running context," |
8453d674 ZW |
188 | "ctx id %x running ctx id %x\n", |
189 | ctx->context_id, | |
190 | execlist->running_context->context_id); | |
191 | return -EINVAL; | |
192 | } | |
193 | ||
194 | /* ctx1 is valid, ctx0/ctx is scheduled-out -> element switch */ | |
195 | if (valid_context(ctx1) && same_context(ctx0, ctx)) { | |
196 | gvt_dbg_el("ctx 1 valid, ctx/ctx 0 is scheduled-out\n"); | |
197 | ||
198 | execlist->running_context = ctx1; | |
199 | ||
200 | emulate_execlist_status(execlist); | |
201 | ||
202 | status.context_complete = status.element_switch = 1; | |
203 | status.context_id = ctx->context_id; | |
204 | ||
205 | emulate_csb_update(execlist, &status, false); | |
206 | /* | |
207 | * ctx1 is not valid, ctx == ctx0 | |
208 | * ctx1 is valid, ctx1 == ctx | |
209 | * --> last element is finished | |
210 | * emulate: | |
211 | * active-to-idle if there is *no* pending execlist | |
212 | * context-complete if there *is* pending execlist | |
213 | */ | |
214 | } else if ((!valid_context(ctx1) && same_context(ctx0, ctx)) | |
215 | || (valid_context(ctx1) && same_context(ctx1, ctx))) { | |
216 | gvt_dbg_el("need to switch virtual execlist slot\n"); | |
217 | ||
218 | switch_virtual_execlist_slot(execlist); | |
219 | ||
220 | emulate_execlist_status(execlist); | |
221 | ||
222 | status.context_complete = status.active_to_idle = 1; | |
223 | status.context_id = ctx->context_id; | |
224 | ||
225 | if (!pending) { | |
226 | emulate_csb_update(execlist, &status, false); | |
227 | } else { | |
228 | emulate_csb_update(execlist, &status, true); | |
229 | ||
230 | memset(&status, 0, sizeof(status)); | |
231 | ||
232 | status.idle_to_active = 1; | |
233 | status.context_id = 0; | |
234 | ||
235 | emulate_csb_update(execlist, &status, false); | |
236 | } | |
237 | } else { | |
238 | WARN_ON(1); | |
239 | return -EINVAL; | |
240 | } | |
241 | ||
242 | return 0; | |
243 | } | |
244 | ||
245 | static struct intel_vgpu_execlist_slot *get_next_execlist_slot( | |
246 | struct intel_vgpu_execlist *execlist) | |
247 | { | |
248 | struct intel_vgpu *vgpu = execlist->vgpu; | |
249 | int ring_id = execlist->ring_id; | |
250 | u32 status_reg = execlist_ring_mmio(vgpu->gvt, ring_id, | |
251 | _EL_OFFSET_STATUS); | |
252 | struct execlist_status_format status; | |
253 | ||
254 | status.ldw = vgpu_vreg(vgpu, status_reg); | |
255 | status.udw = vgpu_vreg(vgpu, status_reg + 4); | |
256 | ||
257 | if (status.execlist_queue_full) { | |
695fbc08 | 258 | gvt_vgpu_err("virtual execlist slots are full\n"); |
8453d674 ZW |
259 | return NULL; |
260 | } | |
261 | ||
262 | return &execlist->slot[status.execlist_write_pointer]; | |
263 | } | |
264 | ||
28c4c6ca | 265 | static int emulate_execlist_schedule_in(struct intel_vgpu_execlist *execlist, |
8453d674 ZW |
266 | struct execlist_ctx_descriptor_format ctx[2]) |
267 | { | |
268 | struct intel_vgpu_execlist_slot *running = execlist->running_slot; | |
269 | struct intel_vgpu_execlist_slot *slot = | |
270 | get_next_execlist_slot(execlist); | |
271 | ||
272 | struct execlist_ctx_descriptor_format *ctx0, *ctx1; | |
273 | struct execlist_context_status_format status; | |
695fbc08 | 274 | struct intel_vgpu *vgpu = execlist->vgpu; |
8453d674 ZW |
275 | |
276 | gvt_dbg_el("emulate schedule-in\n"); | |
277 | ||
278 | if (!slot) { | |
695fbc08 | 279 | gvt_vgpu_err("no available execlist slot\n"); |
8453d674 ZW |
280 | return -EINVAL; |
281 | } | |
282 | ||
283 | memset(&status, 0, sizeof(status)); | |
284 | memset(slot->ctx, 0, sizeof(slot->ctx)); | |
285 | ||
286 | slot->ctx[0] = ctx[0]; | |
287 | slot->ctx[1] = ctx[1]; | |
288 | ||
289 | gvt_dbg_el("alloc slot index %d ctx 0 %x ctx 1 %x\n", | |
290 | slot->index, ctx[0].context_id, | |
291 | ctx[1].context_id); | |
292 | ||
293 | /* | |
294 | * no running execlist, make this write bundle as running execlist | |
295 | * -> idle-to-active | |
296 | */ | |
297 | if (!running) { | |
298 | gvt_dbg_el("no current running execlist\n"); | |
299 | ||
300 | execlist->running_slot = slot; | |
301 | execlist->pending_slot = NULL; | |
302 | execlist->running_context = &slot->ctx[0]; | |
303 | ||
304 | gvt_dbg_el("running slot index %d running context %x\n", | |
305 | execlist->running_slot->index, | |
306 | execlist->running_context->context_id); | |
307 | ||
308 | emulate_execlist_status(execlist); | |
309 | ||
310 | status.idle_to_active = 1; | |
311 | status.context_id = 0; | |
312 | ||
313 | emulate_csb_update(execlist, &status, false); | |
314 | return 0; | |
315 | } | |
316 | ||
317 | ctx0 = &running->ctx[0]; | |
318 | ctx1 = &running->ctx[1]; | |
319 | ||
320 | gvt_dbg_el("current running slot index %d ctx 0 %x ctx 1 %x\n", | |
321 | running->index, ctx0->context_id, ctx1->context_id); | |
322 | ||
323 | /* | |
324 | * already has an running execlist | |
325 | * a. running ctx1 is valid, | |
326 | * ctx0 is finished, and running ctx1 == new execlist ctx[0] | |
327 | * b. running ctx1 is not valid, | |
328 | * ctx0 == new execlist ctx[0] | |
329 | * ----> lite-restore + preempted | |
330 | */ | |
331 | if ((valid_context(ctx1) && same_context(ctx1, &slot->ctx[0]) && | |
332 | /* condition a */ | |
333 | (!same_context(ctx0, execlist->running_context))) || | |
334 | (!valid_context(ctx1) && | |
335 | same_context(ctx0, &slot->ctx[0]))) { /* condition b */ | |
336 | gvt_dbg_el("need to switch virtual execlist slot\n"); | |
337 | ||
338 | execlist->pending_slot = slot; | |
339 | switch_virtual_execlist_slot(execlist); | |
340 | ||
341 | emulate_execlist_status(execlist); | |
342 | ||
343 | status.lite_restore = status.preempted = 1; | |
344 | status.context_id = ctx[0].context_id; | |
345 | ||
346 | emulate_csb_update(execlist, &status, false); | |
347 | } else { | |
348 | gvt_dbg_el("emulate as pending slot\n"); | |
349 | /* | |
350 | * otherwise | |
351 | * --> emulate pending execlist exist + but no preemption case | |
352 | */ | |
353 | execlist->pending_slot = slot; | |
354 | emulate_execlist_status(execlist); | |
355 | } | |
356 | return 0; | |
357 | } | |
358 | ||
28c4c6ca ZW |
359 | static void free_workload(struct intel_vgpu_workload *workload) |
360 | { | |
361 | intel_vgpu_unpin_mm(workload->shadow_mm); | |
362 | intel_gvt_mm_unreference(workload->shadow_mm); | |
363 | kmem_cache_free(workload->vgpu->workloads, workload); | |
364 | } | |
365 | ||
366 | #define get_desc_from_elsp_dwords(ed, i) \ | |
367 | ((struct execlist_ctx_descriptor_format *)&((ed)->data[i * 2])) | |
368 | ||
be1da707 ZW |
369 | static void prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload) |
370 | { | |
62f0a11e CW |
371 | const int gmadr_bytes = workload->vgpu->gvt->device_info.gmadr_bytes_in_cmd; |
372 | struct intel_shadow_bb_entry *entry_obj; | |
be1da707 ZW |
373 | |
374 | /* pin the gem object to ggtt */ | |
62f0a11e CW |
375 | list_for_each_entry(entry_obj, &workload->shadow_bb, list) { |
376 | struct i915_vma *vma; | |
be1da707 | 377 | |
62f0a11e CW |
378 | vma = i915_gem_object_ggtt_pin(entry_obj->obj, NULL, 0, 4, 0); |
379 | if (IS_ERR(vma)) { | |
62f0a11e | 380 | return; |
be1da707 | 381 | } |
62f0a11e CW |
382 | |
383 | /* FIXME: we are not tracking our pinned VMA leaving it | |
384 | * up to the core to fix up the stray pin_count upon | |
385 | * free. | |
386 | */ | |
387 | ||
388 | /* update the relocate gma with shadow batch buffer*/ | |
389 | entry_obj->bb_start_cmd_va[1] = i915_ggtt_offset(vma); | |
390 | if (gmadr_bytes == 8) | |
391 | entry_obj->bb_start_cmd_va[2] = 0; | |
be1da707 ZW |
392 | } |
393 | } | |
394 | ||
395 | static int update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx) | |
396 | { | |
c10c1255 TZ |
397 | struct intel_vgpu_workload *workload = container_of(wa_ctx, |
398 | struct intel_vgpu_workload, | |
399 | wa_ctx); | |
400 | int ring_id = workload->ring_id; | |
401 | struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx; | |
be1da707 ZW |
402 | struct drm_i915_gem_object *ctx_obj = |
403 | shadow_ctx->engine[ring_id].state->obj; | |
404 | struct execlist_ring_context *shadow_ring_context; | |
405 | struct page *page; | |
406 | ||
407 | page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); | |
408 | shadow_ring_context = kmap_atomic(page); | |
409 | ||
410 | shadow_ring_context->bb_per_ctx_ptr.val = | |
411 | (shadow_ring_context->bb_per_ctx_ptr.val & | |
412 | (~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma; | |
413 | shadow_ring_context->rcs_indirect_ctx.val = | |
414 | (shadow_ring_context->rcs_indirect_ctx.val & | |
415 | (~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma; | |
416 | ||
417 | kunmap_atomic(shadow_ring_context); | |
418 | return 0; | |
419 | } | |
420 | ||
421 | static void prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) | |
422 | { | |
423 | struct i915_vma *vma; | |
be1da707 ZW |
424 | unsigned char *per_ctx_va = |
425 | (unsigned char *)wa_ctx->indirect_ctx.shadow_va + | |
426 | wa_ctx->indirect_ctx.size; | |
427 | ||
428 | if (wa_ctx->indirect_ctx.size == 0) | |
429 | return; | |
430 | ||
b6d89142 CW |
431 | vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL, |
432 | 0, CACHELINE_BYTES, 0); | |
be1da707 | 433 | if (IS_ERR(vma)) { |
be1da707 ZW |
434 | return; |
435 | } | |
eeacd86e CW |
436 | |
437 | /* FIXME: we are not tracking our pinned VMA leaving it | |
438 | * up to the core to fix up the stray pin_count upon | |
439 | * free. | |
440 | */ | |
be1da707 | 441 | |
b6d89142 | 442 | wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma); |
be1da707 ZW |
443 | |
444 | wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1); | |
445 | memset(per_ctx_va, 0, CACHELINE_BYTES); | |
446 | ||
447 | update_wa_ctx_2_shadow_ctx(wa_ctx); | |
448 | } | |
449 | ||
28c4c6ca ZW |
450 | static int prepare_execlist_workload(struct intel_vgpu_workload *workload) |
451 | { | |
452 | struct intel_vgpu *vgpu = workload->vgpu; | |
453 | struct execlist_ctx_descriptor_format ctx[2]; | |
454 | int ring_id = workload->ring_id; | |
455 | ||
456 | intel_vgpu_pin_mm(workload->shadow_mm); | |
457 | intel_vgpu_sync_oos_pages(workload->vgpu); | |
458 | intel_vgpu_flush_post_shadow(workload->vgpu); | |
be1da707 ZW |
459 | prepare_shadow_batch_buffer(workload); |
460 | prepare_shadow_wa_ctx(&workload->wa_ctx); | |
28c4c6ca ZW |
461 | if (!workload->emulate_schedule_in) |
462 | return 0; | |
463 | ||
464 | ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1); | |
465 | ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0); | |
466 | ||
467 | return emulate_execlist_schedule_in(&vgpu->execlist[ring_id], ctx); | |
468 | } | |
469 | ||
be1da707 ZW |
470 | static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload) |
471 | { | |
472 | /* release all the shadow batch buffer */ | |
473 | if (!list_empty(&workload->shadow_bb)) { | |
474 | struct intel_shadow_bb_entry *entry_obj = | |
475 | list_first_entry(&workload->shadow_bb, | |
476 | struct intel_shadow_bb_entry, | |
477 | list); | |
478 | struct intel_shadow_bb_entry *temp; | |
479 | ||
480 | list_for_each_entry_safe(entry_obj, temp, &workload->shadow_bb, | |
481 | list) { | |
a2861504 | 482 | i915_gem_object_unpin_map(entry_obj->obj); |
bbc36933 | 483 | i915_gem_object_put(entry_obj->obj); |
be1da707 ZW |
484 | list_del(&entry_obj->list); |
485 | kfree(entry_obj); | |
486 | } | |
487 | } | |
488 | } | |
489 | ||
490 | static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) | |
491 | { | |
7e5f3d30 | 492 | if (!wa_ctx->indirect_ctx.obj) |
be1da707 ZW |
493 | return; |
494 | ||
bcd0aede | 495 | i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj); |
bbc36933 | 496 | i915_gem_object_put(wa_ctx->indirect_ctx.obj); |
be1da707 ZW |
497 | } |
498 | ||
28c4c6ca ZW |
499 | static int complete_execlist_workload(struct intel_vgpu_workload *workload) |
500 | { | |
501 | struct intel_vgpu *vgpu = workload->vgpu; | |
502 | struct intel_vgpu_execlist *execlist = | |
503 | &vgpu->execlist[workload->ring_id]; | |
504 | struct intel_vgpu_workload *next_workload; | |
505 | struct list_head *next = workload_q_head(vgpu, workload->ring_id)->next; | |
506 | bool lite_restore = false; | |
507 | int ret; | |
508 | ||
509 | gvt_dbg_el("complete workload %p status %d\n", workload, | |
510 | workload->status); | |
511 | ||
be1da707 ZW |
512 | release_shadow_batch_buffer(workload); |
513 | release_shadow_wa_ctx(&workload->wa_ctx); | |
514 | ||
e4734057 | 515 | if (workload->status || vgpu->resetting) |
28c4c6ca ZW |
516 | goto out; |
517 | ||
518 | if (!list_empty(workload_q_head(vgpu, workload->ring_id))) { | |
519 | struct execlist_ctx_descriptor_format *this_desc, *next_desc; | |
520 | ||
521 | next_workload = container_of(next, | |
522 | struct intel_vgpu_workload, list); | |
523 | this_desc = &workload->ctx_desc; | |
524 | next_desc = &next_workload->ctx_desc; | |
525 | ||
526 | lite_restore = same_context(this_desc, next_desc); | |
527 | } | |
528 | ||
529 | if (lite_restore) { | |
530 | gvt_dbg_el("next context == current - no schedule-out\n"); | |
531 | free_workload(workload); | |
532 | return 0; | |
533 | } | |
534 | ||
535 | ret = emulate_execlist_ctx_schedule_out(execlist, &workload->ctx_desc); | |
536 | if (ret) | |
537 | goto err; | |
538 | out: | |
539 | free_workload(workload); | |
540 | return 0; | |
541 | err: | |
542 | free_workload(workload); | |
543 | return ret; | |
544 | } | |
545 | ||
546 | #define RING_CTX_OFF(x) \ | |
547 | offsetof(struct execlist_ring_context, x) | |
548 | ||
549 | static void read_guest_pdps(struct intel_vgpu *vgpu, | |
550 | u64 ring_context_gpa, u32 pdp[8]) | |
551 | { | |
552 | u64 gpa; | |
553 | int i; | |
554 | ||
555 | gpa = ring_context_gpa + RING_CTX_OFF(pdp3_UDW.val); | |
556 | ||
557 | for (i = 0; i < 8; i++) | |
558 | intel_gvt_hypervisor_read_gpa(vgpu, | |
559 | gpa + i * 8, &pdp[7 - i], 4); | |
560 | } | |
561 | ||
562 | static int prepare_mm(struct intel_vgpu_workload *workload) | |
563 | { | |
564 | struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc; | |
565 | struct intel_vgpu_mm *mm; | |
695fbc08 | 566 | struct intel_vgpu *vgpu = workload->vgpu; |
28c4c6ca ZW |
567 | int page_table_level; |
568 | u32 pdp[8]; | |
569 | ||
570 | if (desc->addressing_mode == 1) { /* legacy 32-bit */ | |
571 | page_table_level = 3; | |
572 | } else if (desc->addressing_mode == 3) { /* legacy 64 bit */ | |
573 | page_table_level = 4; | |
574 | } else { | |
695fbc08 | 575 | gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n"); |
28c4c6ca ZW |
576 | return -EINVAL; |
577 | } | |
578 | ||
579 | read_guest_pdps(workload->vgpu, workload->ring_context_gpa, pdp); | |
580 | ||
581 | mm = intel_vgpu_find_ppgtt_mm(workload->vgpu, page_table_level, pdp); | |
582 | if (mm) { | |
583 | intel_gvt_mm_reference(mm); | |
584 | } else { | |
585 | ||
586 | mm = intel_vgpu_create_mm(workload->vgpu, INTEL_GVT_MM_PPGTT, | |
587 | pdp, page_table_level, 0); | |
588 | if (IS_ERR(mm)) { | |
695fbc08 | 589 | gvt_vgpu_err("fail to create mm object.\n"); |
28c4c6ca ZW |
590 | return PTR_ERR(mm); |
591 | } | |
592 | } | |
593 | workload->shadow_mm = mm; | |
594 | return 0; | |
595 | } | |
596 | ||
597 | #define get_last_workload(q) \ | |
598 | (list_empty(q) ? NULL : container_of(q->prev, \ | |
599 | struct intel_vgpu_workload, list)) | |
600 | ||
76a79d59 | 601 | static int submit_context(struct intel_vgpu *vgpu, int ring_id, |
28c4c6ca ZW |
602 | struct execlist_ctx_descriptor_format *desc, |
603 | bool emulate_schedule_in) | |
604 | { | |
605 | struct list_head *q = workload_q_head(vgpu, ring_id); | |
606 | struct intel_vgpu_workload *last_workload = get_last_workload(q); | |
607 | struct intel_vgpu_workload *workload = NULL; | |
608 | u64 ring_context_gpa; | |
be1da707 | 609 | u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx; |
28c4c6ca ZW |
610 | int ret; |
611 | ||
612 | ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, | |
613 | (u32)((desc->lrca + 1) << GTT_PAGE_SHIFT)); | |
614 | if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) { | |
695fbc08 | 615 | gvt_vgpu_err("invalid guest context LRCA: %x\n", desc->lrca); |
28c4c6ca ZW |
616 | return -EINVAL; |
617 | } | |
618 | ||
619 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
620 | RING_CTX_OFF(ring_header.val), &head, 4); | |
621 | ||
622 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
623 | RING_CTX_OFF(ring_tail.val), &tail, 4); | |
624 | ||
625 | head &= RB_HEAD_OFF_MASK; | |
626 | tail &= RB_TAIL_OFF_MASK; | |
627 | ||
628 | if (last_workload && same_context(&last_workload->ctx_desc, desc)) { | |
629 | gvt_dbg_el("ring id %d cur workload == last\n", ring_id); | |
630 | gvt_dbg_el("ctx head %x real head %lx\n", head, | |
631 | last_workload->rb_tail); | |
632 | /* | |
633 | * cannot use guest context head pointer here, | |
634 | * as it might not be updated at this time | |
635 | */ | |
636 | head = last_workload->rb_tail; | |
637 | } | |
638 | ||
639 | gvt_dbg_el("ring id %d begin a new workload\n", ring_id); | |
640 | ||
641 | workload = kmem_cache_zalloc(vgpu->workloads, GFP_KERNEL); | |
642 | if (!workload) | |
643 | return -ENOMEM; | |
644 | ||
645 | /* record some ring buffer register values for scan and shadow */ | |
646 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
647 | RING_CTX_OFF(rb_start.val), &start, 4); | |
648 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
649 | RING_CTX_OFF(rb_ctrl.val), &ctl, 4); | |
650 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
651 | RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4); | |
652 | ||
653 | INIT_LIST_HEAD(&workload->list); | |
be1da707 | 654 | INIT_LIST_HEAD(&workload->shadow_bb); |
28c4c6ca ZW |
655 | |
656 | init_waitqueue_head(&workload->shadow_ctx_status_wq); | |
657 | atomic_set(&workload->shadow_ctx_active, 0); | |
658 | ||
659 | workload->vgpu = vgpu; | |
660 | workload->ring_id = ring_id; | |
661 | workload->ctx_desc = *desc; | |
662 | workload->ring_context_gpa = ring_context_gpa; | |
663 | workload->rb_head = head; | |
664 | workload->rb_tail = tail; | |
665 | workload->rb_start = start; | |
666 | workload->rb_ctl = ctl; | |
667 | workload->prepare = prepare_execlist_workload; | |
668 | workload->complete = complete_execlist_workload; | |
669 | workload->status = -EINPROGRESS; | |
670 | workload->emulate_schedule_in = emulate_schedule_in; | |
671 | ||
be1da707 ZW |
672 | if (ring_id == RCS) { |
673 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
674 | RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4); | |
675 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
676 | RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4); | |
677 | ||
678 | workload->wa_ctx.indirect_ctx.guest_gma = | |
679 | indirect_ctx & INDIRECT_CTX_ADDR_MASK; | |
680 | workload->wa_ctx.indirect_ctx.size = | |
681 | (indirect_ctx & INDIRECT_CTX_SIZE_MASK) * | |
682 | CACHELINE_BYTES; | |
683 | workload->wa_ctx.per_ctx.guest_gma = | |
684 | per_ctx & PER_CTX_ADDR_MASK; | |
be1da707 ZW |
685 | |
686 | WARN_ON(workload->wa_ctx.indirect_ctx.size && !(per_ctx & 0x1)); | |
687 | } | |
688 | ||
28c4c6ca ZW |
689 | if (emulate_schedule_in) |
690 | memcpy(&workload->elsp_dwords, | |
691 | &vgpu->execlist[ring_id].elsp_dwords, | |
692 | sizeof(workload->elsp_dwords)); | |
693 | ||
694 | gvt_dbg_el("workload %p ring id %d head %x tail %x start %x ctl %x\n", | |
695 | workload, ring_id, head, tail, start, ctl); | |
696 | ||
697 | gvt_dbg_el("workload %p emulate schedule_in %d\n", workload, | |
698 | emulate_schedule_in); | |
699 | ||
700 | ret = prepare_mm(workload); | |
701 | if (ret) { | |
702 | kmem_cache_free(vgpu->workloads, workload); | |
703 | return ret; | |
704 | } | |
705 | ||
706 | queue_workload(workload); | |
707 | return 0; | |
708 | } | |
709 | ||
710 | int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id) | |
711 | { | |
712 | struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id]; | |
713 | struct execlist_ctx_descriptor_format *desc[2], valid_desc[2]; | |
714 | unsigned long valid_desc_bitmap = 0; | |
715 | bool emulate_schedule_in = true; | |
716 | int ret; | |
717 | int i; | |
718 | ||
719 | memset(valid_desc, 0, sizeof(valid_desc)); | |
720 | ||
721 | desc[0] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1); | |
722 | desc[1] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0); | |
723 | ||
724 | for (i = 0; i < 2; i++) { | |
725 | if (!desc[i]->valid) | |
726 | continue; | |
727 | ||
728 | if (!desc[i]->privilege_access) { | |
695fbc08 | 729 | gvt_vgpu_err("unexpected GGTT elsp submission\n"); |
28c4c6ca ZW |
730 | return -EINVAL; |
731 | } | |
732 | ||
733 | /* TODO: add another guest context checks here. */ | |
734 | set_bit(i, &valid_desc_bitmap); | |
735 | valid_desc[i] = *desc[i]; | |
736 | } | |
737 | ||
738 | if (!valid_desc_bitmap) { | |
695fbc08 | 739 | gvt_vgpu_err("no valid desc in a elsp submission\n"); |
28c4c6ca ZW |
740 | return -EINVAL; |
741 | } | |
742 | ||
743 | if (!test_bit(0, (void *)&valid_desc_bitmap) && | |
744 | test_bit(1, (void *)&valid_desc_bitmap)) { | |
695fbc08 | 745 | gvt_vgpu_err("weird elsp submission, desc 0 is not valid\n"); |
28c4c6ca ZW |
746 | return -EINVAL; |
747 | } | |
748 | ||
749 | /* submit workload */ | |
750 | for_each_set_bit(i, (void *)&valid_desc_bitmap, 2) { | |
751 | ret = submit_context(vgpu, ring_id, &valid_desc[i], | |
752 | emulate_schedule_in); | |
753 | if (ret) { | |
695fbc08 | 754 | gvt_vgpu_err("fail to schedule workload\n"); |
28c4c6ca ZW |
755 | return ret; |
756 | } | |
757 | emulate_schedule_in = false; | |
758 | } | |
759 | return 0; | |
760 | } | |
761 | ||
8453d674 ZW |
762 | static void init_vgpu_execlist(struct intel_vgpu *vgpu, int ring_id) |
763 | { | |
764 | struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id]; | |
765 | struct execlist_context_status_pointer_format ctx_status_ptr; | |
766 | u32 ctx_status_ptr_reg; | |
767 | ||
768 | memset(execlist, 0, sizeof(*execlist)); | |
769 | ||
770 | execlist->vgpu = vgpu; | |
771 | execlist->ring_id = ring_id; | |
772 | execlist->slot[0].index = 0; | |
773 | execlist->slot[1].index = 1; | |
774 | ||
775 | ctx_status_ptr_reg = execlist_ring_mmio(vgpu->gvt, ring_id, | |
776 | _EL_OFFSET_STATUS_PTR); | |
777 | ||
778 | ctx_status_ptr.dw = vgpu_vreg(vgpu, ctx_status_ptr_reg); | |
779 | ctx_status_ptr.read_ptr = ctx_status_ptr.write_ptr = 0x7; | |
780 | vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw; | |
781 | } | |
782 | ||
28c4c6ca ZW |
783 | void intel_vgpu_clean_execlist(struct intel_vgpu *vgpu) |
784 | { | |
785 | kmem_cache_destroy(vgpu->workloads); | |
786 | } | |
787 | ||
8453d674 ZW |
788 | int intel_vgpu_init_execlist(struct intel_vgpu *vgpu) |
789 | { | |
0fac21e7 ZW |
790 | enum intel_engine_id i; |
791 | struct intel_engine_cs *engine; | |
8453d674 ZW |
792 | |
793 | /* each ring has a virtual execlist engine */ | |
0fac21e7 | 794 | for_each_engine(engine, vgpu->gvt->dev_priv, i) { |
8453d674 | 795 | init_vgpu_execlist(vgpu, i); |
28c4c6ca ZW |
796 | INIT_LIST_HEAD(&vgpu->workload_q_head[i]); |
797 | } | |
798 | ||
7283accf | 799 | vgpu->workloads = kmem_cache_create("gvt-g_vgpu_workload", |
28c4c6ca ZW |
800 | sizeof(struct intel_vgpu_workload), 0, |
801 | SLAB_HWCACHE_ALIGN, | |
802 | NULL); | |
803 | ||
804 | if (!vgpu->workloads) | |
805 | return -ENOMEM; | |
8453d674 ZW |
806 | |
807 | return 0; | |
808 | } | |
e4734057 ZW |
809 | |
810 | void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu, | |
0427f06a | 811 | unsigned long engine_mask) |
e4734057 | 812 | { |
0427f06a DC |
813 | struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; |
814 | struct intel_engine_cs *engine; | |
815 | struct intel_vgpu_workload *pos, *n; | |
816 | unsigned int tmp; | |
e4734057 | 817 | |
0427f06a | 818 | for_each_engine_masked(engine, dev_priv, engine_mask, tmp) { |
e4734057 | 819 | /* free the unsubmited workload in the queue */ |
0427f06a DC |
820 | list_for_each_entry_safe(pos, n, |
821 | &vgpu->workload_q_head[engine->id], list) { | |
822 | list_del_init(&pos->list); | |
823 | free_workload(pos); | |
e4734057 ZW |
824 | } |
825 | ||
0427f06a | 826 | init_vgpu_execlist(vgpu, engine->id); |
e4734057 ZW |
827 | } |
828 | } |