]>
Commit | Line | Data |
---|---|---|
e4734057 ZW |
1 | /* |
2 | * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
21 | * SOFTWARE. | |
22 | * | |
23 | * Authors: | |
24 | * Zhi Wang <zhi.a.wang@intel.com> | |
25 | * | |
26 | * Contributors: | |
27 | * Ping Gao <ping.a.gao@intel.com> | |
28 | * Tina Zhang <tina.zhang@intel.com> | |
29 | * Chanbin Du <changbin.du@intel.com> | |
30 | * Min He <min.he@intel.com> | |
31 | * Bing Niu <bing.niu@intel.com> | |
32 | * Zhenyu Wang <zhenyuw@linux.intel.com> | |
33 | * | |
34 | */ | |
35 | ||
e4734057 ZW |
36 | #include <linux/kthread.h> |
37 | ||
10be98a7 CW |
38 | #include "gem/i915_gem_context.h" |
39 | #include "gem/i915_gem_pm.h" | |
40 | #include "gt/intel_context.h" | |
41 | ||
feddf6e8 ZW |
42 | #include "i915_drv.h" |
43 | #include "gvt.h" | |
44 | ||
e4734057 ZW |
45 | #define RING_CTX_OFF(x) \ |
46 | offsetof(struct execlist_ring_context, x) | |
47 | ||
999ccb40 CD |
48 | static void set_context_pdp_root_pointer( |
49 | struct execlist_ring_context *ring_context, | |
e4734057 ZW |
50 | u32 pdp[8]) |
51 | { | |
e4734057 ZW |
52 | int i; |
53 | ||
54 | for (i = 0; i < 8; i++) | |
1417fad7 | 55 | ring_context->pdps[i].val = pdp[7 - i]; |
e4734057 ZW |
56 | } |
57 | ||
b20c0d5c ZW |
58 | static void update_shadow_pdps(struct intel_vgpu_workload *workload) |
59 | { | |
b20c0d5c | 60 | struct drm_i915_gem_object *ctx_obj = |
1fc44d9b | 61 | workload->req->hw_context->state->obj; |
b20c0d5c ZW |
62 | struct execlist_ring_context *shadow_ring_context; |
63 | struct page *page; | |
64 | ||
65 | if (WARN_ON(!workload->shadow_mm)) | |
66 | return; | |
67 | ||
68 | if (WARN_ON(!atomic_read(&workload->shadow_mm->pincount))) | |
69 | return; | |
70 | ||
71 | page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); | |
72 | shadow_ring_context = kmap(page); | |
73 | set_context_pdp_root_pointer(shadow_ring_context, | |
74 | (void *)workload->shadow_mm->ppgtt_mm.shadow_pdps); | |
75 | kunmap(page); | |
76 | } | |
77 | ||
fa3dd623 MH |
78 | /* |
79 | * when populating shadow ctx from guest, we should not overrride oa related | |
80 | * registers, so that they will not be overlapped by guest oa configs. Thus | |
81 | * made it possible to capture oa data from host for both host and guests. | |
82 | */ | |
83 | static void sr_oa_regs(struct intel_vgpu_workload *workload, | |
84 | u32 *reg_state, bool save) | |
85 | { | |
86 | struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv; | |
87 | u32 ctx_oactxctrl = dev_priv->perf.oa.ctx_oactxctrl_offset; | |
88 | u32 ctx_flexeu0 = dev_priv->perf.oa.ctx_flexeu0_offset; | |
89 | int i = 0; | |
90 | u32 flex_mmio[] = { | |
91 | i915_mmio_reg_offset(EU_PERF_CNTL0), | |
92 | i915_mmio_reg_offset(EU_PERF_CNTL1), | |
93 | i915_mmio_reg_offset(EU_PERF_CNTL2), | |
94 | i915_mmio_reg_offset(EU_PERF_CNTL3), | |
95 | i915_mmio_reg_offset(EU_PERF_CNTL4), | |
96 | i915_mmio_reg_offset(EU_PERF_CNTL5), | |
97 | i915_mmio_reg_offset(EU_PERF_CNTL6), | |
98 | }; | |
99 | ||
8a68d464 | 100 | if (workload->ring_id != RCS0) |
fa3dd623 MH |
101 | return; |
102 | ||
103 | if (save) { | |
104 | workload->oactxctrl = reg_state[ctx_oactxctrl + 1]; | |
105 | ||
106 | for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) { | |
107 | u32 state_offset = ctx_flexeu0 + i * 2; | |
108 | ||
109 | workload->flex_mmio[i] = reg_state[state_offset + 1]; | |
110 | } | |
111 | } else { | |
112 | reg_state[ctx_oactxctrl] = | |
113 | i915_mmio_reg_offset(GEN8_OACTXCONTROL); | |
114 | reg_state[ctx_oactxctrl + 1] = workload->oactxctrl; | |
115 | ||
116 | for (i = 0; i < ARRAY_SIZE(workload->flex_mmio); i++) { | |
117 | u32 state_offset = ctx_flexeu0 + i * 2; | |
118 | u32 mmio = flex_mmio[i]; | |
119 | ||
120 | reg_state[state_offset] = mmio; | |
121 | reg_state[state_offset + 1] = workload->flex_mmio[i]; | |
122 | } | |
123 | } | |
124 | } | |
125 | ||
e4734057 ZW |
126 | static int populate_shadow_context(struct intel_vgpu_workload *workload) |
127 | { | |
128 | struct intel_vgpu *vgpu = workload->vgpu; | |
129 | struct intel_gvt *gvt = vgpu->gvt; | |
130 | int ring_id = workload->ring_id; | |
e4734057 | 131 | struct drm_i915_gem_object *ctx_obj = |
1fc44d9b | 132 | workload->req->hw_context->state->obj; |
e4734057 ZW |
133 | struct execlist_ring_context *shadow_ring_context; |
134 | struct page *page; | |
135 | void *dst; | |
136 | unsigned long context_gpa, context_page_num; | |
137 | int i; | |
138 | ||
e4734057 | 139 | page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); |
c754936f | 140 | shadow_ring_context = kmap(page); |
e4734057 | 141 | |
fa3dd623 | 142 | sr_oa_regs(workload, (u32 *)shadow_ring_context, true); |
e4734057 ZW |
143 | #define COPY_REG(name) \ |
144 | intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \ | |
145 | + RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4) | |
d8303075 ZW |
146 | #define COPY_REG_MASKED(name) {\ |
147 | intel_gvt_hypervisor_read_gpa(vgpu, workload->ring_context_gpa \ | |
148 | + RING_CTX_OFF(name.val),\ | |
149 | &shadow_ring_context->name.val, 4);\ | |
150 | shadow_ring_context->name.val |= 0xffff << 16;\ | |
151 | } | |
e4734057 | 152 | |
d8303075 | 153 | COPY_REG_MASKED(ctx_ctrl); |
e4734057 ZW |
154 | COPY_REG(ctx_timestamp); |
155 | ||
8a68d464 | 156 | if (ring_id == RCS0) { |
e4734057 ZW |
157 | COPY_REG(bb_per_ctx_ptr); |
158 | COPY_REG(rcs_indirect_ctx); | |
159 | COPY_REG(rcs_indirect_ctx_offset); | |
160 | } | |
161 | #undef COPY_REG | |
d8303075 | 162 | #undef COPY_REG_MASKED |
e4734057 | 163 | |
e4734057 ZW |
164 | intel_gvt_hypervisor_read_gpa(vgpu, |
165 | workload->ring_context_gpa + | |
166 | sizeof(*shadow_ring_context), | |
167 | (void *)shadow_ring_context + | |
168 | sizeof(*shadow_ring_context), | |
9556e118 | 169 | I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context)); |
e4734057 | 170 | |
fa3dd623 | 171 | sr_oa_regs(workload, (u32 *)shadow_ring_context, false); |
c754936f | 172 | kunmap(page); |
8bfa02c8 ZY |
173 | |
174 | if (IS_RESTORE_INHIBIT(shadow_ring_context->ctx_ctrl.val)) | |
175 | return 0; | |
176 | ||
177 | gvt_dbg_sched("ring id %d workload lrca %x", ring_id, | |
178 | workload->ctx_desc.lrca); | |
179 | ||
180 | context_page_num = gvt->dev_priv->engine[ring_id]->context_size; | |
181 | ||
182 | context_page_num = context_page_num >> PAGE_SHIFT; | |
183 | ||
8a68d464 | 184 | if (IS_BROADWELL(gvt->dev_priv) && ring_id == RCS0) |
8bfa02c8 ZY |
185 | context_page_num = 19; |
186 | ||
187 | i = 2; | |
188 | while (i < context_page_num) { | |
189 | context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, | |
190 | (u32)((workload->ctx_desc.lrca + i) << | |
191 | I915_GTT_PAGE_SHIFT)); | |
192 | if (context_gpa == INTEL_GVT_INVALID_ADDR) { | |
193 | gvt_vgpu_err("Invalid guest context descriptor\n"); | |
194 | return -EFAULT; | |
195 | } | |
196 | ||
197 | page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i); | |
198 | dst = kmap(page); | |
199 | intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst, | |
200 | I915_GTT_PAGE_SIZE); | |
201 | kunmap(page); | |
202 | i++; | |
203 | } | |
e4734057 ZW |
204 | return 0; |
205 | } | |
206 | ||
e61e0f51 | 207 | static inline bool is_gvt_request(struct i915_request *req) |
bc2d4b62 | 208 | { |
4e0d64db | 209 | return i915_gem_context_force_single_submission(req->gem_context); |
bc2d4b62 CD |
210 | } |
211 | ||
295764cd XZ |
212 | static void save_ring_hw_state(struct intel_vgpu *vgpu, int ring_id) |
213 | { | |
214 | struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; | |
215 | u32 ring_base = dev_priv->engine[ring_id]->mmio_base; | |
216 | i915_reg_t reg; | |
217 | ||
218 | reg = RING_INSTDONE(ring_base); | |
219 | vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg); | |
220 | reg = RING_ACTHD(ring_base); | |
221 | vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg); | |
222 | reg = RING_ACTHD_UDW(ring_base); | |
223 | vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg); | |
224 | } | |
225 | ||
e4734057 ZW |
226 | static int shadow_context_status_change(struct notifier_block *nb, |
227 | unsigned long action, void *data) | |
228 | { | |
e61e0f51 | 229 | struct i915_request *req = data; |
3fc03069 CD |
230 | struct intel_gvt *gvt = container_of(nb, struct intel_gvt, |
231 | shadow_ctx_notifier_block[req->engine->id]); | |
232 | struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; | |
0e86cc9c CD |
233 | enum intel_engine_id ring_id = req->engine->id; |
234 | struct intel_vgpu_workload *workload; | |
679fd3eb | 235 | unsigned long flags; |
0e86cc9c CD |
236 | |
237 | if (!is_gvt_request(req)) { | |
679fd3eb | 238 | spin_lock_irqsave(&scheduler->mmio_context_lock, flags); |
0e86cc9c CD |
239 | if (action == INTEL_CONTEXT_SCHEDULE_IN && |
240 | scheduler->engine_owner[ring_id]) { | |
241 | /* Switch ring from vGPU to host. */ | |
242 | intel_gvt_switch_mmio(scheduler->engine_owner[ring_id], | |
243 | NULL, ring_id); | |
244 | scheduler->engine_owner[ring_id] = NULL; | |
245 | } | |
679fd3eb | 246 | spin_unlock_irqrestore(&scheduler->mmio_context_lock, flags); |
0e86cc9c CD |
247 | |
248 | return NOTIFY_OK; | |
249 | } | |
e4734057 | 250 | |
0e86cc9c CD |
251 | workload = scheduler->current_workload[ring_id]; |
252 | if (unlikely(!workload)) | |
9272f73f CD |
253 | return NOTIFY_OK; |
254 | ||
e4734057 ZW |
255 | switch (action) { |
256 | case INTEL_CONTEXT_SCHEDULE_IN: | |
679fd3eb | 257 | spin_lock_irqsave(&scheduler->mmio_context_lock, flags); |
0e86cc9c CD |
258 | if (workload->vgpu != scheduler->engine_owner[ring_id]) { |
259 | /* Switch ring from host to vGPU or vGPU to vGPU. */ | |
260 | intel_gvt_switch_mmio(scheduler->engine_owner[ring_id], | |
261 | workload->vgpu, ring_id); | |
262 | scheduler->engine_owner[ring_id] = workload->vgpu; | |
263 | } else | |
264 | gvt_dbg_sched("skip ring %d mmio switch for vgpu%d\n", | |
265 | ring_id, workload->vgpu->id); | |
679fd3eb | 266 | spin_unlock_irqrestore(&scheduler->mmio_context_lock, flags); |
e4734057 ZW |
267 | atomic_set(&workload->shadow_ctx_active, 1); |
268 | break; | |
269 | case INTEL_CONTEXT_SCHEDULE_OUT: | |
295764cd | 270 | save_ring_hw_state(workload->vgpu, ring_id); |
e4734057 ZW |
271 | atomic_set(&workload->shadow_ctx_active, 0); |
272 | break; | |
da5f99ea ZW |
273 | case INTEL_CONTEXT_SCHEDULE_PREEMPTED: |
274 | save_ring_hw_state(workload->vgpu, ring_id); | |
275 | break; | |
e4734057 ZW |
276 | default: |
277 | WARN_ON(1); | |
278 | return NOTIFY_OK; | |
279 | } | |
280 | wake_up(&workload->shadow_ctx_status_wq); | |
281 | return NOTIFY_OK; | |
282 | } | |
283 | ||
251d46b0 CW |
284 | static void |
285 | shadow_context_descriptor_update(struct intel_context *ce, | |
286 | struct intel_vgpu_workload *workload) | |
9dfb8e5b | 287 | { |
251d46b0 | 288 | u64 desc = ce->lrc_desc; |
9dfb8e5b | 289 | |
251d46b0 CW |
290 | /* |
291 | * Update bits 0-11 of the context descriptor which includes flags | |
9dfb8e5b KL |
292 | * like GEN8_CTX_* cached in desc_template |
293 | */ | |
294 | desc &= U64_MAX << 12; | |
1fc44d9b | 295 | desc |= ce->gem_context->desc_template & ((1ULL << 12) - 1); |
9dfb8e5b | 296 | |
251d46b0 CW |
297 | desc &= ~(0x3 << GEN8_CTX_ADDRESSING_MODE_SHIFT); |
298 | desc |= workload->ctx_desc.addressing_mode << | |
299 | GEN8_CTX_ADDRESSING_MODE_SHIFT; | |
300 | ||
9dfb8e5b KL |
301 | ce->lrc_desc = desc; |
302 | } | |
303 | ||
0a53bc07 | 304 | static int copy_workload_to_ring_buffer(struct intel_vgpu_workload *workload) |
305 | { | |
306 | struct intel_vgpu *vgpu = workload->vgpu; | |
1fc44d9b | 307 | struct i915_request *req = workload->req; |
0a53bc07 | 308 | void *shadow_ring_buffer_va; |
309 | u32 *cs; | |
a8c2d5ab | 310 | int err; |
cd7e61b9 | 311 | |
df2ea3c2 | 312 | if (IS_GEN(req->i915, 9) && is_inhibit_context(req->hw_context)) |
cd7e61b9 | 313 | intel_vgpu_restore_inhibit_context(vgpu, req); |
0a53bc07 | 314 | |
a8c2d5ab W |
315 | /* |
316 | * To track whether a request has started on HW, we can emit a | |
317 | * breadcrumb at the beginning of the request and check its | |
318 | * timeline's HWSP to see if the breadcrumb has advanced past the | |
319 | * start of this request. Actually, the request must have the | |
320 | * init_breadcrumb if its timeline set has_init_bread_crumb, or the | |
321 | * scheduler might get a wrong state of it during reset. Since the | |
322 | * requests from gvt always set the has_init_breadcrumb flag, here | |
323 | * need to do the emit_init_breadcrumb for all the requests. | |
324 | */ | |
325 | if (req->engine->emit_init_breadcrumb) { | |
326 | err = req->engine->emit_init_breadcrumb(req); | |
327 | if (err) { | |
328 | gvt_vgpu_err("fail to emit init breadcrumb\n"); | |
329 | return err; | |
330 | } | |
331 | } | |
332 | ||
0a53bc07 | 333 | /* allocate shadow ring buffer */ |
334 | cs = intel_ring_begin(workload->req, workload->rb_len / sizeof(u32)); | |
335 | if (IS_ERR(cs)) { | |
336 | gvt_vgpu_err("fail to alloc size =%ld shadow ring buffer\n", | |
337 | workload->rb_len); | |
338 | return PTR_ERR(cs); | |
339 | } | |
340 | ||
341 | shadow_ring_buffer_va = workload->shadow_ring_buffer_va; | |
342 | ||
343 | /* get shadow ring buffer va */ | |
344 | workload->shadow_ring_buffer_va = cs; | |
345 | ||
346 | memcpy(cs, shadow_ring_buffer_va, | |
347 | workload->rb_len); | |
348 | ||
349 | cs += workload->rb_len / sizeof(u32); | |
350 | intel_ring_advance(workload->req, cs); | |
351 | ||
352 | return 0; | |
353 | } | |
354 | ||
7b302556 | 355 | static void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) |
a3cfdca9 | 356 | { |
357 | if (!wa_ctx->indirect_ctx.obj) | |
358 | return; | |
359 | ||
360 | i915_gem_object_unpin_map(wa_ctx->indirect_ctx.obj); | |
361 | i915_gem_object_put(wa_ctx->indirect_ctx.obj); | |
0f755512 WL |
362 | |
363 | wa_ctx->indirect_ctx.obj = NULL; | |
364 | wa_ctx->indirect_ctx.shadow_va = NULL; | |
a3cfdca9 | 365 | } |
366 | ||
41874148 CX |
367 | static void set_context_ppgtt_from_shadow(struct intel_vgpu_workload *workload, |
368 | struct i915_gem_context *ctx) | |
4f15665c XZ |
369 | { |
370 | struct intel_vgpu_mm *mm = workload->shadow_mm; | |
ab53497b | 371 | struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(ctx->vm); |
4f15665c XZ |
372 | int i = 0; |
373 | ||
4f15665c | 374 | if (mm->ppgtt_mm.root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY) { |
b5b7bef9 | 375 | px_dma(ppgtt->pd) = mm->ppgtt_mm.shadow_pdps[0]; |
4f15665c XZ |
376 | } else { |
377 | for (i = 0; i < GVT_RING_CTX_NR_PDPS; i++) { | |
b5b7bef9 MK |
378 | struct i915_page_directory * const pd = |
379 | i915_pd_entry(ppgtt->pd, i); | |
380 | ||
381 | px_dma(pd) = mm->ppgtt_mm.shadow_pdps[i]; | |
4f15665c XZ |
382 | } |
383 | } | |
4f15665c XZ |
384 | } |
385 | ||
f0e99437 ZW |
386 | static int |
387 | intel_gvt_workload_req_alloc(struct intel_vgpu_workload *workload) | |
388 | { | |
389 | struct intel_vgpu *vgpu = workload->vgpu; | |
390 | struct intel_vgpu_submission *s = &vgpu->submission; | |
f0e99437 | 391 | struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; |
f0e99437 | 392 | struct i915_request *rq; |
f0e99437 ZW |
393 | |
394 | lockdep_assert_held(&dev_priv->drm.struct_mutex); | |
395 | ||
396 | if (workload->req) | |
251d46b0 | 397 | return 0; |
f0e99437 | 398 | |
251d46b0 | 399 | rq = i915_request_create(s->shadow[workload->ring_id]); |
f0e99437 ZW |
400 | if (IS_ERR(rq)) { |
401 | gvt_vgpu_err("fail to allocate gem request\n"); | |
251d46b0 | 402 | return PTR_ERR(rq); |
f0e99437 | 403 | } |
251d46b0 | 404 | |
f0e99437 | 405 | workload->req = i915_request_get(rq); |
251d46b0 | 406 | return 0; |
f0e99437 ZW |
407 | } |
408 | ||
89ea20b9 PG |
409 | /** |
410 | * intel_gvt_scan_and_shadow_workload - audit the workload by scanning and | |
411 | * shadow it as well, include ringbuffer,wa_ctx and ctx. | |
412 | * @workload: an abstract entity for each execlist submission. | |
413 | * | |
414 | * This function is called before the workload submitting to i915, to make | |
415 | * sure the content of the workload is valid. | |
416 | */ | |
417 | int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload) | |
e4734057 | 418 | { |
1406a14b ZW |
419 | struct intel_vgpu *vgpu = workload->vgpu; |
420 | struct intel_vgpu_submission *s = &vgpu->submission; | |
1406a14b | 421 | struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; |
e4734057 ZW |
422 | int ret; |
423 | ||
87e919d7 PG |
424 | lockdep_assert_held(&dev_priv->drm.struct_mutex); |
425 | ||
f0e99437 | 426 | if (workload->shadow) |
d0302e74 | 427 | return 0; |
e4734057 | 428 | |
1fc44d9b | 429 | if (!test_and_set_bit(workload->ring_id, s->shadow_ctx_desc_updated)) |
251d46b0 CW |
430 | shadow_context_descriptor_update(s->shadow[workload->ring_id], |
431 | workload); | |
3cd23b82 | 432 | |
89ea20b9 | 433 | ret = intel_gvt_scan_and_shadow_ringbuffer(workload); |
be1da707 | 434 | if (ret) |
251d46b0 | 435 | return ret; |
be1da707 | 436 | |
8a68d464 | 437 | if (workload->ring_id == RCS0 && workload->wa_ctx.indirect_ctx.size) { |
17f1b1a6 TZ |
438 | ret = intel_gvt_scan_and_shadow_wa_ctx(&workload->wa_ctx); |
439 | if (ret) | |
1fc44d9b | 440 | goto err_shadow; |
17f1b1a6 | 441 | } |
be1da707 | 442 | |
f0e99437 | 443 | workload->shadow = true; |
6bb2a2af | 444 | return 0; |
f2880e04 | 445 | err_shadow: |
446 | release_shadow_wa_ctx(&workload->wa_ctx); | |
89ea20b9 PG |
447 | return ret; |
448 | } | |
449 | ||
f52c380a ZW |
450 | static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload); |
451 | ||
d8235b5e ZW |
452 | static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload) |
453 | { | |
454 | struct intel_gvt *gvt = workload->vgpu->gvt; | |
455 | const int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd; | |
f52c380a ZW |
456 | struct intel_vgpu_shadow_bb *bb; |
457 | int ret; | |
458 | ||
459 | list_for_each_entry(bb, &workload->shadow_bb, list) { | |
ef75c685 | 460 | /* For privilge batch buffer and not wa_ctx, the bb_start_cmd_va |
461 | * is only updated into ring_scan_buffer, not real ring address | |
462 | * allocated in later copy_workload_to_ring_buffer. pls be noted | |
463 | * shadow_ring_buffer_va is now pointed to real ring buffer va | |
464 | * in copy_workload_to_ring_buffer. | |
465 | */ | |
466 | ||
467 | if (bb->bb_offset) | |
468 | bb->bb_start_cmd_va = workload->shadow_ring_buffer_va | |
469 | + bb->bb_offset; | |
470 | ||
96bebe39 ZY |
471 | if (bb->ppgtt) { |
472 | /* for non-priv bb, scan&shadow is only for | |
473 | * debugging purpose, so the content of shadow bb | |
474 | * is the same as original bb. Therefore, | |
475 | * here, rather than switch to shadow bb's gma | |
476 | * address, we directly use original batch buffer's | |
477 | * gma address, and send original bb to hardware | |
478 | * directly | |
479 | */ | |
480 | if (bb->clflush & CLFLUSH_AFTER) { | |
481 | drm_clflush_virt_range(bb->va, | |
482 | bb->obj->base.size); | |
483 | bb->clflush &= ~CLFLUSH_AFTER; | |
484 | } | |
f0e4a063 | 485 | i915_gem_object_finish_access(bb->obj); |
96bebe39 ZY |
486 | bb->accessing = false; |
487 | ||
488 | } else { | |
489 | bb->vma = i915_gem_object_ggtt_pin(bb->obj, | |
490 | NULL, 0, 0, 0); | |
491 | if (IS_ERR(bb->vma)) { | |
492 | ret = PTR_ERR(bb->vma); | |
493 | goto err; | |
494 | } | |
495 | ||
496 | /* relocate shadow batch buffer */ | |
497 | bb->bb_start_cmd_va[1] = i915_ggtt_offset(bb->vma); | |
498 | if (gmadr_bytes == 8) | |
499 | bb->bb_start_cmd_va[2] = 0; | |
500 | ||
501 | /* No one is going to touch shadow bb from now on. */ | |
502 | if (bb->clflush & CLFLUSH_AFTER) { | |
503 | drm_clflush_virt_range(bb->va, | |
504 | bb->obj->base.size); | |
505 | bb->clflush &= ~CLFLUSH_AFTER; | |
506 | } | |
507 | ||
508 | ret = i915_gem_object_set_to_gtt_domain(bb->obj, | |
6951e589 | 509 | false); |
96bebe39 ZY |
510 | if (ret) |
511 | goto err; | |
512 | ||
a5236978 CW |
513 | ret = i915_vma_move_to_active(bb->vma, |
514 | workload->req, | |
515 | 0); | |
516 | if (ret) | |
517 | goto err; | |
6951e589 CW |
518 | |
519 | i915_gem_object_finish_access(bb->obj); | |
520 | bb->accessing = false; | |
f52c380a | 521 | } |
d8235b5e ZW |
522 | } |
523 | return 0; | |
f52c380a ZW |
524 | err: |
525 | release_shadow_batch_buffer(workload); | |
526 | return ret; | |
d8235b5e ZW |
527 | } |
528 | ||
1fc44d9b | 529 | static void update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx) |
d8235b5e | 530 | { |
1fc44d9b CW |
531 | struct intel_vgpu_workload *workload = |
532 | container_of(wa_ctx, struct intel_vgpu_workload, wa_ctx); | |
533 | struct i915_request *rq = workload->req; | |
534 | struct execlist_ring_context *shadow_ring_context = | |
535 | (struct execlist_ring_context *)rq->hw_context->lrc_reg_state; | |
d8235b5e ZW |
536 | |
537 | shadow_ring_context->bb_per_ctx_ptr.val = | |
538 | (shadow_ring_context->bb_per_ctx_ptr.val & | |
539 | (~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma; | |
540 | shadow_ring_context->rcs_indirect_ctx.val = | |
541 | (shadow_ring_context->rcs_indirect_ctx.val & | |
542 | (~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma; | |
d8235b5e ZW |
543 | } |
544 | ||
545 | static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) | |
546 | { | |
547 | struct i915_vma *vma; | |
548 | unsigned char *per_ctx_va = | |
549 | (unsigned char *)wa_ctx->indirect_ctx.shadow_va + | |
550 | wa_ctx->indirect_ctx.size; | |
551 | ||
552 | if (wa_ctx->indirect_ctx.size == 0) | |
553 | return 0; | |
554 | ||
555 | vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL, | |
556 | 0, CACHELINE_BYTES, 0); | |
557 | if (IS_ERR(vma)) | |
558 | return PTR_ERR(vma); | |
559 | ||
560 | /* FIXME: we are not tracking our pinned VMA leaving it | |
561 | * up to the core to fix up the stray pin_count upon | |
562 | * free. | |
563 | */ | |
564 | ||
565 | wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma); | |
566 | ||
567 | wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1); | |
568 | memset(per_ctx_va, 0, CACHELINE_BYTES); | |
569 | ||
570 | update_wa_ctx_2_shadow_ctx(wa_ctx); | |
571 | return 0; | |
572 | } | |
573 | ||
574 | static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload) | |
575 | { | |
f52c380a ZW |
576 | struct intel_vgpu *vgpu = workload->vgpu; |
577 | struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; | |
578 | struct intel_vgpu_shadow_bb *bb, *pos; | |
579 | ||
580 | if (list_empty(&workload->shadow_bb)) | |
581 | return; | |
582 | ||
583 | bb = list_first_entry(&workload->shadow_bb, | |
584 | struct intel_vgpu_shadow_bb, list); | |
585 | ||
586 | mutex_lock(&dev_priv->drm.struct_mutex); | |
587 | ||
588 | list_for_each_entry_safe(bb, pos, &workload->shadow_bb, list) { | |
589 | if (bb->obj) { | |
590 | if (bb->accessing) | |
f0e4a063 | 591 | i915_gem_object_finish_access(bb->obj); |
f52c380a ZW |
592 | |
593 | if (bb->va && !IS_ERR(bb->va)) | |
594 | i915_gem_object_unpin_map(bb->obj); | |
595 | ||
596 | if (bb->vma && !IS_ERR(bb->vma)) { | |
597 | i915_vma_unpin(bb->vma); | |
598 | i915_vma_close(bb->vma); | |
599 | } | |
c017cf6b | 600 | i915_gem_object_put(bb->obj); |
d8235b5e | 601 | } |
f52c380a ZW |
602 | list_del(&bb->list); |
603 | kfree(bb); | |
d8235b5e | 604 | } |
f52c380a ZW |
605 | |
606 | mutex_unlock(&dev_priv->drm.struct_mutex); | |
d8235b5e ZW |
607 | } |
608 | ||
497aa3f5 ZW |
609 | static int prepare_workload(struct intel_vgpu_workload *workload) |
610 | { | |
d8235b5e | 611 | struct intel_vgpu *vgpu = workload->vgpu; |
41874148 CX |
612 | struct intel_vgpu_submission *s = &vgpu->submission; |
613 | int ring = workload->ring_id; | |
497aa3f5 ZW |
614 | int ret = 0; |
615 | ||
d8235b5e ZW |
616 | ret = intel_vgpu_pin_mm(workload->shadow_mm); |
617 | if (ret) { | |
618 | gvt_vgpu_err("fail to vgpu pin mm\n"); | |
619 | return ret; | |
620 | } | |
621 | ||
41874148 CX |
622 | if (workload->shadow_mm->type != INTEL_GVT_MM_PPGTT || |
623 | !workload->shadow_mm->ppgtt_mm.shadowed) { | |
624 | gvt_vgpu_err("workload shadow ppgtt isn't ready\n"); | |
625 | return -EINVAL; | |
626 | } | |
627 | ||
b20c0d5c ZW |
628 | update_shadow_pdps(workload); |
629 | ||
41874148 CX |
630 | set_context_ppgtt_from_shadow(workload, s->shadow[ring]->gem_context); |
631 | ||
d8235b5e ZW |
632 | ret = intel_vgpu_sync_oos_pages(workload->vgpu); |
633 | if (ret) { | |
634 | gvt_vgpu_err("fail to vgpu sync oos pages\n"); | |
635 | goto err_unpin_mm; | |
636 | } | |
637 | ||
638 | ret = intel_vgpu_flush_post_shadow(workload->vgpu); | |
639 | if (ret) { | |
640 | gvt_vgpu_err("fail to flush post shadow\n"); | |
641 | goto err_unpin_mm; | |
642 | } | |
643 | ||
6bb2a2af | 644 | ret = copy_workload_to_ring_buffer(workload); |
f2880e04 | 645 | if (ret) { |
646 | gvt_vgpu_err("fail to generate request\n"); | |
647 | goto err_unpin_mm; | |
648 | } | |
649 | ||
d8235b5e ZW |
650 | ret = prepare_shadow_batch_buffer(workload); |
651 | if (ret) { | |
652 | gvt_vgpu_err("fail to prepare_shadow_batch_buffer\n"); | |
653 | goto err_unpin_mm; | |
654 | } | |
655 | ||
656 | ret = prepare_shadow_wa_ctx(&workload->wa_ctx); | |
657 | if (ret) { | |
658 | gvt_vgpu_err("fail to prepare_shadow_wa_ctx\n"); | |
659 | goto err_shadow_batch; | |
660 | } | |
661 | ||
662 | if (workload->prepare) { | |
497aa3f5 | 663 | ret = workload->prepare(workload); |
d8235b5e ZW |
664 | if (ret) |
665 | goto err_shadow_wa_ctx; | |
666 | } | |
497aa3f5 | 667 | |
d8235b5e ZW |
668 | return 0; |
669 | err_shadow_wa_ctx: | |
670 | release_shadow_wa_ctx(&workload->wa_ctx); | |
671 | err_shadow_batch: | |
672 | release_shadow_batch_buffer(workload); | |
673 | err_unpin_mm: | |
674 | intel_vgpu_unpin_mm(workload->shadow_mm); | |
497aa3f5 ZW |
675 | return ret; |
676 | } | |
677 | ||
89ea20b9 PG |
678 | static int dispatch_workload(struct intel_vgpu_workload *workload) |
679 | { | |
1406a14b | 680 | struct intel_vgpu *vgpu = workload->vgpu; |
1406a14b | 681 | struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; |
f552e7bd | 682 | struct i915_request *rq; |
89ea20b9 | 683 | int ring_id = workload->ring_id; |
1fc44d9b | 684 | int ret; |
89ea20b9 PG |
685 | |
686 | gvt_dbg_sched("ring id %d prepare to dispatch workload %p\n", | |
687 | ring_id, workload); | |
688 | ||
f25a49ab | 689 | mutex_lock(&vgpu->vgpu_lock); |
89ea20b9 PG |
690 | mutex_lock(&dev_priv->drm.struct_mutex); |
691 | ||
f0e99437 ZW |
692 | ret = intel_gvt_workload_req_alloc(workload); |
693 | if (ret) | |
694 | goto err_req; | |
695 | ||
89ea20b9 | 696 | ret = intel_gvt_scan_and_shadow_workload(workload); |
e4734057 | 697 | if (ret) |
90d27a1b | 698 | goto out; |
e4734057 | 699 | |
f0e99437 ZW |
700 | ret = populate_shadow_context(workload); |
701 | if (ret) { | |
702 | release_shadow_wa_ctx(&workload->wa_ctx); | |
703 | goto out; | |
704 | } | |
e4734057 | 705 | |
f0e99437 | 706 | ret = prepare_workload(workload); |
90d27a1b | 707 | out: |
f552e7bd ZW |
708 | if (ret) { |
709 | /* We might still need to add request with | |
710 | * clean ctx to retire it properly.. | |
711 | */ | |
712 | rq = fetch_and_zero(&workload->req); | |
713 | i915_request_put(rq); | |
714 | } | |
715 | ||
89ea20b9 PG |
716 | if (!IS_ERR_OR_NULL(workload->req)) { |
717 | gvt_dbg_sched("ring id %d submit workload to i915 %p\n", | |
718 | ring_id, workload->req); | |
e61e0f51 | 719 | i915_request_add(workload->req); |
89ea20b9 PG |
720 | workload->dispatched = true; |
721 | } | |
f0e99437 ZW |
722 | err_req: |
723 | if (ret) | |
724 | workload->status = ret; | |
90d27a1b | 725 | mutex_unlock(&dev_priv->drm.struct_mutex); |
f25a49ab | 726 | mutex_unlock(&vgpu->vgpu_lock); |
e4734057 ZW |
727 | return ret; |
728 | } | |
729 | ||
730 | static struct intel_vgpu_workload *pick_next_workload( | |
731 | struct intel_gvt *gvt, int ring_id) | |
732 | { | |
733 | struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; | |
734 | struct intel_vgpu_workload *workload = NULL; | |
735 | ||
9a512e23 | 736 | mutex_lock(&gvt->sched_lock); |
e4734057 ZW |
737 | |
738 | /* | |
739 | * no current vgpu / will be scheduled out / no workload | |
740 | * bail out | |
741 | */ | |
742 | if (!scheduler->current_vgpu) { | |
743 | gvt_dbg_sched("ring id %d stop - no current vgpu\n", ring_id); | |
744 | goto out; | |
745 | } | |
746 | ||
747 | if (scheduler->need_reschedule) { | |
748 | gvt_dbg_sched("ring id %d stop - will reschedule\n", ring_id); | |
749 | goto out; | |
750 | } | |
751 | ||
9f498477 WL |
752 | if (!scheduler->current_vgpu->active || |
753 | list_empty(workload_q_head(scheduler->current_vgpu, ring_id))) | |
e4734057 | 754 | goto out; |
e4734057 ZW |
755 | |
756 | /* | |
757 | * still have current workload, maybe the workload disptacher | |
758 | * fail to submit it for some reason, resubmit it. | |
759 | */ | |
760 | if (scheduler->current_workload[ring_id]) { | |
761 | workload = scheduler->current_workload[ring_id]; | |
762 | gvt_dbg_sched("ring id %d still have current workload %p\n", | |
763 | ring_id, workload); | |
764 | goto out; | |
765 | } | |
766 | ||
767 | /* | |
768 | * pick a workload as current workload | |
769 | * once current workload is set, schedule policy routines | |
770 | * will wait the current workload is finished when trying to | |
771 | * schedule out a vgpu. | |
772 | */ | |
773 | scheduler->current_workload[ring_id] = container_of( | |
774 | workload_q_head(scheduler->current_vgpu, ring_id)->next, | |
775 | struct intel_vgpu_workload, list); | |
776 | ||
777 | workload = scheduler->current_workload[ring_id]; | |
778 | ||
779 | gvt_dbg_sched("ring id %d pick new workload %p\n", ring_id, workload); | |
780 | ||
1406a14b | 781 | atomic_inc(&workload->vgpu->submission.running_workload_num); |
e4734057 | 782 | out: |
9a512e23 | 783 | mutex_unlock(&gvt->sched_lock); |
e4734057 ZW |
784 | return workload; |
785 | } | |
786 | ||
787 | static void update_guest_context(struct intel_vgpu_workload *workload) | |
788 | { | |
1fc44d9b | 789 | struct i915_request *rq = workload->req; |
e4734057 ZW |
790 | struct intel_vgpu *vgpu = workload->vgpu; |
791 | struct intel_gvt *gvt = vgpu->gvt; | |
1fc44d9b | 792 | struct drm_i915_gem_object *ctx_obj = rq->hw_context->state->obj; |
e4734057 ZW |
793 | struct execlist_ring_context *shadow_ring_context; |
794 | struct page *page; | |
795 | void *src; | |
796 | unsigned long context_gpa, context_page_num; | |
797 | int i; | |
15e7f52a XZ |
798 | struct drm_i915_private *dev_priv = gvt->dev_priv; |
799 | u32 ring_base; | |
800 | u32 head, tail; | |
801 | u16 wrap_count; | |
e4734057 | 802 | |
1fc44d9b CW |
803 | gvt_dbg_sched("ring id %d workload lrca %x\n", rq->engine->id, |
804 | workload->ctx_desc.lrca); | |
e4734057 | 805 | |
15e7f52a XZ |
806 | head = workload->rb_head; |
807 | tail = workload->rb_tail; | |
808 | wrap_count = workload->guest_rb_head >> RB_HEAD_WRAP_CNT_OFF; | |
809 | ||
810 | if (tail < head) { | |
811 | if (wrap_count == RB_HEAD_WRAP_CNT_MAX) | |
812 | wrap_count = 0; | |
813 | else | |
814 | wrap_count += 1; | |
815 | } | |
816 | ||
817 | head = (wrap_count << RB_HEAD_WRAP_CNT_OFF) | tail; | |
818 | ||
819 | ring_base = dev_priv->engine[workload->ring_id]->mmio_base; | |
820 | vgpu_vreg_t(vgpu, RING_TAIL(ring_base)) = tail; | |
821 | vgpu_vreg_t(vgpu, RING_HEAD(ring_base)) = head; | |
822 | ||
1fc44d9b | 823 | context_page_num = rq->engine->context_size; |
e4734057 ZW |
824 | context_page_num = context_page_num >> PAGE_SHIFT; |
825 | ||
8a68d464 | 826 | if (IS_BROADWELL(gvt->dev_priv) && rq->engine->id == RCS0) |
e4734057 ZW |
827 | context_page_num = 19; |
828 | ||
829 | i = 2; | |
830 | ||
831 | while (i < context_page_num) { | |
832 | context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, | |
833 | (u32)((workload->ctx_desc.lrca + i) << | |
9556e118 | 834 | I915_GTT_PAGE_SHIFT)); |
e4734057 | 835 | if (context_gpa == INTEL_GVT_INVALID_ADDR) { |
695fbc08 | 836 | gvt_vgpu_err("invalid guest context descriptor\n"); |
e4734057 ZW |
837 | return; |
838 | } | |
839 | ||
0b29c75a | 840 | page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i); |
c754936f | 841 | src = kmap(page); |
e4734057 | 842 | intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src, |
9556e118 | 843 | I915_GTT_PAGE_SIZE); |
c754936f | 844 | kunmap(page); |
e4734057 ZW |
845 | i++; |
846 | } | |
847 | ||
848 | intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + | |
849 | RING_CTX_OFF(ring_header.val), &workload->rb_tail, 4); | |
850 | ||
851 | page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN); | |
c754936f | 852 | shadow_ring_context = kmap(page); |
e4734057 ZW |
853 | |
854 | #define COPY_REG(name) \ | |
855 | intel_gvt_hypervisor_write_gpa(vgpu, workload->ring_context_gpa + \ | |
856 | RING_CTX_OFF(name.val), &shadow_ring_context->name.val, 4) | |
857 | ||
858 | COPY_REG(ctx_ctrl); | |
859 | COPY_REG(ctx_timestamp); | |
860 | ||
861 | #undef COPY_REG | |
862 | ||
863 | intel_gvt_hypervisor_write_gpa(vgpu, | |
864 | workload->ring_context_gpa + | |
865 | sizeof(*shadow_ring_context), | |
866 | (void *)shadow_ring_context + | |
867 | sizeof(*shadow_ring_context), | |
9556e118 | 868 | I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context)); |
e4734057 | 869 | |
c754936f | 870 | kunmap(page); |
e4734057 ZW |
871 | } |
872 | ||
f9090d4c | 873 | void intel_vgpu_clean_workloads(struct intel_vgpu *vgpu, |
3a891a62 | 874 | intel_engine_mask_t engine_mask) |
e2c43c01 ZW |
875 | { |
876 | struct intel_vgpu_submission *s = &vgpu->submission; | |
877 | struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; | |
878 | struct intel_engine_cs *engine; | |
879 | struct intel_vgpu_workload *pos, *n; | |
3a891a62 | 880 | intel_engine_mask_t tmp; |
e2c43c01 ZW |
881 | |
882 | /* free the unsubmited workloads in the queues. */ | |
883 | for_each_engine_masked(engine, dev_priv, engine_mask, tmp) { | |
884 | list_for_each_entry_safe(pos, n, | |
885 | &s->workload_q_head[engine->id], list) { | |
886 | list_del_init(&pos->list); | |
887 | intel_vgpu_destroy_workload(pos); | |
888 | } | |
889 | clear_bit(engine->id, s->shadow_ctx_desc_updated); | |
890 | } | |
891 | } | |
892 | ||
e4734057 ZW |
893 | static void complete_current_workload(struct intel_gvt *gvt, int ring_id) |
894 | { | |
895 | struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; | |
1406a14b ZW |
896 | struct intel_vgpu_workload *workload = |
897 | scheduler->current_workload[ring_id]; | |
898 | struct intel_vgpu *vgpu = workload->vgpu; | |
899 | struct intel_vgpu_submission *s = &vgpu->submission; | |
6bb2a2af | 900 | struct i915_request *rq = workload->req; |
be1da707 | 901 | int event; |
e4734057 | 902 | |
f25a49ab | 903 | mutex_lock(&vgpu->vgpu_lock); |
9a512e23 | 904 | mutex_lock(&gvt->sched_lock); |
e4734057 | 905 | |
8f1117ab CD |
906 | /* For the workload w/ request, needs to wait for the context |
907 | * switch to make sure request is completed. | |
908 | * For the workload w/o request, directly complete the workload. | |
909 | */ | |
1fc44d9b | 910 | if (rq) { |
e4734057 ZW |
911 | wait_event(workload->shadow_ctx_status_wq, |
912 | !atomic_read(&workload->shadow_ctx_active)); | |
913 | ||
0cf5ec41 CD |
914 | /* If this request caused GPU hang, req->fence.error will |
915 | * be set to -EIO. Use -EIO to set workload status so | |
916 | * that when this request caused GPU hang, didn't trigger | |
917 | * context switch interrupt to guest. | |
918 | */ | |
919 | if (likely(workload->status == -EINPROGRESS)) { | |
920 | if (workload->req->fence.error == -EIO) | |
921 | workload->status = -EIO; | |
922 | else | |
923 | workload->status = 0; | |
924 | } | |
925 | ||
8a68d464 CW |
926 | if (!workload->status && |
927 | !(vgpu->resetting_eng & BIT(ring_id))) { | |
8f1117ab | 928 | update_guest_context(workload); |
be1da707 | 929 | |
8f1117ab CD |
930 | for_each_set_bit(event, workload->pending_events, |
931 | INTEL_GVT_EVENT_MAX) | |
932 | intel_vgpu_trigger_virtual_event(vgpu, event); | |
933 | } | |
1fc44d9b | 934 | |
6bb2a2af | 935 | i915_request_put(fetch_and_zero(&workload->req)); |
e4734057 ZW |
936 | } |
937 | ||
938 | gvt_dbg_sched("ring id %d complete workload %p status %d\n", | |
939 | ring_id, workload, workload->status); | |
940 | ||
941 | scheduler->current_workload[ring_id] = NULL; | |
942 | ||
e4734057 | 943 | list_del_init(&workload->list); |
d8235b5e | 944 | |
8a68d464 | 945 | if (workload->status || vgpu->resetting_eng & BIT(ring_id)) { |
e2c43c01 ZW |
946 | /* if workload->status is not successful means HW GPU |
947 | * has occurred GPU hang or something wrong with i915/GVT, | |
948 | * and GVT won't inject context switch interrupt to guest. | |
949 | * So this error is a vGPU hang actually to the guest. | |
950 | * According to this we should emunlate a vGPU hang. If | |
951 | * there are pending workloads which are already submitted | |
952 | * from guest, we should clean them up like HW GPU does. | |
953 | * | |
954 | * if it is in middle of engine resetting, the pending | |
955 | * workloads won't be submitted to HW GPU and will be | |
956 | * cleaned up during the resetting process later, so doing | |
957 | * the workload clean up here doesn't have any impact. | |
958 | **/ | |
8a68d464 | 959 | intel_vgpu_clean_workloads(vgpu, BIT(ring_id)); |
e2c43c01 ZW |
960 | } |
961 | ||
e4734057 ZW |
962 | workload->complete(workload); |
963 | ||
1406a14b | 964 | atomic_dec(&s->running_workload_num); |
e4734057 | 965 | wake_up(&scheduler->workload_complete_wq); |
f100daec PG |
966 | |
967 | if (gvt->scheduler.need_reschedule) | |
968 | intel_gvt_request_service(gvt, INTEL_GVT_REQUEST_EVENT_SCHED); | |
969 | ||
9a512e23 | 970 | mutex_unlock(&gvt->sched_lock); |
f25a49ab | 971 | mutex_unlock(&vgpu->vgpu_lock); |
e4734057 ZW |
972 | } |
973 | ||
974 | struct workload_thread_param { | |
975 | struct intel_gvt *gvt; | |
976 | int ring_id; | |
977 | }; | |
978 | ||
979 | static int workload_thread(void *priv) | |
980 | { | |
981 | struct workload_thread_param *p = (struct workload_thread_param *)priv; | |
982 | struct intel_gvt *gvt = p->gvt; | |
983 | int ring_id = p->ring_id; | |
984 | struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; | |
985 | struct intel_vgpu_workload *workload = NULL; | |
695fbc08 | 986 | struct intel_vgpu *vgpu = NULL; |
e4734057 | 987 | int ret; |
c3b5a843 | 988 | bool need_force_wake = (INTEL_GEN(gvt->dev_priv) >= 9); |
e45d7b7f | 989 | DEFINE_WAIT_FUNC(wait, woken_wake_function); |
ef5b0b44 | 990 | struct intel_runtime_pm *rpm = &gvt->dev_priv->runtime_pm; |
e4734057 ZW |
991 | |
992 | kfree(p); | |
993 | ||
994 | gvt_dbg_core("workload thread for ring %d started\n", ring_id); | |
995 | ||
996 | while (!kthread_should_stop()) { | |
e45d7b7f CD |
997 | add_wait_queue(&scheduler->waitq[ring_id], &wait); |
998 | do { | |
999 | workload = pick_next_workload(gvt, ring_id); | |
1000 | if (workload) | |
1001 | break; | |
1002 | wait_woken(&wait, TASK_INTERRUPTIBLE, | |
1003 | MAX_SCHEDULE_TIMEOUT); | |
1004 | } while (!kthread_should_stop()); | |
1005 | remove_wait_queue(&scheduler->waitq[ring_id], &wait); | |
1006 | ||
1007 | if (!workload) | |
e4734057 ZW |
1008 | break; |
1009 | ||
1010 | gvt_dbg_sched("ring id %d next workload %p vgpu %d\n", | |
1011 | workload->ring_id, workload, | |
1012 | workload->vgpu->id); | |
1013 | ||
ef5b0b44 XZ |
1014 | intel_runtime_pm_get(rpm); |
1015 | ||
e4734057 ZW |
1016 | gvt_dbg_sched("ring id %d will dispatch workload %p\n", |
1017 | workload->ring_id, workload); | |
1018 | ||
1019 | if (need_force_wake) | |
3ceea6a1 | 1020 | intel_uncore_forcewake_get(&gvt->dev_priv->uncore, |
e4734057 ZW |
1021 | FORCEWAKE_ALL); |
1022 | ||
1023 | ret = dispatch_workload(workload); | |
66bbc3b2 | 1024 | |
e4734057 | 1025 | if (ret) { |
695fbc08 TZ |
1026 | vgpu = workload->vgpu; |
1027 | gvt_vgpu_err("fail to dispatch workload, skip\n"); | |
e4734057 ZW |
1028 | goto complete; |
1029 | } | |
1030 | ||
1031 | gvt_dbg_sched("ring id %d wait workload %p\n", | |
1032 | workload->ring_id, workload); | |
e61e0f51 | 1033 | i915_request_wait(workload->req, 0, MAX_SCHEDULE_TIMEOUT); |
e4734057 ZW |
1034 | |
1035 | complete: | |
3ce3274b | 1036 | gvt_dbg_sched("will complete workload %p, status: %d\n", |
e4734057 ZW |
1037 | workload, workload->status); |
1038 | ||
2e51ef32 CD |
1039 | complete_current_workload(gvt, ring_id); |
1040 | ||
e4734057 | 1041 | if (need_force_wake) |
3ceea6a1 | 1042 | intel_uncore_forcewake_put(&gvt->dev_priv->uncore, |
e4734057 ZW |
1043 | FORCEWAKE_ALL); |
1044 | ||
ef5b0b44 | 1045 | intel_runtime_pm_put_unchecked(rpm); |
6d763035 | 1046 | if (ret && (vgpu_is_vm_unhealthy(ret))) |
e011c6ce | 1047 | enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR); |
e4734057 ZW |
1048 | } |
1049 | return 0; | |
1050 | } | |
1051 | ||
1052 | void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu) | |
1053 | { | |
1406a14b | 1054 | struct intel_vgpu_submission *s = &vgpu->submission; |
e4734057 ZW |
1055 | struct intel_gvt *gvt = vgpu->gvt; |
1056 | struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; | |
1057 | ||
1406a14b | 1058 | if (atomic_read(&s->running_workload_num)) { |
e4734057 ZW |
1059 | gvt_dbg_sched("wait vgpu idle\n"); |
1060 | ||
1061 | wait_event(scheduler->workload_complete_wq, | |
1406a14b | 1062 | !atomic_read(&s->running_workload_num)); |
e4734057 ZW |
1063 | } |
1064 | } | |
1065 | ||
1066 | void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt) | |
1067 | { | |
1068 | struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; | |
3fc03069 CD |
1069 | struct intel_engine_cs *engine; |
1070 | enum intel_engine_id i; | |
e4734057 ZW |
1071 | |
1072 | gvt_dbg_core("clean workload scheduler\n"); | |
1073 | ||
3fc03069 CD |
1074 | for_each_engine(engine, gvt->dev_priv, i) { |
1075 | atomic_notifier_chain_unregister( | |
1076 | &engine->context_status_notifier, | |
1077 | &gvt->shadow_ctx_notifier_block[i]); | |
1078 | kthread_stop(scheduler->thread[i]); | |
e4734057 ZW |
1079 | } |
1080 | } | |
1081 | ||
1082 | int intel_gvt_init_workload_scheduler(struct intel_gvt *gvt) | |
1083 | { | |
1084 | struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler; | |
1085 | struct workload_thread_param *param = NULL; | |
3fc03069 CD |
1086 | struct intel_engine_cs *engine; |
1087 | enum intel_engine_id i; | |
e4734057 | 1088 | int ret; |
e4734057 ZW |
1089 | |
1090 | gvt_dbg_core("init workload scheduler\n"); | |
1091 | ||
1092 | init_waitqueue_head(&scheduler->workload_complete_wq); | |
1093 | ||
3fc03069 | 1094 | for_each_engine(engine, gvt->dev_priv, i) { |
e4734057 ZW |
1095 | init_waitqueue_head(&scheduler->waitq[i]); |
1096 | ||
1097 | param = kzalloc(sizeof(*param), GFP_KERNEL); | |
1098 | if (!param) { | |
1099 | ret = -ENOMEM; | |
1100 | goto err; | |
1101 | } | |
1102 | ||
1103 | param->gvt = gvt; | |
1104 | param->ring_id = i; | |
1105 | ||
1106 | scheduler->thread[i] = kthread_run(workload_thread, param, | |
1107 | "gvt workload %d", i); | |
1108 | if (IS_ERR(scheduler->thread[i])) { | |
1109 | gvt_err("fail to create workload thread\n"); | |
1110 | ret = PTR_ERR(scheduler->thread[i]); | |
1111 | goto err; | |
1112 | } | |
3fc03069 CD |
1113 | |
1114 | gvt->shadow_ctx_notifier_block[i].notifier_call = | |
1115 | shadow_context_status_change; | |
1116 | atomic_notifier_chain_register(&engine->context_status_notifier, | |
1117 | &gvt->shadow_ctx_notifier_block[i]); | |
e4734057 ZW |
1118 | } |
1119 | return 0; | |
1120 | err: | |
1121 | intel_gvt_clean_workload_scheduler(gvt); | |
1122 | kfree(param); | |
1123 | param = NULL; | |
1124 | return ret; | |
1125 | } | |
1126 | ||
f39a89b8 | 1127 | static void |
251d46b0 | 1128 | i915_context_ppgtt_root_restore(struct intel_vgpu_submission *s, |
ab53497b | 1129 | struct i915_ppgtt *ppgtt) |
f39a89b8 | 1130 | { |
f39a89b8 XZ |
1131 | int i; |
1132 | ||
251d46b0 | 1133 | if (i915_vm_is_4lvl(&ppgtt->vm)) { |
b5b7bef9 | 1134 | px_dma(ppgtt->pd) = s->i915_context_pml4; |
a9fe9ca4 | 1135 | } else { |
b5b7bef9 MK |
1136 | for (i = 0; i < GEN8_3LVL_PDPES; i++) { |
1137 | struct i915_page_directory * const pd = | |
1138 | i915_pd_entry(ppgtt->pd, i); | |
1139 | ||
1140 | px_dma(pd) = s->i915_context_pdps[i]; | |
1141 | } | |
f39a89b8 XZ |
1142 | } |
1143 | } | |
1144 | ||
874b6a91 ZW |
1145 | /** |
1146 | * intel_vgpu_clean_submission - free submission-related resource for vGPU | |
1147 | * @vgpu: a vGPU | |
1148 | * | |
1149 | * This function is called when a vGPU is being destroyed. | |
1150 | * | |
1151 | */ | |
1152 | void intel_vgpu_clean_submission(struct intel_vgpu *vgpu) | |
e4734057 | 1153 | { |
1406a14b | 1154 | struct intel_vgpu_submission *s = &vgpu->submission; |
251d46b0 CW |
1155 | struct intel_engine_cs *engine; |
1156 | enum intel_engine_id id; | |
1406a14b | 1157 | |
7569a06d | 1158 | intel_vgpu_select_submission_ops(vgpu, ALL_ENGINES, 0); |
251d46b0 | 1159 | |
e568ac38 | 1160 | i915_context_ppgtt_root_restore(s, i915_vm_to_ppgtt(s->shadow[0]->gem_context->vm)); |
251d46b0 CW |
1161 | for_each_engine(engine, vgpu->gvt->dev_priv, id) |
1162 | intel_context_unpin(s->shadow[id]); | |
1163 | ||
1406a14b | 1164 | kmem_cache_destroy(s->workloads); |
e4734057 ZW |
1165 | } |
1166 | ||
06bb372f ZW |
1167 | |
1168 | /** | |
1169 | * intel_vgpu_reset_submission - reset submission-related resource for vGPU | |
1170 | * @vgpu: a vGPU | |
1171 | * @engine_mask: engines expected to be reset | |
1172 | * | |
1173 | * This function is called when a vGPU is being destroyed. | |
1174 | * | |
1175 | */ | |
1176 | void intel_vgpu_reset_submission(struct intel_vgpu *vgpu, | |
3a891a62 | 1177 | intel_engine_mask_t engine_mask) |
06bb372f ZW |
1178 | { |
1179 | struct intel_vgpu_submission *s = &vgpu->submission; | |
1180 | ||
1181 | if (!s->active) | |
1182 | return; | |
1183 | ||
f9090d4c | 1184 | intel_vgpu_clean_workloads(vgpu, engine_mask); |
06bb372f ZW |
1185 | s->ops->reset(vgpu, engine_mask); |
1186 | } | |
1187 | ||
f39a89b8 | 1188 | static void |
251d46b0 | 1189 | i915_context_ppgtt_root_save(struct intel_vgpu_submission *s, |
ab53497b | 1190 | struct i915_ppgtt *ppgtt) |
f39a89b8 | 1191 | { |
f39a89b8 XZ |
1192 | int i; |
1193 | ||
251d46b0 | 1194 | if (i915_vm_is_4lvl(&ppgtt->vm)) { |
b5b7bef9 | 1195 | s->i915_context_pml4 = px_dma(ppgtt->pd); |
251d46b0 | 1196 | } else { |
b5b7bef9 MK |
1197 | for (i = 0; i < GEN8_3LVL_PDPES; i++) { |
1198 | struct i915_page_directory * const pd = | |
1199 | i915_pd_entry(ppgtt->pd, i); | |
1200 | ||
1201 | s->i915_context_pdps[i] = px_dma(pd); | |
1202 | } | |
f39a89b8 XZ |
1203 | } |
1204 | } | |
1205 | ||
874b6a91 ZW |
1206 | /** |
1207 | * intel_vgpu_setup_submission - setup submission-related resource for vGPU | |
1208 | * @vgpu: a vGPU | |
1209 | * | |
1210 | * This function is called when a vGPU is being created. | |
1211 | * | |
1212 | * Returns: | |
1213 | * Zero on success, negative error code if failed. | |
1214 | * | |
1215 | */ | |
1216 | int intel_vgpu_setup_submission(struct intel_vgpu *vgpu) | |
e4734057 | 1217 | { |
1406a14b | 1218 | struct intel_vgpu_submission *s = &vgpu->submission; |
9a9829e9 | 1219 | struct intel_engine_cs *engine; |
251d46b0 CW |
1220 | struct i915_gem_context *ctx; |
1221 | enum intel_engine_id i; | |
9a9829e9 | 1222 | int ret; |
e4734057 | 1223 | |
251d46b0 CW |
1224 | ctx = i915_gem_context_create_gvt(&vgpu->gvt->dev_priv->drm); |
1225 | if (IS_ERR(ctx)) | |
1226 | return PTR_ERR(ctx); | |
1227 | ||
e568ac38 | 1228 | i915_context_ppgtt_root_save(s, i915_vm_to_ppgtt(ctx->vm)); |
e4734057 | 1229 | |
251d46b0 CW |
1230 | for_each_engine(engine, vgpu->gvt->dev_priv, i) { |
1231 | struct intel_context *ce; | |
1232 | ||
1233 | INIT_LIST_HEAD(&s->workload_q_head[i]); | |
1234 | s->shadow[i] = ERR_PTR(-EINVAL); | |
1235 | ||
5e2a0419 | 1236 | ce = i915_gem_context_get_engine(ctx, i); |
251d46b0 CW |
1237 | if (IS_ERR(ce)) { |
1238 | ret = PTR_ERR(ce); | |
1239 | goto out_shadow_ctx; | |
1240 | } | |
e4734057 | 1241 | |
fa9f6681 CW |
1242 | ret = intel_context_pin(ce); |
1243 | intel_context_put(ce); | |
1244 | if (ret) | |
1245 | goto out_shadow_ctx; | |
1246 | ||
251d46b0 CW |
1247 | s->shadow[i] = ce; |
1248 | } | |
f39a89b8 | 1249 | |
1406a14b | 1250 | bitmap_zero(s->shadow_ctx_desc_updated, I915_NUM_ENGINES); |
9dfb8e5b | 1251 | |
850555d1 ZW |
1252 | s->workloads = kmem_cache_create_usercopy("gvt-g_vgpu_workload", |
1253 | sizeof(struct intel_vgpu_workload), 0, | |
1254 | SLAB_HWCACHE_ALIGN, | |
1255 | offsetof(struct intel_vgpu_workload, rb_tail), | |
1256 | sizeof_field(struct intel_vgpu_workload, rb_tail), | |
1257 | NULL); | |
9a9829e9 | 1258 | |
1406a14b | 1259 | if (!s->workloads) { |
9a9829e9 ZW |
1260 | ret = -ENOMEM; |
1261 | goto out_shadow_ctx; | |
1262 | } | |
1263 | ||
1406a14b | 1264 | atomic_set(&s->running_workload_num, 0); |
91d5d854 | 1265 | bitmap_zero(s->tlb_handle_pending, I915_NUM_ENGINES); |
9a9829e9 | 1266 | |
251d46b0 | 1267 | i915_gem_context_put(ctx); |
e4734057 | 1268 | return 0; |
9a9829e9 ZW |
1269 | |
1270 | out_shadow_ctx: | |
e568ac38 | 1271 | i915_context_ppgtt_root_restore(s, i915_vm_to_ppgtt(ctx->vm)); |
251d46b0 CW |
1272 | for_each_engine(engine, vgpu->gvt->dev_priv, i) { |
1273 | if (IS_ERR(s->shadow[i])) | |
1274 | break; | |
1275 | ||
1276 | intel_context_unpin(s->shadow[i]); | |
1277 | } | |
1278 | i915_gem_context_put(ctx); | |
9a9829e9 | 1279 | return ret; |
e4734057 | 1280 | } |
21527a8d | 1281 | |
ad1d3636 ZW |
1282 | /** |
1283 | * intel_vgpu_select_submission_ops - select virtual submission interface | |
1284 | * @vgpu: a vGPU | |
a752b070 | 1285 | * @engine_mask: either ALL_ENGINES or target engine mask |
ad1d3636 ZW |
1286 | * @interface: expected vGPU virtual submission interface |
1287 | * | |
1288 | * This function is called when guest configures submission interface. | |
1289 | * | |
1290 | * Returns: | |
1291 | * Zero on success, negative error code if failed. | |
1292 | * | |
1293 | */ | |
1294 | int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu, | |
3a891a62 | 1295 | intel_engine_mask_t engine_mask, |
ad1d3636 ZW |
1296 | unsigned int interface) |
1297 | { | |
1298 | struct intel_vgpu_submission *s = &vgpu->submission; | |
1299 | const struct intel_vgpu_submission_ops *ops[] = { | |
1300 | [INTEL_VGPU_EXECLIST_SUBMISSION] = | |
1301 | &intel_vgpu_execlist_submission_ops, | |
1302 | }; | |
1303 | int ret; | |
1304 | ||
1305 | if (WARN_ON(interface >= ARRAY_SIZE(ops))) | |
1306 | return -EINVAL; | |
1307 | ||
9212b13f WL |
1308 | if (WARN_ON(interface == 0 && engine_mask != ALL_ENGINES)) |
1309 | return -EINVAL; | |
1310 | ||
1311 | if (s->active) | |
7569a06d | 1312 | s->ops->clean(vgpu, engine_mask); |
ad1d3636 ZW |
1313 | |
1314 | if (interface == 0) { | |
1315 | s->ops = NULL; | |
1316 | s->virtual_submission_interface = 0; | |
9212b13f WL |
1317 | s->active = false; |
1318 | gvt_dbg_core("vgpu%d: remove submission ops\n", vgpu->id); | |
ad1d3636 ZW |
1319 | return 0; |
1320 | } | |
1321 | ||
7569a06d | 1322 | ret = ops[interface]->init(vgpu, engine_mask); |
ad1d3636 ZW |
1323 | if (ret) |
1324 | return ret; | |
1325 | ||
1326 | s->ops = ops[interface]; | |
1327 | s->virtual_submission_interface = interface; | |
1328 | s->active = true; | |
1329 | ||
1330 | gvt_dbg_core("vgpu%d: activate ops [ %s ]\n", | |
1331 | vgpu->id, s->ops->name); | |
1332 | ||
1333 | return 0; | |
1334 | } | |
1335 | ||
21527a8d ZW |
1336 | /** |
1337 | * intel_vgpu_destroy_workload - destroy a vGPU workload | |
a752b070 | 1338 | * @workload: workload to destroy |
21527a8d ZW |
1339 | * |
1340 | * This function is called when destroy a vGPU workload. | |
1341 | * | |
1342 | */ | |
1343 | void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload) | |
1344 | { | |
1345 | struct intel_vgpu_submission *s = &workload->vgpu->submission; | |
1346 | ||
0f755512 WL |
1347 | release_shadow_batch_buffer(workload); |
1348 | release_shadow_wa_ctx(&workload->wa_ctx); | |
1349 | ||
21527a8d | 1350 | if (workload->shadow_mm) |
1bc25851 | 1351 | intel_vgpu_mm_put(workload->shadow_mm); |
21527a8d ZW |
1352 | |
1353 | kmem_cache_free(s->workloads, workload); | |
1354 | } | |
1355 | ||
6d763035 ZW |
1356 | static struct intel_vgpu_workload * |
1357 | alloc_workload(struct intel_vgpu *vgpu) | |
1358 | { | |
1359 | struct intel_vgpu_submission *s = &vgpu->submission; | |
1360 | struct intel_vgpu_workload *workload; | |
1361 | ||
1362 | workload = kmem_cache_zalloc(s->workloads, GFP_KERNEL); | |
1363 | if (!workload) | |
1364 | return ERR_PTR(-ENOMEM); | |
1365 | ||
1366 | INIT_LIST_HEAD(&workload->list); | |
1367 | INIT_LIST_HEAD(&workload->shadow_bb); | |
1368 | ||
1369 | init_waitqueue_head(&workload->shadow_ctx_status_wq); | |
1370 | atomic_set(&workload->shadow_ctx_active, 0); | |
1371 | ||
1372 | workload->status = -EINPROGRESS; | |
6d763035 ZW |
1373 | workload->vgpu = vgpu; |
1374 | ||
1375 | return workload; | |
1376 | } | |
1377 | ||
1378 | #define RING_CTX_OFF(x) \ | |
1379 | offsetof(struct execlist_ring_context, x) | |
1380 | ||
1381 | static void read_guest_pdps(struct intel_vgpu *vgpu, | |
1382 | u64 ring_context_gpa, u32 pdp[8]) | |
1383 | { | |
1384 | u64 gpa; | |
1385 | int i; | |
1386 | ||
1417fad7 | 1387 | gpa = ring_context_gpa + RING_CTX_OFF(pdps[0].val); |
6d763035 ZW |
1388 | |
1389 | for (i = 0; i < 8; i++) | |
1390 | intel_gvt_hypervisor_read_gpa(vgpu, | |
1391 | gpa + i * 8, &pdp[7 - i], 4); | |
1392 | } | |
1393 | ||
1394 | static int prepare_mm(struct intel_vgpu_workload *workload) | |
1395 | { | |
1396 | struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc; | |
1397 | struct intel_vgpu_mm *mm; | |
1398 | struct intel_vgpu *vgpu = workload->vgpu; | |
0cf8f58d | 1399 | enum intel_gvt_gtt_type root_entry_type; |
ede9d0cf | 1400 | u64 pdps[GVT_RING_CTX_NR_PDPS]; |
6d763035 | 1401 | |
ede9d0cf CD |
1402 | switch (desc->addressing_mode) { |
1403 | case 1: /* legacy 32-bit */ | |
1404 | root_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY; | |
1405 | break; | |
1406 | case 3: /* legacy 64-bit */ | |
1407 | root_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY; | |
1408 | break; | |
1409 | default: | |
6d763035 ZW |
1410 | gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n"); |
1411 | return -EINVAL; | |
1412 | } | |
1413 | ||
ede9d0cf | 1414 | read_guest_pdps(workload->vgpu, workload->ring_context_gpa, (void *)pdps); |
6d763035 | 1415 | |
e6e9c46f CD |
1416 | mm = intel_vgpu_get_ppgtt_mm(workload->vgpu, root_entry_type, pdps); |
1417 | if (IS_ERR(mm)) | |
1418 | return PTR_ERR(mm); | |
1419 | ||
6d763035 ZW |
1420 | workload->shadow_mm = mm; |
1421 | return 0; | |
1422 | } | |
1423 | ||
1424 | #define same_context(a, b) (((a)->context_id == (b)->context_id) && \ | |
1425 | ((a)->lrca == (b)->lrca)) | |
1426 | ||
21527a8d ZW |
1427 | /** |
1428 | * intel_vgpu_create_workload - create a vGPU workload | |
1429 | * @vgpu: a vGPU | |
a752b070 | 1430 | * @ring_id: ring index |
6d763035 | 1431 | * @desc: a guest context descriptor |
21527a8d ZW |
1432 | * |
1433 | * This function is called when creating a vGPU workload. | |
1434 | * | |
1435 | * Returns: | |
1436 | * struct intel_vgpu_workload * on success, negative error code in | |
1437 | * pointer if failed. | |
1438 | * | |
1439 | */ | |
1440 | struct intel_vgpu_workload * | |
6d763035 ZW |
1441 | intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id, |
1442 | struct execlist_ctx_descriptor_format *desc) | |
21527a8d ZW |
1443 | { |
1444 | struct intel_vgpu_submission *s = &vgpu->submission; | |
6d763035 | 1445 | struct list_head *q = workload_q_head(vgpu, ring_id); |
80a47263 | 1446 | struct intel_vgpu_workload *last_workload = NULL; |
6d763035 ZW |
1447 | struct intel_vgpu_workload *workload = NULL; |
1448 | struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; | |
1449 | u64 ring_context_gpa; | |
1450 | u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx; | |
15e7f52a | 1451 | u32 guest_head; |
6d763035 | 1452 | int ret; |
21527a8d | 1453 | |
6d763035 | 1454 | ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm, |
9556e118 | 1455 | (u32)((desc->lrca + 1) << I915_GTT_PAGE_SHIFT)); |
6d763035 ZW |
1456 | if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) { |
1457 | gvt_vgpu_err("invalid guest context LRCA: %x\n", desc->lrca); | |
1458 | return ERR_PTR(-EINVAL); | |
1459 | } | |
21527a8d | 1460 | |
6d763035 ZW |
1461 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + |
1462 | RING_CTX_OFF(ring_header.val), &head, 4); | |
21527a8d | 1463 | |
6d763035 ZW |
1464 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + |
1465 | RING_CTX_OFF(ring_tail.val), &tail, 4); | |
21527a8d | 1466 | |
15e7f52a XZ |
1467 | guest_head = head; |
1468 | ||
6d763035 ZW |
1469 | head &= RB_HEAD_OFF_MASK; |
1470 | tail &= RB_TAIL_OFF_MASK; | |
1471 | ||
80a47263 XZ |
1472 | list_for_each_entry_reverse(last_workload, q, list) { |
1473 | ||
1474 | if (same_context(&last_workload->ctx_desc, desc)) { | |
1475 | gvt_dbg_el("ring id %d cur workload == last\n", | |
1476 | ring_id); | |
1477 | gvt_dbg_el("ctx head %x real head %lx\n", head, | |
1478 | last_workload->rb_tail); | |
1479 | /* | |
1480 | * cannot use guest context head pointer here, | |
1481 | * as it might not be updated at this time | |
1482 | */ | |
1483 | head = last_workload->rb_tail; | |
1484 | break; | |
1485 | } | |
6d763035 ZW |
1486 | } |
1487 | ||
1488 | gvt_dbg_el("ring id %d begin a new workload\n", ring_id); | |
1489 | ||
1490 | /* record some ring buffer register values for scan and shadow */ | |
1491 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
1492 | RING_CTX_OFF(rb_start.val), &start, 4); | |
1493 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
1494 | RING_CTX_OFF(rb_ctrl.val), &ctl, 4); | |
1495 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
1496 | RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4); | |
1497 | ||
2089a76a XZ |
1498 | if (!intel_gvt_ggtt_validate_range(vgpu, start, |
1499 | _RING_CTL_BUF_SIZE(ctl))) { | |
1500 | gvt_vgpu_err("context contain invalid rb at: 0x%x\n", start); | |
1501 | return ERR_PTR(-EINVAL); | |
1502 | } | |
1503 | ||
6d763035 ZW |
1504 | workload = alloc_workload(vgpu); |
1505 | if (IS_ERR(workload)) | |
1506 | return workload; | |
1507 | ||
1508 | workload->ring_id = ring_id; | |
1509 | workload->ctx_desc = *desc; | |
1510 | workload->ring_context_gpa = ring_context_gpa; | |
1511 | workload->rb_head = head; | |
15e7f52a | 1512 | workload->guest_rb_head = guest_head; |
6d763035 ZW |
1513 | workload->rb_tail = tail; |
1514 | workload->rb_start = start; | |
1515 | workload->rb_ctl = ctl; | |
1516 | ||
8a68d464 | 1517 | if (ring_id == RCS0) { |
6d763035 ZW |
1518 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + |
1519 | RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4); | |
1520 | intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa + | |
1521 | RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4); | |
1522 | ||
1523 | workload->wa_ctx.indirect_ctx.guest_gma = | |
1524 | indirect_ctx & INDIRECT_CTX_ADDR_MASK; | |
1525 | workload->wa_ctx.indirect_ctx.size = | |
1526 | (indirect_ctx & INDIRECT_CTX_SIZE_MASK) * | |
1527 | CACHELINE_BYTES; | |
2089a76a XZ |
1528 | |
1529 | if (workload->wa_ctx.indirect_ctx.size != 0) { | |
1530 | if (!intel_gvt_ggtt_validate_range(vgpu, | |
1531 | workload->wa_ctx.indirect_ctx.guest_gma, | |
1532 | workload->wa_ctx.indirect_ctx.size)) { | |
2089a76a XZ |
1533 | gvt_vgpu_err("invalid wa_ctx at: 0x%lx\n", |
1534 | workload->wa_ctx.indirect_ctx.guest_gma); | |
eac4471d | 1535 | kmem_cache_free(s->workloads, workload); |
2089a76a XZ |
1536 | return ERR_PTR(-EINVAL); |
1537 | } | |
1538 | } | |
1539 | ||
6d763035 ZW |
1540 | workload->wa_ctx.per_ctx.guest_gma = |
1541 | per_ctx & PER_CTX_ADDR_MASK; | |
1542 | workload->wa_ctx.per_ctx.valid = per_ctx & 1; | |
2089a76a XZ |
1543 | if (workload->wa_ctx.per_ctx.valid) { |
1544 | if (!intel_gvt_ggtt_validate_range(vgpu, | |
1545 | workload->wa_ctx.per_ctx.guest_gma, | |
1546 | CACHELINE_BYTES)) { | |
2089a76a XZ |
1547 | gvt_vgpu_err("invalid per_ctx at: 0x%lx\n", |
1548 | workload->wa_ctx.per_ctx.guest_gma); | |
eac4471d | 1549 | kmem_cache_free(s->workloads, workload); |
2089a76a XZ |
1550 | return ERR_PTR(-EINVAL); |
1551 | } | |
1552 | } | |
6d763035 ZW |
1553 | } |
1554 | ||
1555 | gvt_dbg_el("workload %p ring id %d head %x tail %x start %x ctl %x\n", | |
1556 | workload, ring_id, head, tail, start, ctl); | |
1557 | ||
1558 | ret = prepare_mm(workload); | |
1559 | if (ret) { | |
1560 | kmem_cache_free(s->workloads, workload); | |
1561 | return ERR_PTR(ret); | |
1562 | } | |
1563 | ||
1564 | /* Only scan and shadow the first workload in the queue | |
1565 | * as there is only one pre-allocated buf-obj for shadow. | |
1566 | */ | |
1567 | if (list_empty(workload_q_head(vgpu, ring_id))) { | |
d858d569 | 1568 | intel_runtime_pm_get(&dev_priv->runtime_pm); |
6d763035 ZW |
1569 | mutex_lock(&dev_priv->drm.struct_mutex); |
1570 | ret = intel_gvt_scan_and_shadow_workload(workload); | |
1571 | mutex_unlock(&dev_priv->drm.struct_mutex); | |
d858d569 | 1572 | intel_runtime_pm_put_unchecked(&dev_priv->runtime_pm); |
6d763035 ZW |
1573 | } |
1574 | ||
dade58ed YZ |
1575 | if (ret) { |
1576 | if (vgpu_is_vm_unhealthy(ret)) | |
1577 | enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR); | |
6d763035 ZW |
1578 | intel_vgpu_destroy_workload(workload); |
1579 | return ERR_PTR(ret); | |
1580 | } | |
21527a8d ZW |
1581 | |
1582 | return workload; | |
1583 | } | |
59a716c6 CD |
1584 | |
1585 | /** | |
1586 | * intel_vgpu_queue_workload - Qeue a vGPU workload | |
1587 | * @workload: the workload to queue in | |
1588 | */ | |
1589 | void intel_vgpu_queue_workload(struct intel_vgpu_workload *workload) | |
1590 | { | |
1591 | list_add_tail(&workload->list, | |
1592 | workload_q_head(workload->vgpu, workload->ring_id)); | |
c130456c | 1593 | intel_gvt_kick_schedule(workload->vgpu->gvt); |
59a716c6 CD |
1594 | wake_up(&workload->vgpu->gvt->scheduler.waitq[workload->ring_id]); |
1595 | } |