]>
Commit | Line | Data |
---|---|---|
e39c5add ZW |
1 | /* |
2 | * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice (including the next | |
12 | * paragraph) shall be included in all copies or substantial portions of the | |
13 | * Software. | |
14 | * | |
15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | |
19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
20 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
21 | * SOFTWARE. | |
22 | * | |
23 | * Authors: | |
24 | * Ke Yu | |
25 | * Kevin Tian <kevin.tian@intel.com> | |
26 | * Dexuan Cui | |
27 | * | |
28 | * Contributors: | |
29 | * Tina Zhang <tina.zhang@intel.com> | |
30 | * Min He <min.he@intel.com> | |
31 | * Niu Bing <bing.niu@intel.com> | |
32 | * Zhi Wang <zhi.a.wang@intel.com> | |
33 | * | |
34 | */ | |
35 | ||
36 | #include "i915_drv.h" | |
feddf6e8 | 37 | #include "gvt.h" |
e39c5add ZW |
38 | |
39 | /** | |
40 | * intel_vgpu_gpa_to_mmio_offset - translate a GPA to MMIO offset | |
41 | * @vgpu: a vGPU | |
42 | * | |
43 | * Returns: | |
44 | * Zero on success, negative error code if failed | |
45 | */ | |
46 | int intel_vgpu_gpa_to_mmio_offset(struct intel_vgpu *vgpu, u64 gpa) | |
47 | { | |
f090a00d | 48 | u64 gttmmio_gpa = intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0); |
e39c5add ZW |
49 | return gpa - gttmmio_gpa; |
50 | } | |
51 | ||
52 | #define reg_is_mmio(gvt, reg) \ | |
53 | (reg >= 0 && reg < gvt->device_info.mmio_size) | |
54 | ||
55 | #define reg_is_gtt(gvt, reg) \ | |
56 | (reg >= gvt->device_info.gtt_start_offset \ | |
57 | && reg < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) | |
58 | ||
f090a00d CD |
59 | static bool vgpu_gpa_is_aperture(struct intel_vgpu *vgpu, uint64_t gpa) |
60 | { | |
61 | u64 aperture_gpa = intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_2); | |
62 | u64 aperture_sz = vgpu_aperture_sz(vgpu); | |
63 | ||
64 | return gpa >= aperture_gpa && gpa < aperture_gpa + aperture_sz; | |
65 | } | |
66 | ||
67 | static int vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t gpa, | |
68 | void *pdata, unsigned int size, bool is_read) | |
69 | { | |
70 | u64 aperture_gpa = intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_2); | |
71 | u64 offset = gpa - aperture_gpa; | |
72 | ||
73 | if (!vgpu_gpa_is_aperture(vgpu, gpa + size - 1)) { | |
74 | gvt_vgpu_err("Aperture rw out of range, offset %llx, size %d\n", | |
75 | offset, size); | |
76 | return -EINVAL; | |
77 | } | |
78 | ||
79 | if (!vgpu->gm.aperture_va) { | |
80 | gvt_vgpu_err("BAR is not enabled\n"); | |
81 | return -ENXIO; | |
82 | } | |
83 | ||
84 | if (is_read) | |
85 | memcpy(pdata, vgpu->gm.aperture_va + offset, size); | |
86 | else | |
87 | memcpy(vgpu->gm.aperture_va + offset, pdata, size); | |
88 | return 0; | |
89 | } | |
90 | ||
fd64be63 MH |
91 | static void failsafe_emulate_mmio_rw(struct intel_vgpu *vgpu, uint64_t pa, |
92 | void *p_data, unsigned int bytes, bool read) | |
93 | { | |
94 | struct intel_gvt *gvt = NULL; | |
95 | void *pt = NULL; | |
96 | unsigned int offset = 0; | |
97 | ||
98 | if (!vgpu || !p_data) | |
99 | return; | |
100 | ||
101 | gvt = vgpu->gvt; | |
102 | mutex_lock(&gvt->lock); | |
103 | offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa); | |
104 | if (reg_is_mmio(gvt, offset)) { | |
105 | if (read) | |
106 | intel_vgpu_default_mmio_read(vgpu, offset, p_data, | |
107 | bytes); | |
108 | else | |
109 | intel_vgpu_default_mmio_write(vgpu, offset, p_data, | |
110 | bytes); | |
111 | } else if (reg_is_gtt(gvt, offset) && | |
112 | vgpu->gtt.ggtt_mm->virtual_page_table) { | |
113 | offset -= gvt->device_info.gtt_start_offset; | |
114 | pt = vgpu->gtt.ggtt_mm->virtual_page_table + offset; | |
115 | if (read) | |
116 | memcpy(p_data, pt, bytes); | |
117 | else | |
118 | memcpy(pt, p_data, bytes); | |
119 | ||
120 | } else if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) { | |
121 | struct intel_vgpu_guest_page *gp; | |
122 | ||
123 | /* Since we enter the failsafe mode early during guest boot, | |
124 | * guest may not have chance to set up its ppgtt table, so | |
125 | * there should not be any wp pages for guest. Keep the wp | |
126 | * related code here in case we need to handle it in furture. | |
127 | */ | |
128 | gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT); | |
129 | if (gp) { | |
130 | /* remove write protection to prevent furture traps */ | |
131 | intel_vgpu_clean_guest_page(vgpu, gp); | |
132 | if (read) | |
133 | intel_gvt_hypervisor_read_gpa(vgpu, pa, | |
134 | p_data, bytes); | |
135 | else | |
136 | intel_gvt_hypervisor_write_gpa(vgpu, pa, | |
137 | p_data, bytes); | |
138 | } | |
139 | } | |
140 | mutex_unlock(&gvt->lock); | |
141 | } | |
142 | ||
e39c5add ZW |
143 | /** |
144 | * intel_vgpu_emulate_mmio_read - emulate MMIO read | |
145 | * @vgpu: a vGPU | |
146 | * @pa: guest physical address | |
147 | * @p_data: data return buffer | |
148 | * @bytes: access data length | |
149 | * | |
150 | * Returns: | |
151 | * Zero on success, negative error code if failed | |
152 | */ | |
9ec1e66b | 153 | int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa, |
e39c5add ZW |
154 | void *p_data, unsigned int bytes) |
155 | { | |
e39c5add | 156 | struct intel_gvt *gvt = vgpu->gvt; |
e39c5add ZW |
157 | unsigned int offset = 0; |
158 | int ret = -EINVAL; | |
159 | ||
fd64be63 MH |
160 | |
161 | if (vgpu->failsafe) { | |
162 | failsafe_emulate_mmio_rw(vgpu, pa, p_data, bytes, true); | |
163 | return 0; | |
164 | } | |
e39c5add ZW |
165 | mutex_lock(&gvt->lock); |
166 | ||
f090a00d CD |
167 | if (vgpu_gpa_is_aperture(vgpu, pa)) { |
168 | ret = vgpu_aperture_rw(vgpu, pa, p_data, bytes, true); | |
169 | mutex_unlock(&gvt->lock); | |
170 | return ret; | |
171 | } | |
172 | ||
e39c5add ZW |
173 | if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) { |
174 | struct intel_vgpu_guest_page *gp; | |
175 | ||
176 | gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT); | |
177 | if (gp) { | |
178 | ret = intel_gvt_hypervisor_read_gpa(vgpu, pa, | |
179 | p_data, bytes); | |
180 | if (ret) { | |
695fbc08 | 181 | gvt_vgpu_err("guest page read error %d, " |
e39c5add | 182 | "gfn 0x%lx, pa 0x%llx, var 0x%x, len %d\n", |
695fbc08 TZ |
183 | ret, gp->gfn, pa, *(u32 *)p_data, |
184 | bytes); | |
e39c5add ZW |
185 | } |
186 | mutex_unlock(&gvt->lock); | |
187 | return ret; | |
188 | } | |
189 | } | |
190 | ||
191 | offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa); | |
192 | ||
193 | if (WARN_ON(bytes > 8)) | |
194 | goto err; | |
195 | ||
196 | if (reg_is_gtt(gvt, offset)) { | |
197 | if (WARN_ON(!IS_ALIGNED(offset, 4) && !IS_ALIGNED(offset, 8))) | |
198 | goto err; | |
199 | if (WARN_ON(bytes != 4 && bytes != 8)) | |
200 | goto err; | |
201 | if (WARN_ON(!reg_is_gtt(gvt, offset + bytes - 1))) | |
202 | goto err; | |
203 | ||
204 | ret = intel_vgpu_emulate_gtt_mmio_read(vgpu, offset, | |
205 | p_data, bytes); | |
206 | if (ret) | |
207 | goto err; | |
208 | mutex_unlock(&gvt->lock); | |
209 | return ret; | |
210 | } | |
211 | ||
212 | if (WARN_ON_ONCE(!reg_is_mmio(gvt, offset))) { | |
213 | ret = intel_gvt_hypervisor_read_gpa(vgpu, pa, p_data, bytes); | |
214 | mutex_unlock(&gvt->lock); | |
215 | return ret; | |
216 | } | |
217 | ||
218 | if (WARN_ON(!reg_is_mmio(gvt, offset + bytes - 1))) | |
219 | goto err; | |
220 | ||
e39c5add ZW |
221 | if (!intel_gvt_mmio_is_unalign(gvt, offset)) { |
222 | if (WARN_ON(!IS_ALIGNED(offset, bytes))) | |
223 | goto err; | |
224 | } | |
225 | ||
65f9f6fe CD |
226 | ret = intel_vgpu_mmio_reg_rw(vgpu, offset, p_data, bytes, true); |
227 | if (ret < 0) | |
e39c5add ZW |
228 | goto err; |
229 | ||
230 | intel_gvt_mmio_set_accessed(gvt, offset); | |
231 | mutex_unlock(&gvt->lock); | |
232 | return 0; | |
233 | err: | |
695fbc08 TZ |
234 | gvt_vgpu_err("fail to emulate MMIO read %08x len %d\n", |
235 | offset, bytes); | |
e39c5add ZW |
236 | mutex_unlock(&gvt->lock); |
237 | return ret; | |
238 | } | |
239 | ||
240 | /** | |
241 | * intel_vgpu_emulate_mmio_write - emulate MMIO write | |
242 | * @vgpu: a vGPU | |
243 | * @pa: guest physical address | |
244 | * @p_data: write data buffer | |
245 | * @bytes: access data length | |
246 | * | |
247 | * Returns: | |
248 | * Zero on success, negative error code if failed | |
249 | */ | |
9ec1e66b | 250 | int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa, |
e39c5add ZW |
251 | void *p_data, unsigned int bytes) |
252 | { | |
e39c5add | 253 | struct intel_gvt *gvt = vgpu->gvt; |
e39c5add | 254 | unsigned int offset = 0; |
e39c5add ZW |
255 | int ret = -EINVAL; |
256 | ||
fd64be63 MH |
257 | if (vgpu->failsafe) { |
258 | failsafe_emulate_mmio_rw(vgpu, pa, p_data, bytes, false); | |
259 | return 0; | |
260 | } | |
261 | ||
e39c5add ZW |
262 | mutex_lock(&gvt->lock); |
263 | ||
f090a00d CD |
264 | if (vgpu_gpa_is_aperture(vgpu, pa)) { |
265 | ret = vgpu_aperture_rw(vgpu, pa, p_data, bytes, false); | |
266 | mutex_unlock(&gvt->lock); | |
267 | return ret; | |
268 | } | |
269 | ||
e39c5add ZW |
270 | if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) { |
271 | struct intel_vgpu_guest_page *gp; | |
272 | ||
273 | gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT); | |
274 | if (gp) { | |
275 | ret = gp->handler(gp, pa, p_data, bytes); | |
276 | if (ret) { | |
695fbc08 TZ |
277 | gvt_err("guest page write error %d, " |
278 | "gfn 0x%lx, pa 0x%llx, " | |
279 | "var 0x%x, len %d\n", | |
280 | ret, gp->gfn, pa, | |
281 | *(u32 *)p_data, bytes); | |
e39c5add ZW |
282 | } |
283 | mutex_unlock(&gvt->lock); | |
284 | return ret; | |
285 | } | |
286 | } | |
287 | ||
288 | offset = intel_vgpu_gpa_to_mmio_offset(vgpu, pa); | |
289 | ||
290 | if (WARN_ON(bytes > 8)) | |
291 | goto err; | |
292 | ||
293 | if (reg_is_gtt(gvt, offset)) { | |
294 | if (WARN_ON(!IS_ALIGNED(offset, 4) && !IS_ALIGNED(offset, 8))) | |
295 | goto err; | |
296 | if (WARN_ON(bytes != 4 && bytes != 8)) | |
297 | goto err; | |
298 | if (WARN_ON(!reg_is_gtt(gvt, offset + bytes - 1))) | |
299 | goto err; | |
300 | ||
301 | ret = intel_vgpu_emulate_gtt_mmio_write(vgpu, offset, | |
302 | p_data, bytes); | |
303 | if (ret) | |
304 | goto err; | |
305 | mutex_unlock(&gvt->lock); | |
306 | return ret; | |
307 | } | |
308 | ||
309 | if (WARN_ON_ONCE(!reg_is_mmio(gvt, offset))) { | |
310 | ret = intel_gvt_hypervisor_write_gpa(vgpu, pa, p_data, bytes); | |
311 | mutex_unlock(&gvt->lock); | |
312 | return ret; | |
313 | } | |
314 | ||
65f9f6fe CD |
315 | ret = intel_vgpu_mmio_reg_rw(vgpu, offset, p_data, bytes, false); |
316 | if (ret < 0) | |
e39c5add | 317 | goto err; |
65f9f6fe | 318 | |
e39c5add ZW |
319 | intel_gvt_mmio_set_accessed(gvt, offset); |
320 | mutex_unlock(&gvt->lock); | |
321 | return 0; | |
322 | err: | |
695fbc08 TZ |
323 | gvt_vgpu_err("fail to emulate MMIO write %08x len %d\n", offset, |
324 | bytes); | |
e39c5add ZW |
325 | mutex_unlock(&gvt->lock); |
326 | return ret; | |
327 | } | |
cdcc4347 | 328 | |
97d58f7d CD |
329 | |
330 | /** | |
331 | * intel_vgpu_reset_mmio - reset virtual MMIO space | |
332 | * @vgpu: a vGPU | |
333 | * | |
334 | */ | |
615c16a9 | 335 | void intel_vgpu_reset_mmio(struct intel_vgpu *vgpu, bool dmlr) |
97d58f7d CD |
336 | { |
337 | struct intel_gvt *gvt = vgpu->gvt; | |
338 | const struct intel_gvt_device_info *info = &gvt->device_info; | |
615c16a9 | 339 | void *mmio = gvt->firmware.mmio; |
97d58f7d | 340 | |
615c16a9 | 341 | if (dmlr) { |
342 | memcpy(vgpu->mmio.vreg, mmio, info->mmio_size); | |
343 | memcpy(vgpu->mmio.sreg, mmio, info->mmio_size); | |
97d58f7d | 344 | |
615c16a9 | 345 | vgpu_vreg(vgpu, GEN6_GT_THREAD_STATUS_REG) = 0; |
97d58f7d | 346 | |
615c16a9 | 347 | /* set the bit 0:2(Core C-State ) to C0 */ |
348 | vgpu_vreg(vgpu, GEN6_GT_CORE_STATUS) = 0; | |
349 | ||
350 | vgpu->mmio.disable_warn_untrack = false; | |
351 | } else { | |
352 | #define GVT_GEN8_MMIO_RESET_OFFSET (0x44200) | |
353 | /* only reset the engine related, so starting with 0x44200 | |
354 | * interrupt include DE,display mmio related will not be | |
355 | * touched | |
356 | */ | |
357 | memcpy(vgpu->mmio.vreg, mmio, GVT_GEN8_MMIO_RESET_OFFSET); | |
358 | memcpy(vgpu->mmio.sreg, mmio, GVT_GEN8_MMIO_RESET_OFFSET); | |
359 | } | |
d1be371d | 360 | |
97d58f7d CD |
361 | } |
362 | ||
cdcc4347 CD |
363 | /** |
364 | * intel_vgpu_init_mmio - init MMIO space | |
365 | * @vgpu: a vGPU | |
366 | * | |
367 | * Returns: | |
368 | * Zero on success, negative error code if failed | |
369 | */ | |
370 | int intel_vgpu_init_mmio(struct intel_vgpu *vgpu) | |
371 | { | |
372 | const struct intel_gvt_device_info *info = &vgpu->gvt->device_info; | |
373 | ||
97d58f7d CD |
374 | vgpu->mmio.vreg = vzalloc(info->mmio_size * 2); |
375 | if (!vgpu->mmio.vreg) | |
376 | return -ENOMEM; | |
cdcc4347 | 377 | |
97d58f7d | 378 | vgpu->mmio.sreg = vgpu->mmio.vreg + info->mmio_size; |
cdcc4347 | 379 | |
615c16a9 | 380 | intel_vgpu_reset_mmio(vgpu, true); |
cdcc4347 CD |
381 | |
382 | return 0; | |
383 | } | |
384 | ||
385 | /** | |
386 | * intel_vgpu_clean_mmio - clean MMIO space | |
387 | * @vgpu: a vGPU | |
388 | * | |
389 | */ | |
390 | void intel_vgpu_clean_mmio(struct intel_vgpu *vgpu) | |
391 | { | |
392 | vfree(vgpu->mmio.vreg); | |
393 | vgpu->mmio.vreg = vgpu->mmio.sreg = NULL; | |
394 | } |