]> git.proxmox.com Git - mirror_qemu.git/blame - hw/i386/intel_iommu.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
intel_iommu: get rid of {0} initializers
[mirror_qemu.git] / hw / i386 / intel_iommu.c
CommitLineData
1da12ec4
LT		 1/*
2 * QEMU emulation of an Intel IOMMU (VT-d)
3 * (DMA Remapping device)
4 *
5 * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com>
6 * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, see <http://www.gnu.org/licenses/>.
20 */
21
b6a0aa05 22#include "qemu/osdep.h"
1da12ec4
LT		 23#include "hw/sysbus.h"
24#include "exec/address-spaces.h"
25#include "intel_iommu_internal.h"
7df953bd 26#include "hw/pci/pci.h"
3cb3b154 27#include "hw/pci/pci_bus.h"
621d983a 28#include "hw/i386/pc.h"
04af0e18
PX		 29#include "hw/boards.h"
30#include "hw/i386/x86-iommu.h"
1da12ec4
LT		 31
32/*#define DEBUG_INTEL_IOMMU*/
33#ifdef DEBUG_INTEL_IOMMU
34enum {
35 DEBUG_GENERAL, DEBUG_CSR, DEBUG_INV, DEBUG_MMU, DEBUG_FLOG,
a5861439 36 DEBUG_CACHE, DEBUG_IR,
1da12ec4
LT		 37};
38#define VTD_DBGBIT(x) (1 << DEBUG_##x)
39static int vtd_dbgflags = VTD_DBGBIT(GENERAL) | VTD_DBGBIT(CSR);
40
41#define VTD_DPRINTF(what, fmt, ...) do { \
42 if (vtd_dbgflags & VTD_DBGBIT(what)) { \
43 fprintf(stderr, "(vtd)%s: " fmt "\n", __func__, \
44 ## __VA_ARGS__); } \
45 } while (0)
46#else
47#define VTD_DPRINTF(what, fmt, ...) do {} while (0)
48#endif
49
50static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val,
51 uint64_t wmask, uint64_t w1cmask)
52{
53 stq_le_p(&s->csr[addr], val);
54 stq_le_p(&s->wmask[addr], wmask);
55 stq_le_p(&s->w1cmask[addr], w1cmask);
56}
57
58static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask)
59{
60 stq_le_p(&s->womask[addr], mask);
61}
62
63static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val,
64 uint32_t wmask, uint32_t w1cmask)
65{
66 stl_le_p(&s->csr[addr], val);
67 stl_le_p(&s->wmask[addr], wmask);
68 stl_le_p(&s->w1cmask[addr], w1cmask);
69}
70
71static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask)
72{
73 stl_le_p(&s->womask[addr], mask);
74}
75
76/* "External" get/set operations */
77static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val)
78{
79 uint64_t oldval = ldq_le_p(&s->csr[addr]);
80 uint64_t wmask = ldq_le_p(&s->wmask[addr]);
81 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]);
82 stq_le_p(&s->csr[addr],
83 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val));
84}
85
86static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val)
87{
88 uint32_t oldval = ldl_le_p(&s->csr[addr]);
89 uint32_t wmask = ldl_le_p(&s->wmask[addr]);
90 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]);
91 stl_le_p(&s->csr[addr],
92 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val));
93}
94
95static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr)
96{
97 uint64_t val = ldq_le_p(&s->csr[addr]);
98 uint64_t womask = ldq_le_p(&s->womask[addr]);
99 return val & ~womask;
100}
101
102static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr)
103{
104 uint32_t val = ldl_le_p(&s->csr[addr]);
105 uint32_t womask = ldl_le_p(&s->womask[addr]);
106 return val & ~womask;
107}
108
109/* "Internal" get/set operations */
110static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr)
111{
112 return ldq_le_p(&s->csr[addr]);
113}
114
115static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr)
116{
117 return ldl_le_p(&s->csr[addr]);
118}
119
120static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val)
121{
122 stq_le_p(&s->csr[addr], val);
123}
124
125static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr,
126 uint32_t clear, uint32_t mask)
127{
128 uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask;
129 stl_le_p(&s->csr[addr], new_val);
130 return new_val;
131}
132
133static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr,
134 uint64_t clear, uint64_t mask)
135{
136 uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask;
137 stq_le_p(&s->csr[addr], new_val);
138 return new_val;
139}
140
b5a280c0
LT		 141/* GHashTable functions */
142static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2)
143{
144 return *((const uint64_t *)v1) == *((const uint64_t *)v2);
145}
146
147static guint vtd_uint64_hash(gconstpointer v)
148{
149 return (guint)*(const uint64_t *)v;
150}
151
152static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value,
153 gpointer user_data)
154{
155 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
156 uint16_t domain_id = *(uint16_t *)user_data;
157 return entry->domain_id == domain_id;
158}
159
d66b969b
JW		 160/* The shift of an addr for a certain level of paging structure */
161static inline uint32_t vtd_slpt_level_shift(uint32_t level)
162{
163 return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS;
164}
165
166static inline uint64_t vtd_slpt_level_page_mask(uint32_t level)
167{
168 return ~((1ULL << vtd_slpt_level_shift(level)) - 1);
169}
170
b5a280c0
LT		 171static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value,
172 gpointer user_data)
173{
174 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
175 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data;
d66b969b
JW		 176 uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask;
177 uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K;
b5a280c0 178 return (entry->domain_id == info->domain_id) &&
d66b969b
JW		 179 (((entry->gfn & info->mask) == gfn) ||
180 (entry->gfn == gfn_tlb));
b5a280c0
LT		 181}
182
d92fa2dc
LT		 183/* Reset all the gen of VTDAddressSpace to zero and set the gen of
184 * IntelIOMMUState to 1.
185 */
186static void vtd_reset_context_cache(IntelIOMMUState *s)
187{
d92fa2dc 188 VTDAddressSpace *vtd_as;
7df953bd
KO		 189 VTDBus *vtd_bus;
190 GHashTableIter bus_it;
d92fa2dc
LT		 191 uint32_t devfn_it;
192
7df953bd
KO		 193 g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr);
194
d92fa2dc 195 VTD_DPRINTF(CACHE, "global context_cache_gen=1");
7df953bd 196 while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) {
04af0e18 197 for (devfn_it = 0; devfn_it < X86_IOMMU_PCI_DEVFN_MAX; ++devfn_it) {
7df953bd 198 vtd_as = vtd_bus->dev_as[devfn_it];
d92fa2dc
LT		 199 if (!vtd_as) {
200 continue;
201 }
202 vtd_as->context_cache_entry.context_cache_gen = 0;
203 }
204 }
205 s->context_cache_gen = 1;
206}
207
b5a280c0
LT		 208static void vtd_reset_iotlb(IntelIOMMUState *s)
209{
210 assert(s->iotlb);
211 g_hash_table_remove_all(s->iotlb);
212}
213
d66b969b
JW		 214static uint64_t vtd_get_iotlb_key(uint64_t gfn, uint8_t source_id,
215 uint32_t level)
216{
217 return gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT) |
218 ((uint64_t)(level) << VTD_IOTLB_LVL_SHIFT);
219}
220
221static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level)
222{
223 return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K;
224}
225
b5a280c0
LT		 226static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id,
227 hwaddr addr)
228{
d66b969b 229 VTDIOTLBEntry *entry;
b5a280c0 230 uint64_t key;
d66b969b
JW		 231 int level;
232
233 for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) {
234 key = vtd_get_iotlb_key(vtd_get_iotlb_gfn(addr, level),
235 source_id, level);
236 entry = g_hash_table_lookup(s->iotlb, &key);
237 if (entry) {
238 goto out;
239 }
240 }
b5a280c0 241
d66b969b
JW		 242out:
243 return entry;
b5a280c0
LT		 244}
245
246static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id,
247 uint16_t domain_id, hwaddr addr, uint64_t slpte,
d66b969b
JW		 248 bool read_flags, bool write_flags,
249 uint32_t level)
b5a280c0
LT		 250{
251 VTDIOTLBEntry *entry = g_malloc(sizeof(*entry));
252 uint64_t *key = g_malloc(sizeof(*key));
d66b969b 253 uint64_t gfn = vtd_get_iotlb_gfn(addr, level);
b5a280c0
LT		 254
255 VTD_DPRINTF(CACHE, "update iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64
256 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr, slpte,
257 domain_id);
258 if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) {
259 VTD_DPRINTF(CACHE, "iotlb exceeds size limit, forced to reset");
260 vtd_reset_iotlb(s);
261 }
262
263 entry->gfn = gfn;
264 entry->domain_id = domain_id;
265 entry->slpte = slpte;
266 entry->read_flags = read_flags;
267 entry->write_flags = write_flags;
d66b969b
JW		 268 entry->mask = vtd_slpt_level_page_mask(level);
269 *key = vtd_get_iotlb_key(gfn, source_id, level);
b5a280c0
LT		 270 g_hash_table_replace(s->iotlb, key, entry);
271}
272
1da12ec4
LT		 273/* Given the reg addr of both the message data and address, generate an
274 * interrupt via MSI.
275 */
276static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg,
277 hwaddr mesg_data_reg)
278{
279 hwaddr addr;
280 uint32_t data;
281
282 assert(mesg_data_reg < DMAR_REG_SIZE);
283 assert(mesg_addr_reg < DMAR_REG_SIZE);
284
285 addr = vtd_get_long_raw(s, mesg_addr_reg);
286 data = vtd_get_long_raw(s, mesg_data_reg);
287
288 VTD_DPRINTF(FLOG, "msi: addr 0x%"PRIx64 " data 0x%"PRIx32, addr, data);
42874d3a
PM		 289 address_space_stl_le(&address_space_memory, addr, data,
290 MEMTXATTRS_UNSPECIFIED, NULL);
1da12ec4
LT		 291}
292
293/* Generate a fault event to software via MSI if conditions are met.
294 * Notice that the value of FSTS_REG being passed to it should be the one
295 * before any update.
296 */
297static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts)
298{
299 if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO ||
300 pre_fsts & VTD_FSTS_IQE) {
301 VTD_DPRINTF(FLOG, "there are previous interrupt conditions "
302 "to be serviced by software, fault event is not generated "
303 "(FSTS_REG 0x%"PRIx32 ")", pre_fsts);
304 return;
305 }
306 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP);
307 if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) {
308 VTD_DPRINTF(FLOG, "Interrupt Mask set, fault event is not generated");
309 } else {
310 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
311 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
312 }
313}
314
315/* Check if the Fault (F) field of the Fault Recording Register referenced by
316 * @index is Set.
317 */
318static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index)
319{
320 /* Each reg is 128-bit */
321 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
322 addr += 8; /* Access the high 64-bit half */
323
324 assert(index < DMAR_FRCD_REG_NR);
325
326 return vtd_get_quad_raw(s, addr) & VTD_FRCD_F;
327}
328
329/* Update the PPF field of Fault Status Register.
330 * Should be called whenever change the F field of any fault recording
331 * registers.
332 */
333static void vtd_update_fsts_ppf(IntelIOMMUState *s)
334{
335 uint32_t i;
336 uint32_t ppf_mask = 0;
337
338 for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
339 if (vtd_is_frcd_set(s, i)) {
340 ppf_mask = VTD_FSTS_PPF;
341 break;
342 }
343 }
344 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask);
345 VTD_DPRINTF(FLOG, "set PPF of FSTS_REG to %d", ppf_mask ? 1 : 0);
346}
347
348static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index)
349{
350 /* Each reg is 128-bit */
351 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
352 addr += 8; /* Access the high 64-bit half */
353
354 assert(index < DMAR_FRCD_REG_NR);
355
356 vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F);
357 vtd_update_fsts_ppf(s);
358}
359
360/* Must not update F field now, should be done later */
361static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index,
362 uint16_t source_id, hwaddr addr,
363 VTDFaultReason fault, bool is_write)
364{
365 uint64_t hi = 0, lo;
366 hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
367
368 assert(index < DMAR_FRCD_REG_NR);
369
370 lo = VTD_FRCD_FI(addr);
371 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault);
372 if (!is_write) {
373 hi |= VTD_FRCD_T;
374 }
375 vtd_set_quad_raw(s, frcd_reg_addr, lo);
376 vtd_set_quad_raw(s, frcd_reg_addr + 8, hi);
377 VTD_DPRINTF(FLOG, "record to FRCD_REG #%"PRIu16 ": hi 0x%"PRIx64
378 ", lo 0x%"PRIx64, index, hi, lo);
379}
380
381/* Try to collapse multiple pending faults from the same requester */
382static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id)
383{
384 uint32_t i;
385 uint64_t frcd_reg;
386 hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */
387
388 for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
389 frcd_reg = vtd_get_quad_raw(s, addr);
390 VTD_DPRINTF(FLOG, "frcd_reg #%d 0x%"PRIx64, i, frcd_reg);
391 if ((frcd_reg & VTD_FRCD_F) &&
392 ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) {
393 return true;
394 }
395 addr += 16; /* 128-bit for each */
396 }
397 return false;
398}
399
400/* Log and report an DMAR (address translation) fault to software */
401static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id,
402 hwaddr addr, VTDFaultReason fault,
403 bool is_write)
404{
405 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
406
407 assert(fault < VTD_FR_MAX);
408
409 if (fault == VTD_FR_RESERVED_ERR) {
410 /* This is not a normal fault reason case. Drop it. */
411 return;
412 }
413 VTD_DPRINTF(FLOG, "sid 0x%"PRIx16 ", fault %d, addr 0x%"PRIx64
414 ", is_write %d", source_id, fault, addr, is_write);
415 if (fsts_reg & VTD_FSTS_PFO) {
416 VTD_DPRINTF(FLOG, "new fault is not recorded due to "
417 "Primary Fault Overflow");
418 return;
419 }
420 if (vtd_try_collapse_fault(s, source_id)) {
421 VTD_DPRINTF(FLOG, "new fault is not recorded due to "
422 "compression of faults");
423 return;
424 }
425 if (vtd_is_frcd_set(s, s->next_frcd_reg)) {
426 VTD_DPRINTF(FLOG, "Primary Fault Overflow and "
427 "new fault is not recorded, set PFO field");
428 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO);
429 return;
430 }
431
432 vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write);
433
434 if (fsts_reg & VTD_FSTS_PPF) {
435 VTD_DPRINTF(FLOG, "there are pending faults already, "
436 "fault event is not generated");
437 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg);
438 s->next_frcd_reg++;
439 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
440 s->next_frcd_reg = 0;
441 }
442 } else {
443 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK,
444 VTD_FSTS_FRI(s->next_frcd_reg));
445 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */
446 s->next_frcd_reg++;
447 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
448 s->next_frcd_reg = 0;
449 }
450 /* This case actually cause the PPF to be Set.
451 * So generate fault event (interrupt).
452 */
453 vtd_generate_fault_event(s, fsts_reg);
454 }
455}
456
ed7b8fbc
LT		 457/* Handle Invalidation Queue Errors of queued invalidation interface error
458 * conditions.
459 */
460static void vtd_handle_inv_queue_error(IntelIOMMUState *s)
461{
462 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
463
464 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE);
465 vtd_generate_fault_event(s, fsts_reg);
466}
467
468/* Set the IWC field and try to generate an invalidation completion interrupt */
469static void vtd_generate_completion_event(IntelIOMMUState *s)
470{
471 VTD_DPRINTF(INV, "completes an invalidation wait command with "
472 "Interrupt Flag");
473 if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) {
474 VTD_DPRINTF(INV, "there is a previous interrupt condition to be "
475 "serviced by software, "
476 "new invalidation event is not generated");
477 return;
478 }
479 vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC);
480 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP);
481 if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) {
482 VTD_DPRINTF(INV, "IM filed in IECTL_REG is set, new invalidation "
483 "event is not generated");
484 return;
485 } else {
486 /* Generate the interrupt event */
487 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
488 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
489 }
490}
491
1da12ec4
LT		 492static inline bool vtd_root_entry_present(VTDRootEntry *root)
493{
494 return root->val & VTD_ROOT_ENTRY_P;
495}
496
497static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index,
498 VTDRootEntry *re)
499{
500 dma_addr_t addr;
501
502 addr = s->root + index * sizeof(*re);
503 if (dma_memory_read(&address_space_memory, addr, re, sizeof(*re))) {
504 VTD_DPRINTF(GENERAL, "error: fail to access root-entry at 0x%"PRIx64
505 " + %"PRIu8, s->root, index);
506 re->val = 0;
507 return -VTD_FR_ROOT_TABLE_INV;
508 }
509 re->val = le64_to_cpu(re->val);
510 return 0;
511}
512
513static inline bool vtd_context_entry_present(VTDContextEntry *context)
514{
515 return context->lo & VTD_CONTEXT_ENTRY_P;
516}
517
518static int vtd_get_context_entry_from_root(VTDRootEntry *root, uint8_t index,
519 VTDContextEntry *ce)
520{
521 dma_addr_t addr;
522
523 if (!vtd_root_entry_present(root)) {
524 VTD_DPRINTF(GENERAL, "error: root-entry is not present");
525 return -VTD_FR_ROOT_ENTRY_P;
526 }
527 addr = (root->val & VTD_ROOT_ENTRY_CTP) + index * sizeof(*ce);
528 if (dma_memory_read(&address_space_memory, addr, ce, sizeof(*ce))) {
529 VTD_DPRINTF(GENERAL, "error: fail to access context-entry at 0x%"PRIx64
530 " + %"PRIu8,
531 (uint64_t)(root->val & VTD_ROOT_ENTRY_CTP), index);
532 return -VTD_FR_CONTEXT_TABLE_INV;
533 }
534 ce->lo = le64_to_cpu(ce->lo);
535 ce->hi = le64_to_cpu(ce->hi);
536 return 0;
537}
538
539static inline dma_addr_t vtd_get_slpt_base_from_context(VTDContextEntry *ce)
540{
541 return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR;
542}
543
1da12ec4
LT		 544static inline uint64_t vtd_get_slpte_addr(uint64_t slpte)
545{
546 return slpte & VTD_SL_PT_BASE_ADDR_MASK;
547}
548
549/* Whether the pte indicates the address of the page frame */
550static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level)
551{
552 return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK);
553}
554
555/* Get the content of a spte located in @base_addr[@index] */
556static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index)
557{
558 uint64_t slpte;
559
560 assert(index < VTD_SL_PT_ENTRY_NR);
561
562 if (dma_memory_read(&address_space_memory,
563 base_addr + index * sizeof(slpte), &slpte,
564 sizeof(slpte))) {
565 slpte = (uint64_t)-1;
566 return slpte;
567 }
568 slpte = le64_to_cpu(slpte);
569 return slpte;
570}
571
572/* Given a gpa and the level of paging structure, return the offset of current
573 * level.
574 */
575static inline uint32_t vtd_gpa_level_offset(uint64_t gpa, uint32_t level)
576{
577 return (gpa >> vtd_slpt_level_shift(level)) &
578 ((1ULL << VTD_SL_LEVEL_BITS) - 1);
579}
580
581/* Check Capability Register to see if the @level of page-table is supported */
582static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level)
583{
584 return VTD_CAP_SAGAW_MASK & s->cap &
585 (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT));
586}
587
588/* Get the page-table level that hardware should use for the second-level
589 * page-table walk from the Address Width field of context-entry.
590 */
591static inline uint32_t vtd_get_level_from_context_entry(VTDContextEntry *ce)
592{
593 return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW);
594}
595
596static inline uint32_t vtd_get_agaw_from_context_entry(VTDContextEntry *ce)
597{
598 return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9;
599}
600
601static const uint64_t vtd_paging_entry_rsvd_field[] = {
602 [0] = ~0ULL,
603 /* For not large page */
604 [1] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
605 [2] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
606 [3] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
607 [4] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
608 /* For large page */
609 [5] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
610 [6] = 0x1ff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
611 [7] = 0x3ffff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
612 [8] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
613};
614
615static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level)
616{
617 if (slpte & VTD_SL_PT_PAGE_SIZE_MASK) {
618 /* Maybe large page */
619 return slpte & vtd_paging_entry_rsvd_field[level + 4];
620 } else {
621 return slpte & vtd_paging_entry_rsvd_field[level];
622 }
623}
624
625/* Given the @gpa, get relevant @slptep. @slpte_level will be the last level
626 * of the translation, can be used for deciding the size of large page.
627 */
628static int vtd_gpa_to_slpte(VTDContextEntry *ce, uint64_t gpa, bool is_write,
629 uint64_t *slptep, uint32_t *slpte_level,
630 bool *reads, bool *writes)
631{
632 dma_addr_t addr = vtd_get_slpt_base_from_context(ce);
633 uint32_t level = vtd_get_level_from_context_entry(ce);
634 uint32_t offset;
635 uint64_t slpte;
636 uint32_t ce_agaw = vtd_get_agaw_from_context_entry(ce);
637 uint64_t access_right_check;
638
639 /* Check if @gpa is above 2^X-1, where X is the minimum of MGAW in CAP_REG
640 * and AW in context-entry.
641 */
642 if (gpa & ~((1ULL << MIN(ce_agaw, VTD_MGAW)) - 1)) {
643 VTD_DPRINTF(GENERAL, "error: gpa 0x%"PRIx64 " exceeds limits", gpa);
644 return -VTD_FR_ADDR_BEYOND_MGAW;
645 }
646
647 /* FIXME: what is the Atomics request here? */
648 access_right_check = is_write ? VTD_SL_W : VTD_SL_R;
649
650 while (true) {
651 offset = vtd_gpa_level_offset(gpa, level);
652 slpte = vtd_get_slpte(addr, offset);
653
654 if (slpte == (uint64_t)-1) {
655 VTD_DPRINTF(GENERAL, "error: fail to access second-level paging "
656 "entry at level %"PRIu32 " for gpa 0x%"PRIx64,
657 level, gpa);
658 if (level == vtd_get_level_from_context_entry(ce)) {
659 /* Invalid programming of context-entry */
660 return -VTD_FR_CONTEXT_ENTRY_INV;
661 } else {
662 return -VTD_FR_PAGING_ENTRY_INV;
663 }
664 }
665 *reads = (*reads) && (slpte & VTD_SL_R);
666 *writes = (*writes) && (slpte & VTD_SL_W);
667 if (!(slpte & access_right_check)) {
668 VTD_DPRINTF(GENERAL, "error: lack of %s permission for "
669 "gpa 0x%"PRIx64 " slpte 0x%"PRIx64,
670 (is_write ? "write" : "read"), gpa, slpte);
671 return is_write ? -VTD_FR_WRITE : -VTD_FR_READ;
672 }
673 if (vtd_slpte_nonzero_rsvd(slpte, level)) {
674 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in second "
675 "level paging entry level %"PRIu32 " slpte 0x%"PRIx64,
676 level, slpte);
677 return -VTD_FR_PAGING_ENTRY_RSVD;
678 }
679
680 if (vtd_is_last_slpte(slpte, level)) {
681 *slptep = slpte;
682 *slpte_level = level;
683 return 0;
684 }
685 addr = vtd_get_slpte_addr(slpte);
686 level--;
687 }
688}
689
690/* Map a device to its corresponding domain (context-entry) */
691static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num,
692 uint8_t devfn, VTDContextEntry *ce)
693{
694 VTDRootEntry re;
695 int ret_fr;
696
697 ret_fr = vtd_get_root_entry(s, bus_num, &re);
698 if (ret_fr) {
699 return ret_fr;
700 }
701
702 if (!vtd_root_entry_present(&re)) {
703 VTD_DPRINTF(GENERAL, "error: root-entry #%"PRIu8 " is not present",
704 bus_num);
705 return -VTD_FR_ROOT_ENTRY_P;
706 } else if (re.rsvd || (re.val & VTD_ROOT_ENTRY_RSVD)) {
707 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in root-entry "
708 "hi 0x%"PRIx64 " lo 0x%"PRIx64, re.rsvd, re.val);
709 return -VTD_FR_ROOT_ENTRY_RSVD;
710 }
711
712 ret_fr = vtd_get_context_entry_from_root(&re, devfn, ce);
713 if (ret_fr) {
714 return ret_fr;
715 }
716
717 if (!vtd_context_entry_present(ce)) {
718 VTD_DPRINTF(GENERAL,
719 "error: context-entry #%"PRIu8 "(bus #%"PRIu8 ") "
720 "is not present", devfn, bus_num);
721 return -VTD_FR_CONTEXT_ENTRY_P;
722 } else if ((ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI) ||
723 (ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO)) {
724 VTD_DPRINTF(GENERAL,
725 "error: non-zero reserved field in context-entry "
726 "hi 0x%"PRIx64 " lo 0x%"PRIx64, ce->hi, ce->lo);
727 return -VTD_FR_CONTEXT_ENTRY_RSVD;
728 }
729 /* Check if the programming of context-entry is valid */
730 if (!vtd_is_level_supported(s, vtd_get_level_from_context_entry(ce))) {
731 VTD_DPRINTF(GENERAL, "error: unsupported Address Width value in "
732 "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64,
733 ce->hi, ce->lo);
734 return -VTD_FR_CONTEXT_ENTRY_INV;
735 } else if (ce->lo & VTD_CONTEXT_ENTRY_TT) {
736 VTD_DPRINTF(GENERAL, "error: unsupported Translation Type in "
737 "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64,
738 ce->hi, ce->lo);
739 return -VTD_FR_CONTEXT_ENTRY_INV;
740 }
741 return 0;
742}
743
744static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn)
745{
746 return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL);
747}
748
749static const bool vtd_qualified_faults[] = {
750 [VTD_FR_RESERVED] = false,
751 [VTD_FR_ROOT_ENTRY_P] = false,
752 [VTD_FR_CONTEXT_ENTRY_P] = true,
753 [VTD_FR_CONTEXT_ENTRY_INV] = true,
754 [VTD_FR_ADDR_BEYOND_MGAW] = true,
755 [VTD_FR_WRITE] = true,
756 [VTD_FR_READ] = true,
757 [VTD_FR_PAGING_ENTRY_INV] = true,
758 [VTD_FR_ROOT_TABLE_INV] = false,
759 [VTD_FR_CONTEXT_TABLE_INV] = false,
760 [VTD_FR_ROOT_ENTRY_RSVD] = false,
761 [VTD_FR_PAGING_ENTRY_RSVD] = true,
762 [VTD_FR_CONTEXT_ENTRY_TT] = true,
763 [VTD_FR_RESERVED_ERR] = false,
764 [VTD_FR_MAX] = false,
765};
766
767/* To see if a fault condition is "qualified", which is reported to software
768 * only if the FPD field in the context-entry used to process the faulting
769 * request is 0.
770 */
771static inline bool vtd_is_qualified_fault(VTDFaultReason fault)
772{
773 return vtd_qualified_faults[fault];
774}
775
776static inline bool vtd_is_interrupt_addr(hwaddr addr)
777{
778 return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST;
779}
780
781/* Map dev to context-entry then do a paging-structures walk to do a iommu
782 * translation.
79e2b9ae
PB		 783 *
784 * Called from RCU critical section.
785 *
1da12ec4
LT		 786 * @bus_num: The bus number
787 * @devfn: The devfn, which is the combined of device and function number
788 * @is_write: The access is a write operation
789 * @entry: IOMMUTLBEntry that contain the addr to be translated and result
790 */
7df953bd 791static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus,
1da12ec4
LT		 792 uint8_t devfn, hwaddr addr, bool is_write,
793 IOMMUTLBEntry *entry)
794{
d92fa2dc 795 IntelIOMMUState *s = vtd_as->iommu_state;
1da12ec4 796 VTDContextEntry ce;
7df953bd 797 uint8_t bus_num = pci_bus_num(bus);
d92fa2dc 798 VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry;
d66b969b 799 uint64_t slpte, page_mask;
1da12ec4
LT		 800 uint32_t level;
801 uint16_t source_id = vtd_make_source_id(bus_num, devfn);
802 int ret_fr;
803 bool is_fpd_set = false;
804 bool reads = true;
805 bool writes = true;
b5a280c0 806 VTDIOTLBEntry *iotlb_entry;
1da12ec4
LT		 807
808 /* Check if the request is in interrupt address range */
809 if (vtd_is_interrupt_addr(addr)) {
810 if (is_write) {
811 /* FIXME: since we don't know the length of the access here, we
812 * treat Non-DWORD length write requests without PASID as
813 * interrupt requests, too. Withoud interrupt remapping support,
814 * we just use 1:1 mapping.
815 */
816 VTD_DPRINTF(MMU, "write request to interrupt address "
817 "gpa 0x%"PRIx64, addr);
818 entry->iova = addr & VTD_PAGE_MASK_4K;
819 entry->translated_addr = addr & VTD_PAGE_MASK_4K;
820 entry->addr_mask = ~VTD_PAGE_MASK_4K;
821 entry->perm = IOMMU_WO;
822 return;
823 } else {
824 VTD_DPRINTF(GENERAL, "error: read request from interrupt address "
825 "gpa 0x%"PRIx64, addr);
826 vtd_report_dmar_fault(s, source_id, addr, VTD_FR_READ, is_write);
827 return;
828 }
829 }
b5a280c0
LT		 830 /* Try to fetch slpte form IOTLB */
831 iotlb_entry = vtd_lookup_iotlb(s, source_id, addr);
832 if (iotlb_entry) {
833 VTD_DPRINTF(CACHE, "hit iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64
834 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr,
835 iotlb_entry->slpte, iotlb_entry->domain_id);
836 slpte = iotlb_entry->slpte;
837 reads = iotlb_entry->read_flags;
838 writes = iotlb_entry->write_flags;
d66b969b 839 page_mask = iotlb_entry->mask;
b5a280c0
LT		 840 goto out;
841 }
d92fa2dc
LT		 842 /* Try to fetch context-entry from cache first */
843 if (cc_entry->context_cache_gen == s->context_cache_gen) {
844 VTD_DPRINTF(CACHE, "hit context-cache bus %d devfn %d "
845 "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 ")",
846 bus_num, devfn, cc_entry->context_entry.hi,
847 cc_entry->context_entry.lo, cc_entry->context_cache_gen);
848 ce = cc_entry->context_entry;
849 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
850 } else {
851 ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce);
852 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
853 if (ret_fr) {
854 ret_fr = -ret_fr;
855 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {
856 VTD_DPRINTF(FLOG, "fault processing is disabled for DMA "
857 "requests through this context-entry "
858 "(with FPD Set)");
859 } else {
860 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);
861 }
862 return;
1da12ec4 863 }
d92fa2dc
LT		 864 /* Update context-cache */
865 VTD_DPRINTF(CACHE, "update context-cache bus %d devfn %d "
866 "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 "->%"PRIu32 ")",
867 bus_num, devfn, ce.hi, ce.lo,
868 cc_entry->context_cache_gen, s->context_cache_gen);
869 cc_entry->context_entry = ce;
870 cc_entry->context_cache_gen = s->context_cache_gen;
1da12ec4
LT		 871 }
872
873 ret_fr = vtd_gpa_to_slpte(&ce, addr, is_write, &slpte, &level,
874 &reads, &writes);
875 if (ret_fr) {
876 ret_fr = -ret_fr;
877 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {
878 VTD_DPRINTF(FLOG, "fault processing is disabled for DMA requests "
879 "through this context-entry (with FPD Set)");
880 } else {
881 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);
882 }
883 return;
884 }
885
d66b969b 886 page_mask = vtd_slpt_level_page_mask(level);
b5a280c0 887 vtd_update_iotlb(s, source_id, VTD_CONTEXT_ENTRY_DID(ce.hi), addr, slpte,
d66b969b 888 reads, writes, level);
b5a280c0 889out:
d66b969b
JW		 890 entry->iova = addr & page_mask;
891 entry->translated_addr = vtd_get_slpte_addr(slpte) & page_mask;
892 entry->addr_mask = ~page_mask;
1da12ec4
LT		 893 entry->perm = (writes ? 2 : 0) + (reads ? 1 : 0);
894}
895
896static void vtd_root_table_setup(IntelIOMMUState *s)
897{
898 s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG);
899 s->root_extended = s->root & VTD_RTADDR_RTT;
900 s->root &= VTD_RTADDR_ADDR_MASK;
901
902 VTD_DPRINTF(CSR, "root_table addr 0x%"PRIx64 " %s", s->root,
903 (s->root_extended ? "(extended)" : ""));
904}
905
a5861439
PX		 906static void vtd_interrupt_remap_table_setup(IntelIOMMUState *s)
907{
908 uint64_t value = 0;
909 value = vtd_get_quad_raw(s, DMAR_IRTA_REG);
910 s->intr_size = 1UL << ((value & VTD_IRTA_SIZE_MASK) + 1);
911 s->intr_root = value & VTD_IRTA_ADDR_MASK;
912
913 /* TODO: invalidate interrupt entry cache */
914
915 VTD_DPRINTF(CSR, "int remap table addr 0x%"PRIx64 " size %"PRIu32,
916 s->intr_root, s->intr_size);
917}
918
d92fa2dc
LT		 919static void vtd_context_global_invalidate(IntelIOMMUState *s)
920{
921 s->context_cache_gen++;
922 if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) {
923 vtd_reset_context_cache(s);
924 }
925}
926
7df953bd
KO		 927
928/* Find the VTD address space currently associated with a given bus number,
929 */
930static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num)
931{
932 VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num];
933 if (!vtd_bus) {
934 /* Iterate over the registered buses to find the one
935 * which currently hold this bus number, and update the bus_num lookup table:
936 */
937 GHashTableIter iter;
938
939 g_hash_table_iter_init(&iter, s->vtd_as_by_busptr);
940 while (g_hash_table_iter_next (&iter, NULL, (void**)&vtd_bus)) {
941 if (pci_bus_num(vtd_bus->bus) == bus_num) {
942 s->vtd_as_by_bus_num[bus_num] = vtd_bus;
943 return vtd_bus;
944 }
945 }
946 }
947 return vtd_bus;
948}
949
d92fa2dc
LT		 950/* Do a context-cache device-selective invalidation.
951 * @func_mask: FM field after shifting
952 */
953static void vtd_context_device_invalidate(IntelIOMMUState *s,
954 uint16_t source_id,
955 uint16_t func_mask)
956{
957 uint16_t mask;
7df953bd 958 VTDBus *vtd_bus;
d92fa2dc
LT		 959 VTDAddressSpace *vtd_as;
960 uint16_t devfn;
961 uint16_t devfn_it;
962
963 switch (func_mask & 3) {
964 case 0:
965 mask = 0; /* No bits in the SID field masked */
966 break;
967 case 1:
968 mask = 4; /* Mask bit 2 in the SID field */
969 break;
970 case 2:
971 mask = 6; /* Mask bit 2:1 in the SID field */
972 break;
973 case 3:
974 mask = 7; /* Mask bit 2:0 in the SID field */
975 break;
976 }
977 VTD_DPRINTF(INV, "device-selective invalidation source 0x%"PRIx16
978 " mask %"PRIu16, source_id, mask);
7df953bd
KO		 979 vtd_bus = vtd_find_as_from_bus_num(s, VTD_SID_TO_BUS(source_id));
980 if (vtd_bus) {
d92fa2dc 981 devfn = VTD_SID_TO_DEVFN(source_id);
04af0e18 982 for (devfn_it = 0; devfn_it < X86_IOMMU_PCI_DEVFN_MAX; ++devfn_it) {
7df953bd 983 vtd_as = vtd_bus->dev_as[devfn_it];
d92fa2dc
LT		 984 if (vtd_as && ((devfn_it & mask) == (devfn & mask))) {
985 VTD_DPRINTF(INV, "invalidate context-cahce of devfn 0x%"PRIx16,
986 devfn_it);
987 vtd_as->context_cache_entry.context_cache_gen = 0;
988 }
989 }
990 }
991}
992
1da12ec4
LT		 993/* Context-cache invalidation
994 * Returns the Context Actual Invalidation Granularity.
995 * @val: the content of the CCMD_REG
996 */
997static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val)
998{
999 uint64_t caig;
1000 uint64_t type = val & VTD_CCMD_CIRG_MASK;
1001
1002 switch (type) {
d92fa2dc
LT		 1003 case VTD_CCMD_DOMAIN_INVL:
1004 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
1005 (uint16_t)VTD_CCMD_DID(val));
1006 /* Fall through */
1da12ec4 1007 case VTD_CCMD_GLOBAL_INVL:
d92fa2dc 1008 VTD_DPRINTF(INV, "global invalidation");
1da12ec4 1009 caig = VTD_CCMD_GLOBAL_INVL_A;
d92fa2dc 1010 vtd_context_global_invalidate(s);
1da12ec4
LT		 1011 break;
1012
1013 case VTD_CCMD_DEVICE_INVL:
1da12ec4 1014 caig = VTD_CCMD_DEVICE_INVL_A;
d92fa2dc 1015 vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val));
1da12ec4
LT		 1016 break;
1017
1018 default:
d92fa2dc 1019 VTD_DPRINTF(GENERAL, "error: invalid granularity");
1da12ec4
LT		 1020 caig = 0;
1021 }
1022 return caig;
1023}
1024
b5a280c0
LT		 1025static void vtd_iotlb_global_invalidate(IntelIOMMUState *s)
1026{
1027 vtd_reset_iotlb(s);
1028}
1029
1030static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id)
1031{
1032 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain,
1033 &domain_id);
1034}
1035
1036static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id,
1037 hwaddr addr, uint8_t am)
1038{
1039 VTDIOTLBPageInvInfo info;
1040
1041 assert(am <= VTD_MAMV);
1042 info.domain_id = domain_id;
d66b969b 1043 info.addr = addr;
b5a280c0
LT		 1044 info.mask = ~((1 << am) - 1);
1045 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info);
1046}
1047
1da12ec4
LT		 1048/* Flush IOTLB
1049 * Returns the IOTLB Actual Invalidation Granularity.
1050 * @val: the content of the IOTLB_REG
1051 */
1052static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val)
1053{
1054 uint64_t iaig;
1055 uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK;
b5a280c0
LT		 1056 uint16_t domain_id;
1057 hwaddr addr;
1058 uint8_t am;
1da12ec4
LT		 1059
1060 switch (type) {
1061 case VTD_TLB_GLOBAL_FLUSH:
b5a280c0 1062 VTD_DPRINTF(INV, "global invalidation");
1da12ec4 1063 iaig = VTD_TLB_GLOBAL_FLUSH_A;
b5a280c0 1064 vtd_iotlb_global_invalidate(s);
1da12ec4
LT		 1065 break;
1066
1067 case VTD_TLB_DSI_FLUSH:
b5a280c0
LT		 1068 domain_id = VTD_TLB_DID(val);
1069 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
1070 domain_id);
1da12ec4 1071 iaig = VTD_TLB_DSI_FLUSH_A;
b5a280c0 1072 vtd_iotlb_domain_invalidate(s, domain_id);
1da12ec4
LT		 1073 break;
1074
1075 case VTD_TLB_PSI_FLUSH:
b5a280c0
LT		 1076 domain_id = VTD_TLB_DID(val);
1077 addr = vtd_get_quad_raw(s, DMAR_IVA_REG);
1078 am = VTD_IVA_AM(addr);
1079 addr = VTD_IVA_ADDR(addr);
1080 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16
1081 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am);
1082 if (am > VTD_MAMV) {
1083 VTD_DPRINTF(GENERAL, "error: supported max address mask value is "
1084 "%"PRIu8, (uint8_t)VTD_MAMV);
1085 iaig = 0;
1086 break;
1087 }
1da12ec4 1088 iaig = VTD_TLB_PSI_FLUSH_A;
b5a280c0 1089 vtd_iotlb_page_invalidate(s, domain_id, addr, am);
1da12ec4
LT		 1090 break;
1091
1092 default:
b5a280c0 1093 VTD_DPRINTF(GENERAL, "error: invalid granularity");
1da12ec4
LT		 1094 iaig = 0;
1095 }
1096 return iaig;
1097}
1098
ed7b8fbc
LT		 1099static inline bool vtd_queued_inv_enable_check(IntelIOMMUState *s)
1100{
1101 return s->iq_tail == 0;
1102}
1103
1104static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s)
1105{
1106 return s->qi_enabled && (s->iq_tail == s->iq_head) &&
1107 (s->iq_last_desc_type == VTD_INV_DESC_WAIT);
1108}
1109
1110static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en)
1111{
1112 uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG);
1113
1114 VTD_DPRINTF(INV, "Queued Invalidation Enable %s", (en ? "on" : "off"));
1115 if (en) {
1116 if (vtd_queued_inv_enable_check(s)) {
1117 s->iq = iqa_val & VTD_IQA_IQA_MASK;
1118 /* 2^(x+8) entries */
1119 s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8);
1120 s->qi_enabled = true;
1121 VTD_DPRINTF(INV, "DMAR_IQA_REG 0x%"PRIx64, iqa_val);
1122 VTD_DPRINTF(INV, "Invalidation Queue addr 0x%"PRIx64 " size %d",
1123 s->iq, s->iq_size);
1124 /* Ok - report back to driver */
1125 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES);
1126 } else {
1127 VTD_DPRINTF(GENERAL, "error: can't enable Queued Invalidation: "
1128 "tail %"PRIu16, s->iq_tail);
1129 }
1130 } else {
1131 if (vtd_queued_inv_disable_check(s)) {
1132 /* disable Queued Invalidation */
1133 vtd_set_quad_raw(s, DMAR_IQH_REG, 0);
1134 s->iq_head = 0;
1135 s->qi_enabled = false;
1136 /* Ok - report back to driver */
1137 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0);
1138 } else {
1139 VTD_DPRINTF(GENERAL, "error: can't disable Queued Invalidation: "
1140 "head %"PRIu16 ", tail %"PRIu16
1141 ", last_descriptor %"PRIu8,
1142 s->iq_head, s->iq_tail, s->iq_last_desc_type);
1143 }
1144 }
1145}
1146
1da12ec4
LT		 1147/* Set Root Table Pointer */
1148static void vtd_handle_gcmd_srtp(IntelIOMMUState *s)
1149{
1150 VTD_DPRINTF(CSR, "set Root Table Pointer");
1151
1152 vtd_root_table_setup(s);
1153 /* Ok - report back to driver */
1154 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS);
1155}
1156
a5861439
PX		 1157/* Set Interrupt Remap Table Pointer */
1158static void vtd_handle_gcmd_sirtp(IntelIOMMUState *s)
1159{
1160 VTD_DPRINTF(CSR, "set Interrupt Remap Table Pointer");
1161
1162 vtd_interrupt_remap_table_setup(s);
1163 /* Ok - report back to driver */
1164 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRTPS);
1165}
1166
1da12ec4
LT		 1167/* Handle Translation Enable/Disable */
1168static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en)
1169{
1170 VTD_DPRINTF(CSR, "Translation Enable %s", (en ? "on" : "off"));
1171
1172 if (en) {
1173 s->dmar_enabled = true;
1174 /* Ok - report back to driver */
1175 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES);
1176 } else {
1177 s->dmar_enabled = false;
1178
1179 /* Clear the index of Fault Recording Register */
1180 s->next_frcd_reg = 0;
1181 /* Ok - report back to driver */
1182 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0);
1183 }
1184}
1185
80de52ba
PX		 1186/* Handle Interrupt Remap Enable/Disable */
1187static void vtd_handle_gcmd_ire(IntelIOMMUState *s, bool en)
1188{
1189 VTD_DPRINTF(CSR, "Interrupt Remap Enable %s", (en ? "on" : "off"));
1190
1191 if (en) {
1192 s->intr_enabled = true;
1193 /* Ok - report back to driver */
1194 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRES);
1195 } else {
1196 s->intr_enabled = false;
1197 /* Ok - report back to driver */
1198 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_IRES, 0);
1199 }
1200}
1201
1da12ec4
LT		 1202/* Handle write to Global Command Register */
1203static void vtd_handle_gcmd_write(IntelIOMMUState *s)
1204{
1205 uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG);
1206 uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG);
1207 uint32_t changed = status ^ val;
1208
1209 VTD_DPRINTF(CSR, "value 0x%"PRIx32 " status 0x%"PRIx32, val, status);
1210 if (changed & VTD_GCMD_TE) {
1211 /* Translation enable/disable */
1212 vtd_handle_gcmd_te(s, val & VTD_GCMD_TE);
1213 }
1214 if (val & VTD_GCMD_SRTP) {
1215 /* Set/update the root-table pointer */
1216 vtd_handle_gcmd_srtp(s);
1217 }
ed7b8fbc
LT		 1218 if (changed & VTD_GCMD_QIE) {
1219 /* Queued Invalidation Enable */
1220 vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE);
1221 }
a5861439
PX		 1222 if (val & VTD_GCMD_SIRTP) {
1223 /* Set/update the interrupt remapping root-table pointer */
1224 vtd_handle_gcmd_sirtp(s);
1225 }
80de52ba
PX		 1226 if (changed & VTD_GCMD_IRE) {
1227 /* Interrupt remap enable/disable */
1228 vtd_handle_gcmd_ire(s, val & VTD_GCMD_IRE);
1229 }
1da12ec4
LT		 1230}
1231
1232/* Handle write to Context Command Register */
1233static void vtd_handle_ccmd_write(IntelIOMMUState *s)
1234{
1235 uint64_t ret;
1236 uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG);
1237
1238 /* Context-cache invalidation request */
1239 if (val & VTD_CCMD_ICC) {
ed7b8fbc
LT		 1240 if (s->qi_enabled) {
1241 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, "
1242 "should not use register-based invalidation");
1243 return;
1244 }
1da12ec4
LT		 1245 ret = vtd_context_cache_invalidate(s, val);
1246 /* Invalidation completed. Change something to show */
1247 vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL);
1248 ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK,
1249 ret);
1250 VTD_DPRINTF(INV, "CCMD_REG write-back val: 0x%"PRIx64, ret);
1251 }
1252}
1253
1254/* Handle write to IOTLB Invalidation Register */
1255static void vtd_handle_iotlb_write(IntelIOMMUState *s)
1256{
1257 uint64_t ret;
1258 uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG);
1259
1260 /* IOTLB invalidation request */
1261 if (val & VTD_TLB_IVT) {
ed7b8fbc
LT		 1262 if (s->qi_enabled) {
1263 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, "
1264 "should not use register-based invalidation");
1265 return;
1266 }
1da12ec4
LT		 1267 ret = vtd_iotlb_flush(s, val);
1268 /* Invalidation completed. Change something to show */
1269 vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL);
1270 ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG,
1271 VTD_TLB_FLUSH_GRANU_MASK_A, ret);
1272 VTD_DPRINTF(INV, "IOTLB_REG write-back val: 0x%"PRIx64, ret);
1273 }
1274}
1275
ed7b8fbc
LT		 1276/* Fetch an Invalidation Descriptor from the Invalidation Queue */
1277static bool vtd_get_inv_desc(dma_addr_t base_addr, uint32_t offset,
1278 VTDInvDesc *inv_desc)
1279{
1280 dma_addr_t addr = base_addr + offset * sizeof(*inv_desc);
1281 if (dma_memory_read(&address_space_memory, addr, inv_desc,
1282 sizeof(*inv_desc))) {
1283 VTD_DPRINTF(GENERAL, "error: fail to fetch Invalidation Descriptor "
1284 "base_addr 0x%"PRIx64 " offset %"PRIu32, base_addr, offset);
1285 inv_desc->lo = 0;
1286 inv_desc->hi = 0;
1287
1288 return false;
1289 }
1290 inv_desc->lo = le64_to_cpu(inv_desc->lo);
1291 inv_desc->hi = le64_to_cpu(inv_desc->hi);
1292 return true;
1293}
1294
1295static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
1296{
1297 if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) ||
1298 (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) {
1299 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Invalidation "
1300 "Wait Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1301 inv_desc->hi, inv_desc->lo);
1302 return false;
1303 }
1304 if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) {
1305 /* Status Write */
1306 uint32_t status_data = (uint32_t)(inv_desc->lo >>
1307 VTD_INV_DESC_WAIT_DATA_SHIFT);
1308
1309 assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF));
1310
1311 /* FIXME: need to be masked with HAW? */
1312 dma_addr_t status_addr = inv_desc->hi;
1313 VTD_DPRINTF(INV, "status data 0x%x, status addr 0x%"PRIx64,
1314 status_data, status_addr);
1315 status_data = cpu_to_le32(status_data);
1316 if (dma_memory_write(&address_space_memory, status_addr, &status_data,
1317 sizeof(status_data))) {
1318 VTD_DPRINTF(GENERAL, "error: fail to perform a coherent write");
1319 return false;
1320 }
1321 } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) {
1322 /* Interrupt flag */
1323 VTD_DPRINTF(INV, "Invalidation Wait Descriptor interrupt completion");
1324 vtd_generate_completion_event(s);
1325 } else {
1326 VTD_DPRINTF(GENERAL, "error: invalid Invalidation Wait Descriptor: "
1327 "hi 0x%"PRIx64 " lo 0x%"PRIx64, inv_desc->hi, inv_desc->lo);
1328 return false;
1329 }
1330 return true;
1331}
1332
d92fa2dc
LT		 1333static bool vtd_process_context_cache_desc(IntelIOMMUState *s,
1334 VTDInvDesc *inv_desc)
1335{
1336 if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) {
1337 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Context-cache "
1338 "Invalidate Descriptor");
1339 return false;
1340 }
1341 switch (inv_desc->lo & VTD_INV_DESC_CC_G) {
1342 case VTD_INV_DESC_CC_DOMAIN:
1343 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
1344 (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo));
1345 /* Fall through */
1346 case VTD_INV_DESC_CC_GLOBAL:
1347 VTD_DPRINTF(INV, "global invalidation");
1348 vtd_context_global_invalidate(s);
1349 break;
1350
1351 case VTD_INV_DESC_CC_DEVICE:
1352 vtd_context_device_invalidate(s, VTD_INV_DESC_CC_SID(inv_desc->lo),
1353 VTD_INV_DESC_CC_FM(inv_desc->lo));
1354 break;
1355
1356 default:
1357 VTD_DPRINTF(GENERAL, "error: invalid granularity in Context-cache "
1358 "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1359 inv_desc->hi, inv_desc->lo);
1360 return false;
1361 }
1362 return true;
1363}
1364
b5a280c0
LT		 1365static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
1366{
1367 uint16_t domain_id;
1368 uint8_t am;
1369 hwaddr addr;
1370
1371 if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) ||
1372 (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) {
1373 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in IOTLB "
1374 "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1375 inv_desc->hi, inv_desc->lo);
1376 return false;
1377 }
1378
1379 switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) {
1380 case VTD_INV_DESC_IOTLB_GLOBAL:
1381 VTD_DPRINTF(INV, "global invalidation");
1382 vtd_iotlb_global_invalidate(s);
1383 break;
1384
1385 case VTD_INV_DESC_IOTLB_DOMAIN:
1386 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
1387 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
1388 domain_id);
1389 vtd_iotlb_domain_invalidate(s, domain_id);
1390 break;
1391
1392 case VTD_INV_DESC_IOTLB_PAGE:
1393 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
1394 addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi);
1395 am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi);
1396 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16
1397 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am);
1398 if (am > VTD_MAMV) {
1399 VTD_DPRINTF(GENERAL, "error: supported max address mask value is "
1400 "%"PRIu8, (uint8_t)VTD_MAMV);
1401 return false;
1402 }
1403 vtd_iotlb_page_invalidate(s, domain_id, addr, am);
1404 break;
1405
1406 default:
1407 VTD_DPRINTF(GENERAL, "error: invalid granularity in IOTLB Invalidate "
1408 "Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1409 inv_desc->hi, inv_desc->lo);
1410 return false;
1411 }
1412 return true;
1413}
1414
ed7b8fbc
LT		 1415static bool vtd_process_inv_desc(IntelIOMMUState *s)
1416{
1417 VTDInvDesc inv_desc;
1418 uint8_t desc_type;
1419
1420 VTD_DPRINTF(INV, "iq head %"PRIu16, s->iq_head);
1421 if (!vtd_get_inv_desc(s->iq, s->iq_head, &inv_desc)) {
1422 s->iq_last_desc_type = VTD_INV_DESC_NONE;
1423 return false;
1424 }
1425 desc_type = inv_desc.lo & VTD_INV_DESC_TYPE;
1426 /* FIXME: should update at first or at last? */
1427 s->iq_last_desc_type = desc_type;
1428
1429 switch (desc_type) {
1430 case VTD_INV_DESC_CC:
1431 VTD_DPRINTF(INV, "Context-cache Invalidate Descriptor hi 0x%"PRIx64
1432 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
d92fa2dc
LT		 1433 if (!vtd_process_context_cache_desc(s, &inv_desc)) {
1434 return false;
1435 }
ed7b8fbc
LT		 1436 break;
1437
1438 case VTD_INV_DESC_IOTLB:
1439 VTD_DPRINTF(INV, "IOTLB Invalidate Descriptor hi 0x%"PRIx64
1440 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
b5a280c0
LT		 1441 if (!vtd_process_iotlb_desc(s, &inv_desc)) {
1442 return false;
1443 }
ed7b8fbc
LT		 1444 break;
1445
1446 case VTD_INV_DESC_WAIT:
1447 VTD_DPRINTF(INV, "Invalidation Wait Descriptor hi 0x%"PRIx64
1448 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
1449 if (!vtd_process_wait_desc(s, &inv_desc)) {
1450 return false;
1451 }
1452 break;
1453
b7910472
PX		 1454 case VTD_INV_DESC_IEC:
1455 VTD_DPRINTF(INV, "Interrupt Entry Cache Invalidation "
1456 "not implemented yet");
1457 /*
1458 * Since currently we do not cache interrupt entries, we can
1459 * just mark this descriptor as "good" and move on.
1460 */
1461 break;
1462
ed7b8fbc
LT		 1463 default:
1464 VTD_DPRINTF(GENERAL, "error: unkonw Invalidation Descriptor type "
1465 "hi 0x%"PRIx64 " lo 0x%"PRIx64 " type %"PRIu8,
1466 inv_desc.hi, inv_desc.lo, desc_type);
1467 return false;
1468 }
1469 s->iq_head++;
1470 if (s->iq_head == s->iq_size) {
1471 s->iq_head = 0;
1472 }
1473 return true;
1474}
1475
1476/* Try to fetch and process more Invalidation Descriptors */
1477static void vtd_fetch_inv_desc(IntelIOMMUState *s)
1478{
1479 VTD_DPRINTF(INV, "fetch Invalidation Descriptors");
1480 if (s->iq_tail >= s->iq_size) {
1481 /* Detects an invalid Tail pointer */
1482 VTD_DPRINTF(GENERAL, "error: iq_tail is %"PRIu16
1483 " while iq_size is %"PRIu16, s->iq_tail, s->iq_size);
1484 vtd_handle_inv_queue_error(s);
1485 return;
1486 }
1487 while (s->iq_head != s->iq_tail) {
1488 if (!vtd_process_inv_desc(s)) {
1489 /* Invalidation Queue Errors */
1490 vtd_handle_inv_queue_error(s);
1491 break;
1492 }
1493 /* Must update the IQH_REG in time */
1494 vtd_set_quad_raw(s, DMAR_IQH_REG,
1495 (((uint64_t)(s->iq_head)) << VTD_IQH_QH_SHIFT) &
1496 VTD_IQH_QH_MASK);
1497 }
1498}
1499
1500/* Handle write to Invalidation Queue Tail Register */
1501static void vtd_handle_iqt_write(IntelIOMMUState *s)
1502{
1503 uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG);
1504
1505 s->iq_tail = VTD_IQT_QT(val);
1506 VTD_DPRINTF(INV, "set iq tail %"PRIu16, s->iq_tail);
1507 if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) {
1508 /* Process Invalidation Queue here */
1509 vtd_fetch_inv_desc(s);
1510 }
1511}
1512
1da12ec4
LT		 1513static void vtd_handle_fsts_write(IntelIOMMUState *s)
1514{
1515 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
1516 uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
1517 uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE;
1518
1519 if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) {
1520 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
1521 VTD_DPRINTF(FLOG, "all pending interrupt conditions serviced, clear "
1522 "IP field of FECTL_REG");
1523 }
ed7b8fbc
LT		 1524 /* FIXME: when IQE is Clear, should we try to fetch some Invalidation
1525 * Descriptors if there are any when Queued Invalidation is enabled?
1526 */
1da12ec4
LT		 1527}
1528
1529static void vtd_handle_fectl_write(IntelIOMMUState *s)
1530{
1531 uint32_t fectl_reg;
1532 /* FIXME: when software clears the IM field, check the IP field. But do we
1533 * need to compare the old value and the new value to conclude that
1534 * software clears the IM field? Or just check if the IM field is zero?
1535 */
1536 fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
1537 if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) {
1538 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
1539 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
1540 VTD_DPRINTF(FLOG, "IM field is cleared, generate "
1541 "fault event interrupt");
1542 }
1543}
1544
ed7b8fbc
LT		 1545static void vtd_handle_ics_write(IntelIOMMUState *s)
1546{
1547 uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG);
1548 uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
1549
1550 if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) {
1551 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
1552 VTD_DPRINTF(INV, "pending completion interrupt condition serviced, "
1553 "clear IP field of IECTL_REG");
1554 }
1555}
1556
1557static void vtd_handle_iectl_write(IntelIOMMUState *s)
1558{
1559 uint32_t iectl_reg;
1560 /* FIXME: when software clears the IM field, check the IP field. But do we
1561 * need to compare the old value and the new value to conclude that
1562 * software clears the IM field? Or just check if the IM field is zero?
1563 */
1564 iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
1565 if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) {
1566 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
1567 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
1568 VTD_DPRINTF(INV, "IM field is cleared, generate "
1569 "invalidation event interrupt");
1570 }
1571}
1572
1da12ec4
LT		 1573static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size)
1574{
1575 IntelIOMMUState *s = opaque;
1576 uint64_t val;
1577
1578 if (addr + size > DMAR_REG_SIZE) {
1579 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64
1580 ", got 0x%"PRIx64 " %d",
1581 (uint64_t)DMAR_REG_SIZE, addr, size);
1582 return (uint64_t)-1;
1583 }
1584
1585 switch (addr) {
1586 /* Root Table Address Register, 64-bit */
1587 case DMAR_RTADDR_REG:
1588 if (size == 4) {
1589 val = s->root & ((1ULL << 32) - 1);
1590 } else {
1591 val = s->root;
1592 }
1593 break;
1594
1595 case DMAR_RTADDR_REG_HI:
1596 assert(size == 4);
1597 val = s->root >> 32;
1598 break;
1599
ed7b8fbc
LT		 1600 /* Invalidation Queue Address Register, 64-bit */
1601 case DMAR_IQA_REG:
1602 val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS);
1603 if (size == 4) {
1604 val = val & ((1ULL << 32) - 1);
1605 }
1606 break;
1607
1608 case DMAR_IQA_REG_HI:
1609 assert(size == 4);
1610 val = s->iq >> 32;
1611 break;
1612
1da12ec4
LT		 1613 default:
1614 if (size == 4) {
1615 val = vtd_get_long(s, addr);
1616 } else {
1617 val = vtd_get_quad(s, addr);
1618 }
1619 }
1620 VTD_DPRINTF(CSR, "addr 0x%"PRIx64 " size %d val 0x%"PRIx64,
1621 addr, size, val);
1622 return val;
1623}
1624
1625static void vtd_mem_write(void *opaque, hwaddr addr,
1626 uint64_t val, unsigned size)
1627{
1628 IntelIOMMUState *s = opaque;
1629
1630 if (addr + size > DMAR_REG_SIZE) {
1631 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64
1632 ", got 0x%"PRIx64 " %d",
1633 (uint64_t)DMAR_REG_SIZE, addr, size);
1634 return;
1635 }
1636
1637 switch (addr) {
1638 /* Global Command Register, 32-bit */
1639 case DMAR_GCMD_REG:
1640 VTD_DPRINTF(CSR, "DMAR_GCMD_REG write addr 0x%"PRIx64
1641 ", size %d, val 0x%"PRIx64, addr, size, val);
1642 vtd_set_long(s, addr, val);
1643 vtd_handle_gcmd_write(s);
1644 break;
1645
1646 /* Context Command Register, 64-bit */
1647 case DMAR_CCMD_REG:
1648 VTD_DPRINTF(CSR, "DMAR_CCMD_REG write addr 0x%"PRIx64
1649 ", size %d, val 0x%"PRIx64, addr, size, val);
1650 if (size == 4) {
1651 vtd_set_long(s, addr, val);
1652 } else {
1653 vtd_set_quad(s, addr, val);
1654 vtd_handle_ccmd_write(s);
1655 }
1656 break;
1657
1658 case DMAR_CCMD_REG_HI:
1659 VTD_DPRINTF(CSR, "DMAR_CCMD_REG_HI write addr 0x%"PRIx64
1660 ", size %d, val 0x%"PRIx64, addr, size, val);
1661 assert(size == 4);
1662 vtd_set_long(s, addr, val);
1663 vtd_handle_ccmd_write(s);
1664 break;
1665
1666 /* IOTLB Invalidation Register, 64-bit */
1667 case DMAR_IOTLB_REG:
1668 VTD_DPRINTF(INV, "DMAR_IOTLB_REG write addr 0x%"PRIx64
1669 ", size %d, val 0x%"PRIx64, addr, size, val);
1670 if (size == 4) {
1671 vtd_set_long(s, addr, val);
1672 } else {
1673 vtd_set_quad(s, addr, val);
1674 vtd_handle_iotlb_write(s);
1675 }
1676 break;
1677
1678 case DMAR_IOTLB_REG_HI:
1679 VTD_DPRINTF(INV, "DMAR_IOTLB_REG_HI write addr 0x%"PRIx64
1680 ", size %d, val 0x%"PRIx64, addr, size, val);
1681 assert(size == 4);
1682 vtd_set_long(s, addr, val);
1683 vtd_handle_iotlb_write(s);
1684 break;
1685
b5a280c0
LT		 1686 /* Invalidate Address Register, 64-bit */
1687 case DMAR_IVA_REG:
1688 VTD_DPRINTF(INV, "DMAR_IVA_REG write addr 0x%"PRIx64
1689 ", size %d, val 0x%"PRIx64, addr, size, val);
1690 if (size == 4) {
1691 vtd_set_long(s, addr, val);
1692 } else {
1693 vtd_set_quad(s, addr, val);
1694 }
1695 break;
1696
1697 case DMAR_IVA_REG_HI:
1698 VTD_DPRINTF(INV, "DMAR_IVA_REG_HI write addr 0x%"PRIx64
1699 ", size %d, val 0x%"PRIx64, addr, size, val);
1700 assert(size == 4);
1701 vtd_set_long(s, addr, val);
1702 break;
1703
1da12ec4
LT		 1704 /* Fault Status Register, 32-bit */
1705 case DMAR_FSTS_REG:
1706 VTD_DPRINTF(FLOG, "DMAR_FSTS_REG write addr 0x%"PRIx64
1707 ", size %d, val 0x%"PRIx64, addr, size, val);
1708 assert(size == 4);
1709 vtd_set_long(s, addr, val);
1710 vtd_handle_fsts_write(s);
1711 break;
1712
1713 /* Fault Event Control Register, 32-bit */
1714 case DMAR_FECTL_REG:
1715 VTD_DPRINTF(FLOG, "DMAR_FECTL_REG write addr 0x%"PRIx64
1716 ", size %d, val 0x%"PRIx64, addr, size, val);
1717 assert(size == 4);
1718 vtd_set_long(s, addr, val);
1719 vtd_handle_fectl_write(s);
1720 break;
1721
1722 /* Fault Event Data Register, 32-bit */
1723 case DMAR_FEDATA_REG:
1724 VTD_DPRINTF(FLOG, "DMAR_FEDATA_REG write addr 0x%"PRIx64
1725 ", size %d, val 0x%"PRIx64, addr, size, val);
1726 assert(size == 4);
1727 vtd_set_long(s, addr, val);
1728 break;
1729
1730 /* Fault Event Address Register, 32-bit */
1731 case DMAR_FEADDR_REG:
1732 VTD_DPRINTF(FLOG, "DMAR_FEADDR_REG write addr 0x%"PRIx64
1733 ", size %d, val 0x%"PRIx64, addr, size, val);
1734 assert(size == 4);
1735 vtd_set_long(s, addr, val);
1736 break;
1737
1738 /* Fault Event Upper Address Register, 32-bit */
1739 case DMAR_FEUADDR_REG:
1740 VTD_DPRINTF(FLOG, "DMAR_FEUADDR_REG write addr 0x%"PRIx64
1741 ", size %d, val 0x%"PRIx64, addr, size, val);
1742 assert(size == 4);
1743 vtd_set_long(s, addr, val);
1744 break;
1745
1746 /* Protected Memory Enable Register, 32-bit */
1747 case DMAR_PMEN_REG:
1748 VTD_DPRINTF(CSR, "DMAR_PMEN_REG write addr 0x%"PRIx64
1749 ", size %d, val 0x%"PRIx64, addr, size, val);
1750 assert(size == 4);
1751 vtd_set_long(s, addr, val);
1752 break;
1753
1754 /* Root Table Address Register, 64-bit */
1755 case DMAR_RTADDR_REG:
1756 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG write addr 0x%"PRIx64
1757 ", size %d, val 0x%"PRIx64, addr, size, val);
1758 if (size == 4) {
1759 vtd_set_long(s, addr, val);
1760 } else {
1761 vtd_set_quad(s, addr, val);
1762 }
1763 break;
1764
1765 case DMAR_RTADDR_REG_HI:
1766 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG_HI write addr 0x%"PRIx64
1767 ", size %d, val 0x%"PRIx64, addr, size, val);
1768 assert(size == 4);
1769 vtd_set_long(s, addr, val);
1770 break;
1771
ed7b8fbc
LT		 1772 /* Invalidation Queue Tail Register, 64-bit */
1773 case DMAR_IQT_REG:
1774 VTD_DPRINTF(INV, "DMAR_IQT_REG write addr 0x%"PRIx64
1775 ", size %d, val 0x%"PRIx64, addr, size, val);
1776 if (size == 4) {
1777 vtd_set_long(s, addr, val);
1778 } else {
1779 vtd_set_quad(s, addr, val);
1780 }
1781 vtd_handle_iqt_write(s);
1782 break;
1783
1784 case DMAR_IQT_REG_HI:
1785 VTD_DPRINTF(INV, "DMAR_IQT_REG_HI write addr 0x%"PRIx64
1786 ", size %d, val 0x%"PRIx64, addr, size, val);
1787 assert(size == 4);
1788 vtd_set_long(s, addr, val);
1789 /* 19:63 of IQT_REG is RsvdZ, do nothing here */
1790 break;
1791
1792 /* Invalidation Queue Address Register, 64-bit */
1793 case DMAR_IQA_REG:
1794 VTD_DPRINTF(INV, "DMAR_IQA_REG write addr 0x%"PRIx64
1795 ", size %d, val 0x%"PRIx64, addr, size, val);
1796 if (size == 4) {
1797 vtd_set_long(s, addr, val);
1798 } else {
1799 vtd_set_quad(s, addr, val);
1800 }
1801 break;
1802
1803 case DMAR_IQA_REG_HI:
1804 VTD_DPRINTF(INV, "DMAR_IQA_REG_HI write addr 0x%"PRIx64
1805 ", size %d, val 0x%"PRIx64, addr, size, val);
1806 assert(size == 4);
1807 vtd_set_long(s, addr, val);
1808 break;
1809
1810 /* Invalidation Completion Status Register, 32-bit */
1811 case DMAR_ICS_REG:
1812 VTD_DPRINTF(INV, "DMAR_ICS_REG write addr 0x%"PRIx64
1813 ", size %d, val 0x%"PRIx64, addr, size, val);
1814 assert(size == 4);
1815 vtd_set_long(s, addr, val);
1816 vtd_handle_ics_write(s);
1817 break;
1818
1819 /* Invalidation Event Control Register, 32-bit */
1820 case DMAR_IECTL_REG:
1821 VTD_DPRINTF(INV, "DMAR_IECTL_REG write addr 0x%"PRIx64
1822 ", size %d, val 0x%"PRIx64, addr, size, val);
1823 assert(size == 4);
1824 vtd_set_long(s, addr, val);
1825 vtd_handle_iectl_write(s);
1826 break;
1827
1828 /* Invalidation Event Data Register, 32-bit */
1829 case DMAR_IEDATA_REG:
1830 VTD_DPRINTF(INV, "DMAR_IEDATA_REG write addr 0x%"PRIx64
1831 ", size %d, val 0x%"PRIx64, addr, size, val);
1832 assert(size == 4);
1833 vtd_set_long(s, addr, val);
1834 break;
1835
1836 /* Invalidation Event Address Register, 32-bit */
1837 case DMAR_IEADDR_REG:
1838 VTD_DPRINTF(INV, "DMAR_IEADDR_REG write addr 0x%"PRIx64
1839 ", size %d, val 0x%"PRIx64, addr, size, val);
1840 assert(size == 4);
1841 vtd_set_long(s, addr, val);
1842 break;
1843
1844 /* Invalidation Event Upper Address Register, 32-bit */
1845 case DMAR_IEUADDR_REG:
1846 VTD_DPRINTF(INV, "DMAR_IEUADDR_REG write addr 0x%"PRIx64
1847 ", size %d, val 0x%"PRIx64, addr, size, val);
1848 assert(size == 4);
1849 vtd_set_long(s, addr, val);
1850 break;
1851
1da12ec4
LT		 1852 /* Fault Recording Registers, 128-bit */
1853 case DMAR_FRCD_REG_0_0:
1854 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_0 write addr 0x%"PRIx64
1855 ", size %d, val 0x%"PRIx64, addr, size, val);
1856 if (size == 4) {
1857 vtd_set_long(s, addr, val);
1858 } else {
1859 vtd_set_quad(s, addr, val);
1860 }
1861 break;
1862
1863 case DMAR_FRCD_REG_0_1:
1864 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_1 write addr 0x%"PRIx64
1865 ", size %d, val 0x%"PRIx64, addr, size, val);
1866 assert(size == 4);
1867 vtd_set_long(s, addr, val);
1868 break;
1869
1870 case DMAR_FRCD_REG_0_2:
1871 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_2 write addr 0x%"PRIx64
1872 ", size %d, val 0x%"PRIx64, addr, size, val);
1873 if (size == 4) {
1874 vtd_set_long(s, addr, val);
1875 } else {
1876 vtd_set_quad(s, addr, val);
1877 /* May clear bit 127 (Fault), update PPF */
1878 vtd_update_fsts_ppf(s);
1879 }
1880 break;
1881
1882 case DMAR_FRCD_REG_0_3:
1883 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_3 write addr 0x%"PRIx64
1884 ", size %d, val 0x%"PRIx64, addr, size, val);
1885 assert(size == 4);
1886 vtd_set_long(s, addr, val);
1887 /* May clear bit 127 (Fault), update PPF */
1888 vtd_update_fsts_ppf(s);
1889 break;
1890
a5861439
PX		 1891 case DMAR_IRTA_REG:
1892 VTD_DPRINTF(IR, "DMAR_IRTA_REG write addr 0x%"PRIx64
1893 ", size %d, val 0x%"PRIx64, addr, size, val);
1894 if (size == 4) {
1895 vtd_set_long(s, addr, val);
1896 } else {
1897 vtd_set_quad(s, addr, val);
1898 }
1899 break;
1900
1901 case DMAR_IRTA_REG_HI:
1902 VTD_DPRINTF(IR, "DMAR_IRTA_REG_HI write addr 0x%"PRIx64
1903 ", size %d, val 0x%"PRIx64, addr, size, val);
1904 assert(size == 4);
1905 vtd_set_long(s, addr, val);
1906 break;
1907
1da12ec4
LT		 1908 default:
1909 VTD_DPRINTF(GENERAL, "error: unhandled reg write addr 0x%"PRIx64
1910 ", size %d, val 0x%"PRIx64, addr, size, val);
1911 if (size == 4) {
1912 vtd_set_long(s, addr, val);
1913 } else {
1914 vtd_set_quad(s, addr, val);
1915 }
1916 }
1917}
1918
1919static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr,
1920 bool is_write)
1921{
1922 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
1923 IntelIOMMUState *s = vtd_as->iommu_state;
1da12ec4
LT		 1924 IOMMUTLBEntry ret = {
1925 .target_as = &address_space_memory,
1926 .iova = addr,
1927 .translated_addr = 0,
1928 .addr_mask = ~(hwaddr)0,
1929 .perm = IOMMU_NONE,
1930 };
1931
1932 if (!s->dmar_enabled) {
1933 /* DMAR disabled, passthrough, use 4k-page*/
1934 ret.iova = addr & VTD_PAGE_MASK_4K;
1935 ret.translated_addr = addr & VTD_PAGE_MASK_4K;
1936 ret.addr_mask = ~VTD_PAGE_MASK_4K;
1937 ret.perm = IOMMU_RW;
1938 return ret;
1939 }
1940
7df953bd 1941 vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, addr,
d92fa2dc 1942 is_write, &ret);
1da12ec4
LT		 1943 VTD_DPRINTF(MMU,
1944 "bus %"PRIu8 " slot %"PRIu8 " func %"PRIu8 " devfn %"PRIu8
7df953bd 1945 " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, pci_bus_num(vtd_as->bus),
d92fa2dc
LT		 1946 VTD_PCI_SLOT(vtd_as->devfn), VTD_PCI_FUNC(vtd_as->devfn),
1947 vtd_as->devfn, addr, ret.translated_addr);
1da12ec4
LT		 1948 return ret;
1949}
1950
3cb3b154
AW		 1951static void vtd_iommu_notify_started(MemoryRegion *iommu)
1952{
1953 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
1954
1955 hw_error("Device at bus %s addr %02x.%d requires iommu notifier which "
1956 "is currently not supported by intel-iommu emulation",
1957 vtd_as->bus->qbus.name, PCI_SLOT(vtd_as->devfn),
1958 PCI_FUNC(vtd_as->devfn));
1959}
1960
1da12ec4
LT		 1961static const VMStateDescription vtd_vmstate = {
1962 .name = "iommu-intel",
1963 .unmigratable = 1,
1964};
1965
1966static const MemoryRegionOps vtd_mem_ops = {
1967 .read = vtd_mem_read,
1968 .write = vtd_mem_write,
1969 .endianness = DEVICE_LITTLE_ENDIAN,
1970 .impl = {
1971 .min_access_size = 4,
1972 .max_access_size = 8,
1973 },
1974 .valid = {
1975 .min_access_size = 4,
1976 .max_access_size = 8,
1977 },
1978};
1979
1980static Property vtd_properties[] = {
1981 DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0),
1982 DEFINE_PROP_END_OF_LIST(),
1983};
1984
651e4cef
PX		 1985/* Read IRTE entry with specific index */
1986static int vtd_irte_get(IntelIOMMUState *iommu, uint16_t index,
1987 VTD_IRTE *entry)
1988{
1989 dma_addr_t addr = 0x00;
1990
1991 addr = iommu->intr_root + index * sizeof(*entry);
1992 if (dma_memory_read(&address_space_memory, addr, entry,
1993 sizeof(*entry))) {
1994 VTD_DPRINTF(GENERAL, "error: fail to access IR root at 0x%"PRIx64
1995 " + %"PRIu16, iommu->intr_root, index);
1996 return -VTD_FR_IR_ROOT_INVAL;
1997 }
1998
1999 if (!entry->present) {
2000 VTD_DPRINTF(GENERAL, "error: present flag not set in IRTE"
2001 " entry index %u value 0x%"PRIx64 " 0x%"PRIx64,
2002 index, le64_to_cpu(entry->data[1]),
2003 le64_to_cpu(entry->data[0]));
2004 return -VTD_FR_IR_ENTRY_P;
2005 }
2006
2007 if (entry->__reserved_0 || entry->__reserved_1 || \
2008 entry->__reserved_2) {
2009 VTD_DPRINTF(GENERAL, "error: IRTE entry index %"PRIu16
2010 " reserved fields non-zero: 0x%"PRIx64 " 0x%"PRIx64,
2011 index, le64_to_cpu(entry->data[1]),
2012 le64_to_cpu(entry->data[0]));
2013 return -VTD_FR_IR_IRTE_RSVD;
2014 }
2015
2016 /*
2017 * TODO: Check Source-ID corresponds to SVT (Source Validation
2018 * Type) bits
2019 */
2020
2021 return 0;
2022}
2023
2024/* Fetch IRQ information of specific IR index */
2025static int vtd_remap_irq_get(IntelIOMMUState *iommu, uint16_t index, VTDIrq *irq)
2026{
09cd058a 2027 VTD_IRTE irte = {};
651e4cef
PX		 2028 int ret = 0;
2029
2030 ret = vtd_irte_get(iommu, index, &irte);
2031 if (ret) {
2032 return ret;
2033 }
2034
2035 irq->trigger_mode = irte.trigger_mode;
2036 irq->vector = irte.vector;
2037 irq->delivery_mode = irte.delivery_mode;
2038 /* Not support EIM yet: please refer to vt-d 9.10 DST bits */
2039#define VTD_IR_APIC_DEST_MASK (0xff00ULL)
2040#define VTD_IR_APIC_DEST_SHIFT (8)
2041 irq->dest = (le32_to_cpu(irte.dest_id) & VTD_IR_APIC_DEST_MASK) >> \
2042 VTD_IR_APIC_DEST_SHIFT;
2043 irq->dest_mode = irte.dest_mode;
2044 irq->redir_hint = irte.redir_hint;
2045
2046 VTD_DPRINTF(IR, "remapping interrupt index %d: trig:%u,vec:%u,"
2047 "deliver:%u,dest:%u,dest_mode:%u", index,
2048 irq->trigger_mode, irq->vector, irq->delivery_mode,
2049 irq->dest, irq->dest_mode);
2050
2051 return 0;
2052}
2053
2054/* Generate one MSI message from VTDIrq info */
2055static void vtd_generate_msi_message(VTDIrq *irq, MSIMessage *msg_out)
2056{
2057 VTD_MSIMessage msg = {};
2058
2059 /* Generate address bits */
2060 msg.dest_mode = irq->dest_mode;
2061 msg.redir_hint = irq->redir_hint;
2062 msg.dest = irq->dest;
2063 msg.__addr_head = cpu_to_le32(0xfee);
2064 /* Keep this from original MSI address bits */
2065 msg.__not_used = irq->msi_addr_last_bits;
2066
2067 /* Generate data bits */
2068 msg.vector = irq->vector;
2069 msg.delivery_mode = irq->delivery_mode;
2070 msg.level = 1;
2071 msg.trigger_mode = irq->trigger_mode;
2072
2073 msg_out->address = msg.msi_addr;
2074 msg_out->data = msg.msi_data;
2075}
2076
2077/* Interrupt remapping for MSI/MSI-X entry */
2078static int vtd_interrupt_remap_msi(IntelIOMMUState *iommu,
2079 MSIMessage *origin,
2080 MSIMessage *translated)
2081{
2082 int ret = 0;
2083 VTD_IR_MSIAddress addr;
2084 uint16_t index;
09cd058a 2085 VTDIrq irq = {};
651e4cef
PX		 2086
2087 assert(origin && translated);
2088
2089 if (!iommu || !iommu->intr_enabled) {
2090 goto do_not_translate;
2091 }
2092
2093 if (origin->address & VTD_MSI_ADDR_HI_MASK) {
2094 VTD_DPRINTF(GENERAL, "error: MSI addr high 32 bits nonzero"
2095 " during interrupt remapping: 0x%"PRIx32,
2096 (uint32_t)((origin->address & VTD_MSI_ADDR_HI_MASK) >> \
2097 VTD_MSI_ADDR_HI_SHIFT));
2098 return -VTD_FR_IR_REQ_RSVD;
2099 }
2100
2101 addr.data = origin->address & VTD_MSI_ADDR_LO_MASK;
2102 if (le16_to_cpu(addr.__head) != 0xfee) {
2103 VTD_DPRINTF(GENERAL, "error: MSI addr low 32 bits invalid: "
2104 "0x%"PRIx32, addr.data);
2105 return -VTD_FR_IR_REQ_RSVD;
2106 }
2107
2108 /* This is compatible mode. */
2109 if (addr.int_mode != VTD_IR_INT_FORMAT_REMAP) {
2110 goto do_not_translate;
2111 }
2112
2113 index = addr.index_h << 15 | le16_to_cpu(addr.index_l);
2114
2115#define VTD_IR_MSI_DATA_SUBHANDLE (0x0000ffff)
2116#define VTD_IR_MSI_DATA_RESERVED (0xffff0000)
2117
2118 if (addr.sub_valid) {
2119 /* See VT-d spec 5.1.2.2 and 5.1.3 on subhandle */
2120 index += origin->data & VTD_IR_MSI_DATA_SUBHANDLE;
2121 }
2122
2123 ret = vtd_remap_irq_get(iommu, index, &irq);
2124 if (ret) {
2125 return ret;
2126 }
2127
2128 if (addr.sub_valid) {
2129 VTD_DPRINTF(IR, "received MSI interrupt");
2130 if (origin->data & VTD_IR_MSI_DATA_RESERVED) {
2131 VTD_DPRINTF(GENERAL, "error: MSI data bits non-zero for "
2132 "interrupt remappable entry: 0x%"PRIx32,
2133 origin->data);
2134 return -VTD_FR_IR_REQ_RSVD;
2135 }
2136 } else {
2137 uint8_t vector = origin->data & 0xff;
2138 VTD_DPRINTF(IR, "received IOAPIC interrupt");
2139 /* IOAPIC entry vector should be aligned with IRTE vector
2140 * (see vt-d spec 5.1.5.1). */
2141 if (vector != irq.vector) {
2142 VTD_DPRINTF(GENERAL, "IOAPIC vector inconsistent: "
2143 "entry: %d, IRTE: %d, index: %d",
2144 vector, irq.vector, index);
2145 }
2146 }
2147
2148 /*
2149 * We'd better keep the last two bits, assuming that guest OS
2150 * might modify it. Keep it does not hurt after all.
2151 */
2152 irq.msi_addr_last_bits = addr.__not_care;
2153
2154 /* Translate VTDIrq to MSI message */
2155 vtd_generate_msi_message(&irq, translated);
2156
2157 VTD_DPRINTF(IR, "mapping MSI 0x%"PRIx64":0x%"PRIx32 " -> "
2158 "0x%"PRIx64":0x%"PRIx32, origin->address, origin->data,
2159 translated->address, translated->data);
2160 return 0;
2161
2162do_not_translate:
2163 memcpy(translated, origin, sizeof(*origin));
2164 return 0;
2165}
2166
2167static MemTxResult vtd_mem_ir_read(void *opaque, hwaddr addr,
2168 uint64_t *data, unsigned size,
2169 MemTxAttrs attrs)
2170{
2171 return MEMTX_OK;
2172}
2173
2174static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr,
2175 uint64_t value, unsigned size,
2176 MemTxAttrs attrs)
2177{
2178 int ret = 0;
09cd058a 2179 MSIMessage from = {}, to = {};
651e4cef
PX		 2180
2181 from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST;
2182 from.data = (uint32_t) value;
2183
2184 ret = vtd_interrupt_remap_msi(opaque, &from, &to);
2185 if (ret) {
2186 /* TODO: report error */
2187 VTD_DPRINTF(GENERAL, "int remap fail for addr 0x%"PRIx64
2188 " data 0x%"PRIx32, from.address, from.data);
2189 /* Drop this interrupt */
2190 return MEMTX_ERROR;
2191 }
2192
2193 VTD_DPRINTF(IR, "delivering MSI 0x%"PRIx64":0x%"PRIx32
2194 " for device sid 0x%04x",
2195 to.address, to.data, sid);
2196
2197 if (dma_memory_write(&address_space_memory, to.address,
2198 &to.data, size)) {
2199 VTD_DPRINTF(GENERAL, "error: fail to write 0x%"PRIx64
2200 " value 0x%"PRIx32, to.address, to.data);
2201 }
2202
2203 return MEMTX_OK;
2204}
2205
2206static const MemoryRegionOps vtd_mem_ir_ops = {
2207 .read_with_attrs = vtd_mem_ir_read,
2208 .write_with_attrs = vtd_mem_ir_write,
2209 .endianness = DEVICE_LITTLE_ENDIAN,
2210 .impl = {
2211 .min_access_size = 4,
2212 .max_access_size = 4,
2213 },
2214 .valid = {
2215 .min_access_size = 4,
2216 .max_access_size = 4,
2217 },
2218};
7df953bd
KO		 2219
2220VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn)
2221{
2222 uintptr_t key = (uintptr_t)bus;
2223 VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key);
2224 VTDAddressSpace *vtd_dev_as;
2225
2226 if (!vtd_bus) {
2227 /* No corresponding free() */
04af0e18
PX		 2228 vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * \
2229 X86_IOMMU_PCI_DEVFN_MAX);
7df953bd
KO		 2230 vtd_bus->bus = bus;
2231 key = (uintptr_t)bus;
2232 g_hash_table_insert(s->vtd_as_by_busptr, &key, vtd_bus);
2233 }
2234
2235 vtd_dev_as = vtd_bus->dev_as[devfn];
2236
2237 if (!vtd_dev_as) {
2238 vtd_bus->dev_as[devfn] = vtd_dev_as = g_malloc0(sizeof(VTDAddressSpace));
2239
2240 vtd_dev_as->bus = bus;
2241 vtd_dev_as->devfn = (uint8_t)devfn;
2242 vtd_dev_as->iommu_state = s;
2243 vtd_dev_as->context_cache_entry.context_cache_gen = 0;
2244 memory_region_init_iommu(&vtd_dev_as->iommu, OBJECT(s),
2245 &s->iommu_ops, "intel_iommu", UINT64_MAX);
651e4cef
PX		 2246 memory_region_init_io(&vtd_dev_as->iommu_ir, OBJECT(s),
2247 &vtd_mem_ir_ops, s, "intel_iommu_ir",
2248 VTD_INTERRUPT_ADDR_SIZE);
2249 memory_region_add_subregion(&vtd_dev_as->iommu, VTD_INTERRUPT_ADDR_FIRST,
2250 &vtd_dev_as->iommu_ir);
7df953bd
KO		 2251 address_space_init(&vtd_dev_as->as,
2252 &vtd_dev_as->iommu, "intel_iommu");
2253 }
2254 return vtd_dev_as;
2255}
2256
1da12ec4
LT		 2257/* Do the initialization. It will also be called when reset, so pay
2258 * attention when adding new initialization stuff.
2259 */
2260static void vtd_init(IntelIOMMUState *s)
2261{
d54bd7f8
PX		 2262 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
2263
1da12ec4
LT		 2264 memset(s->csr, 0, DMAR_REG_SIZE);
2265 memset(s->wmask, 0, DMAR_REG_SIZE);
2266 memset(s->w1cmask, 0, DMAR_REG_SIZE);
2267 memset(s->womask, 0, DMAR_REG_SIZE);
2268
2269 s->iommu_ops.translate = vtd_iommu_translate;
3cb3b154 2270 s->iommu_ops.notify_started = vtd_iommu_notify_started;
1da12ec4
LT		 2271 s->root = 0;
2272 s->root_extended = false;
2273 s->dmar_enabled = false;
2274 s->iq_head = 0;
2275 s->iq_tail = 0;
2276 s->iq = 0;
2277 s->iq_size = 0;
2278 s->qi_enabled = false;
2279 s->iq_last_desc_type = VTD_INV_DESC_NONE;
2280 s->next_frcd_reg = 0;
2281 s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | VTD_CAP_MGAW |
d66b969b 2282 VTD_CAP_SAGAW | VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS;
ed7b8fbc 2283 s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO;
1da12ec4 2284
d54bd7f8
PX		 2285 if (x86_iommu->intr_supported) {
2286 s->ecap |= VTD_ECAP_IR;
2287 }
2288
d92fa2dc 2289 vtd_reset_context_cache(s);
b5a280c0 2290 vtd_reset_iotlb(s);
d92fa2dc 2291
1da12ec4
LT		 2292 /* Define registers with default values and bit semantics */
2293 vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0);
2294 vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0);
2295 vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0);
2296 vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0);
2297 vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL);
2298 vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0);
2299 vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffff000ULL, 0);
2300 vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0);
2301 vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL);
2302
2303 /* Advanced Fault Logging not supported */
2304 vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL);
2305 vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0);
2306 vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0);
2307 vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0);
2308
2309 /* Treated as RsvdZ when EIM in ECAP_REG is not supported
2310 * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0);
2311 */
2312 vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0);
2313
2314 /* Treated as RO for implementations that PLMR and PHMR fields reported
2315 * as Clear in the CAP_REG.
2316 * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0);
2317 */
2318 vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0);
2319
ed7b8fbc
LT		 2320 vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0);
2321 vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0);
2322 vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff007ULL, 0);
2323 vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL);
2324 vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0);
2325 vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0);
2326 vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0);
2327 /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */
2328 vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0);
2329
1da12ec4
LT		 2330 /* IOTLB registers */
2331 vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0);
2332 vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0);
2333 vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL);
2334
2335 /* Fault Recording Registers, 128-bit */
2336 vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0);
2337 vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL);
a5861439
PX		 2338
2339 /*
2340 * Interrupt remapping registers, not support extended interrupt
2341 * mode for now.
2342 */
2343 vtd_define_quad(s, DMAR_IRTA_REG, 0, 0xfffffffffffff00fULL, 0);
1da12ec4
LT		 2344}
2345
2346/* Should not reset address_spaces when reset because devices will still use
2347 * the address space they got at first (won't ask the bus again).
2348 */
2349static void vtd_reset(DeviceState *dev)
2350{
2351 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
2352
2353 VTD_DPRINTF(GENERAL, "");
2354 vtd_init(s);
2355}
2356
621d983a
MA		 2357static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
2358{
2359 IntelIOMMUState *s = opaque;
2360 VTDAddressSpace *vtd_as;
2361
04af0e18 2362 assert(0 <= devfn && devfn <= X86_IOMMU_PCI_DEVFN_MAX);
621d983a
MA		 2363
2364 vtd_as = vtd_find_add_as(s, bus, devfn);
2365 return &vtd_as->as;
2366}
2367
1da12ec4
LT		 2368static void vtd_realize(DeviceState *dev, Error **errp)
2369{
621d983a 2370 PCIBus *bus = PC_MACHINE(qdev_get_machine())->bus;
1da12ec4
LT		 2371 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
2372
2373 VTD_DPRINTF(GENERAL, "");
7df953bd 2374 memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num));
1da12ec4
LT		 2375 memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s,
2376 "intel_iommu", DMAR_REG_SIZE);
2377 sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem);
b5a280c0
LT		 2378 /* No corresponding destroy */
2379 s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
2380 g_free, g_free);
7df953bd
KO		 2381 s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
2382 g_free, g_free);
1da12ec4 2383 vtd_init(s);
621d983a
MA		 2384 sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, Q35_HOST_BRIDGE_IOMMU_ADDR);
2385 pci_setup_iommu(bus, vtd_host_dma_iommu, dev);
1da12ec4
LT		 2386}
2387
2388static void vtd_class_init(ObjectClass *klass, void *data)
2389{
2390 DeviceClass *dc = DEVICE_CLASS(klass);
1c7955c4 2391 X86IOMMUClass *x86_class = X86_IOMMU_CLASS(klass);
1da12ec4
LT		 2392
2393 dc->reset = vtd_reset;
1da12ec4
LT		 2394 dc->vmsd = &vtd_vmstate;
2395 dc->props = vtd_properties;
621d983a 2396 dc->hotpluggable = false;
1c7955c4 2397 x86_class->realize = vtd_realize;
1da12ec4
LT		 2398}
2399
2400static const TypeInfo vtd_info = {
2401 .name = TYPE_INTEL_IOMMU_DEVICE,
1c7955c4 2402 .parent = TYPE_X86_IOMMU_DEVICE,
1da12ec4
LT		 2403 .instance_size = sizeof(IntelIOMMUState),
2404 .class_init = vtd_class_init,
2405};
2406
2407static void vtd_register_types(void)
2408{
2409 VTD_DPRINTF(GENERAL, "");
2410 type_register_static(&vtd_info);
2411}
2412
2413type_init(vtd_register_types)
QEMU mirror