]> git.proxmox.com Git - mirror_qemu.git/blame - hw/i386/intel_iommu.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
intel_iommu: use the correct memory region for device IOTLB notification
[mirror_qemu.git] / hw / i386 / intel_iommu.c
CommitLineData
1da12ec4
LT		 1/*
2 * QEMU emulation of an Intel IOMMU (VT-d)
3 * (DMA Remapping device)
4 *
5 * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com>
6 * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, see <http://www.gnu.org/licenses/>.
20 */
21
b6a0aa05 22#include "qemu/osdep.h"
4684a204 23#include "qemu/error-report.h"
6333e93c 24#include "qapi/error.h"
1da12ec4
LT		 25#include "hw/sysbus.h"
26#include "exec/address-spaces.h"
27#include "intel_iommu_internal.h"
7df953bd 28#include "hw/pci/pci.h"
3cb3b154 29#include "hw/pci/pci_bus.h"
621d983a 30#include "hw/i386/pc.h"
dea651a9 31#include "hw/i386/apic-msidef.h"
04af0e18
PX		 32#include "hw/boards.h"
33#include "hw/i386/x86-iommu.h"
cb135f59 34#include "hw/pci-host/q35.h"
4684a204 35#include "sysemu/kvm.h"
32946019 36#include "hw/i386/apic_internal.h"
fb506e70 37#include "kvm_i386.h"
bc535e59 38#include "trace.h"
1da12ec4
LT		 39
40/*#define DEBUG_INTEL_IOMMU*/
41#ifdef DEBUG_INTEL_IOMMU
42enum {
43 DEBUG_GENERAL, DEBUG_CSR, DEBUG_INV, DEBUG_MMU, DEBUG_FLOG,
a5861439 44 DEBUG_CACHE, DEBUG_IR,
1da12ec4
LT		 45};
46#define VTD_DBGBIT(x) (1 << DEBUG_##x)
47static int vtd_dbgflags = VTD_DBGBIT(GENERAL) | VTD_DBGBIT(CSR);
48
49#define VTD_DPRINTF(what, fmt, ...) do { \
50 if (vtd_dbgflags & VTD_DBGBIT(what)) { \
51 fprintf(stderr, "(vtd)%s: " fmt "\n", __func__, \
52 ## __VA_ARGS__); } \
53 } while (0)
54#else
55#define VTD_DPRINTF(what, fmt, ...) do {} while (0)
56#endif
57
58static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val,
59 uint64_t wmask, uint64_t w1cmask)
60{
61 stq_le_p(&s->csr[addr], val);
62 stq_le_p(&s->wmask[addr], wmask);
63 stq_le_p(&s->w1cmask[addr], w1cmask);
64}
65
66static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask)
67{
68 stq_le_p(&s->womask[addr], mask);
69}
70
71static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val,
72 uint32_t wmask, uint32_t w1cmask)
73{
74 stl_le_p(&s->csr[addr], val);
75 stl_le_p(&s->wmask[addr], wmask);
76 stl_le_p(&s->w1cmask[addr], w1cmask);
77}
78
79static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask)
80{
81 stl_le_p(&s->womask[addr], mask);
82}
83
84/* "External" get/set operations */
85static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val)
86{
87 uint64_t oldval = ldq_le_p(&s->csr[addr]);
88 uint64_t wmask = ldq_le_p(&s->wmask[addr]);
89 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]);
90 stq_le_p(&s->csr[addr],
91 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val));
92}
93
94static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val)
95{
96 uint32_t oldval = ldl_le_p(&s->csr[addr]);
97 uint32_t wmask = ldl_le_p(&s->wmask[addr]);
98 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]);
99 stl_le_p(&s->csr[addr],
100 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val));
101}
102
103static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr)
104{
105 uint64_t val = ldq_le_p(&s->csr[addr]);
106 uint64_t womask = ldq_le_p(&s->womask[addr]);
107 return val & ~womask;
108}
109
110static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr)
111{
112 uint32_t val = ldl_le_p(&s->csr[addr]);
113 uint32_t womask = ldl_le_p(&s->womask[addr]);
114 return val & ~womask;
115}
116
117/* "Internal" get/set operations */
118static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr)
119{
120 return ldq_le_p(&s->csr[addr]);
121}
122
123static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr)
124{
125 return ldl_le_p(&s->csr[addr]);
126}
127
128static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val)
129{
130 stq_le_p(&s->csr[addr], val);
131}
132
133static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr,
134 uint32_t clear, uint32_t mask)
135{
136 uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask;
137 stl_le_p(&s->csr[addr], new_val);
138 return new_val;
139}
140
141static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr,
142 uint64_t clear, uint64_t mask)
143{
144 uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask;
145 stq_le_p(&s->csr[addr], new_val);
146 return new_val;
147}
148
b5a280c0
LT		 149/* GHashTable functions */
150static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2)
151{
152 return *((const uint64_t *)v1) == *((const uint64_t *)v2);
153}
154
155static guint vtd_uint64_hash(gconstpointer v)
156{
157 return (guint)*(const uint64_t *)v;
158}
159
160static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value,
161 gpointer user_data)
162{
163 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
164 uint16_t domain_id = *(uint16_t *)user_data;
165 return entry->domain_id == domain_id;
166}
167
d66b969b
JW		 168/* The shift of an addr for a certain level of paging structure */
169static inline uint32_t vtd_slpt_level_shift(uint32_t level)
170{
7e58326a 171 assert(level != 0);
d66b969b
JW		 172 return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS;
173}
174
175static inline uint64_t vtd_slpt_level_page_mask(uint32_t level)
176{
177 return ~((1ULL << vtd_slpt_level_shift(level)) - 1);
178}
179
b5a280c0
LT		 180static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value,
181 gpointer user_data)
182{
183 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
184 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data;
d66b969b
JW		 185 uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask;
186 uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K;
b5a280c0 187 return (entry->domain_id == info->domain_id) &&
d66b969b
JW		 188 (((entry->gfn & info->mask) == gfn) ||
189 (entry->gfn == gfn_tlb));
b5a280c0
LT		 190}
191
d92fa2dc
LT		 192/* Reset all the gen of VTDAddressSpace to zero and set the gen of
193 * IntelIOMMUState to 1.
194 */
195static void vtd_reset_context_cache(IntelIOMMUState *s)
196{
d92fa2dc 197 VTDAddressSpace *vtd_as;
7df953bd
KO		 198 VTDBus *vtd_bus;
199 GHashTableIter bus_it;
d92fa2dc
LT		 200 uint32_t devfn_it;
201
7df953bd
KO		 202 g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr);
203
d92fa2dc 204 VTD_DPRINTF(CACHE, "global context_cache_gen=1");
7df953bd 205 while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) {
04af0e18 206 for (devfn_it = 0; devfn_it < X86_IOMMU_PCI_DEVFN_MAX; ++devfn_it) {
7df953bd 207 vtd_as = vtd_bus->dev_as[devfn_it];
d92fa2dc
LT		 208 if (!vtd_as) {
209 continue;
210 }
211 vtd_as->context_cache_entry.context_cache_gen = 0;
212 }
213 }
214 s->context_cache_gen = 1;
215}
216
b5a280c0
LT		 217static void vtd_reset_iotlb(IntelIOMMUState *s)
218{
219 assert(s->iotlb);
220 g_hash_table_remove_all(s->iotlb);
221}
222
bacabb0a 223static uint64_t vtd_get_iotlb_key(uint64_t gfn, uint16_t source_id,
d66b969b
JW		 224 uint32_t level)
225{
226 return gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT) |
227 ((uint64_t)(level) << VTD_IOTLB_LVL_SHIFT);
228}
229
230static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level)
231{
232 return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K;
233}
234
b5a280c0
LT		 235static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id,
236 hwaddr addr)
237{
d66b969b 238 VTDIOTLBEntry *entry;
b5a280c0 239 uint64_t key;
d66b969b
JW		 240 int level;
241
242 for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) {
243 key = vtd_get_iotlb_key(vtd_get_iotlb_gfn(addr, level),
244 source_id, level);
245 entry = g_hash_table_lookup(s->iotlb, &key);
246 if (entry) {
247 goto out;
248 }
249 }
b5a280c0 250
d66b969b
JW		 251out:
252 return entry;
b5a280c0
LT		 253}
254
255static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id,
256 uint16_t domain_id, hwaddr addr, uint64_t slpte,
d66b969b
JW		 257 bool read_flags, bool write_flags,
258 uint32_t level)
b5a280c0
LT		 259{
260 VTDIOTLBEntry *entry = g_malloc(sizeof(*entry));
261 uint64_t *key = g_malloc(sizeof(*key));
d66b969b 262 uint64_t gfn = vtd_get_iotlb_gfn(addr, level);
b5a280c0 263
6c441e1d 264 trace_vtd_iotlb_page_update(source_id, addr, slpte, domain_id);
b5a280c0 265 if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) {
6c441e1d 266 trace_vtd_iotlb_reset("iotlb exceeds size limit");
b5a280c0
LT		 267 vtd_reset_iotlb(s);
268 }
269
270 entry->gfn = gfn;
271 entry->domain_id = domain_id;
272 entry->slpte = slpte;
273 entry->read_flags = read_flags;
274 entry->write_flags = write_flags;
d66b969b
JW		 275 entry->mask = vtd_slpt_level_page_mask(level);
276 *key = vtd_get_iotlb_key(gfn, source_id, level);
b5a280c0
LT		 277 g_hash_table_replace(s->iotlb, key, entry);
278}
279
1da12ec4
LT		 280/* Given the reg addr of both the message data and address, generate an
281 * interrupt via MSI.
282 */
283static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg,
284 hwaddr mesg_data_reg)
285{
32946019 286 MSIMessage msi;
1da12ec4
LT		 287
288 assert(mesg_data_reg < DMAR_REG_SIZE);
289 assert(mesg_addr_reg < DMAR_REG_SIZE);
290
32946019
RK		 291 msi.address = vtd_get_long_raw(s, mesg_addr_reg);
292 msi.data = vtd_get_long_raw(s, mesg_data_reg);
1da12ec4 293
32946019
RK		 294 VTD_DPRINTF(FLOG, "msi: addr 0x%"PRIx64 " data 0x%"PRIx32,
295 msi.address, msi.data);
296 apic_get_class()->send_msi(&msi);
1da12ec4
LT		 297}
298
299/* Generate a fault event to software via MSI if conditions are met.
300 * Notice that the value of FSTS_REG being passed to it should be the one
301 * before any update.
302 */
303static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts)
304{
305 if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO ||
306 pre_fsts & VTD_FSTS_IQE) {
307 VTD_DPRINTF(FLOG, "there are previous interrupt conditions "
308 "to be serviced by software, fault event is not generated "
309 "(FSTS_REG 0x%"PRIx32 ")", pre_fsts);
310 return;
311 }
312 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP);
313 if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) {
314 VTD_DPRINTF(FLOG, "Interrupt Mask set, fault event is not generated");
315 } else {
316 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
317 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
318 }
319}
320
321/* Check if the Fault (F) field of the Fault Recording Register referenced by
322 * @index is Set.
323 */
324static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index)
325{
326 /* Each reg is 128-bit */
327 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
328 addr += 8; /* Access the high 64-bit half */
329
330 assert(index < DMAR_FRCD_REG_NR);
331
332 return vtd_get_quad_raw(s, addr) & VTD_FRCD_F;
333}
334
335/* Update the PPF field of Fault Status Register.
336 * Should be called whenever change the F field of any fault recording
337 * registers.
338 */
339static void vtd_update_fsts_ppf(IntelIOMMUState *s)
340{
341 uint32_t i;
342 uint32_t ppf_mask = 0;
343
344 for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
345 if (vtd_is_frcd_set(s, i)) {
346 ppf_mask = VTD_FSTS_PPF;
347 break;
348 }
349 }
350 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask);
351 VTD_DPRINTF(FLOG, "set PPF of FSTS_REG to %d", ppf_mask ? 1 : 0);
352}
353
354static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index)
355{
356 /* Each reg is 128-bit */
357 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
358 addr += 8; /* Access the high 64-bit half */
359
360 assert(index < DMAR_FRCD_REG_NR);
361
362 vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F);
363 vtd_update_fsts_ppf(s);
364}
365
366/* Must not update F field now, should be done later */
367static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index,
368 uint16_t source_id, hwaddr addr,
369 VTDFaultReason fault, bool is_write)
370{
371 uint64_t hi = 0, lo;
372 hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
373
374 assert(index < DMAR_FRCD_REG_NR);
375
376 lo = VTD_FRCD_FI(addr);
377 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault);
378 if (!is_write) {
379 hi |= VTD_FRCD_T;
380 }
381 vtd_set_quad_raw(s, frcd_reg_addr, lo);
382 vtd_set_quad_raw(s, frcd_reg_addr + 8, hi);
383 VTD_DPRINTF(FLOG, "record to FRCD_REG #%"PRIu16 ": hi 0x%"PRIx64
384 ", lo 0x%"PRIx64, index, hi, lo);
385}
386
387/* Try to collapse multiple pending faults from the same requester */
388static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id)
389{
390 uint32_t i;
391 uint64_t frcd_reg;
392 hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */
393
394 for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
395 frcd_reg = vtd_get_quad_raw(s, addr);
396 VTD_DPRINTF(FLOG, "frcd_reg #%d 0x%"PRIx64, i, frcd_reg);
397 if ((frcd_reg & VTD_FRCD_F) &&
398 ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) {
399 return true;
400 }
401 addr += 16; /* 128-bit for each */
402 }
403 return false;
404}
405
406/* Log and report an DMAR (address translation) fault to software */
407static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id,
408 hwaddr addr, VTDFaultReason fault,
409 bool is_write)
410{
411 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
412
413 assert(fault < VTD_FR_MAX);
414
415 if (fault == VTD_FR_RESERVED_ERR) {
416 /* This is not a normal fault reason case. Drop it. */
417 return;
418 }
419 VTD_DPRINTF(FLOG, "sid 0x%"PRIx16 ", fault %d, addr 0x%"PRIx64
420 ", is_write %d", source_id, fault, addr, is_write);
421 if (fsts_reg & VTD_FSTS_PFO) {
422 VTD_DPRINTF(FLOG, "new fault is not recorded due to "
423 "Primary Fault Overflow");
424 return;
425 }
426 if (vtd_try_collapse_fault(s, source_id)) {
427 VTD_DPRINTF(FLOG, "new fault is not recorded due to "
428 "compression of faults");
429 return;
430 }
431 if (vtd_is_frcd_set(s, s->next_frcd_reg)) {
432 VTD_DPRINTF(FLOG, "Primary Fault Overflow and "
433 "new fault is not recorded, set PFO field");
434 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO);
435 return;
436 }
437
438 vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write);
439
440 if (fsts_reg & VTD_FSTS_PPF) {
441 VTD_DPRINTF(FLOG, "there are pending faults already, "
442 "fault event is not generated");
443 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg);
444 s->next_frcd_reg++;
445 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
446 s->next_frcd_reg = 0;
447 }
448 } else {
449 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK,
450 VTD_FSTS_FRI(s->next_frcd_reg));
451 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */
452 s->next_frcd_reg++;
453 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
454 s->next_frcd_reg = 0;
455 }
456 /* This case actually cause the PPF to be Set.
457 * So generate fault event (interrupt).
458 */
459 vtd_generate_fault_event(s, fsts_reg);
460 }
461}
462
ed7b8fbc
LT		 463/* Handle Invalidation Queue Errors of queued invalidation interface error
464 * conditions.
465 */
466static void vtd_handle_inv_queue_error(IntelIOMMUState *s)
467{
468 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
469
470 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE);
471 vtd_generate_fault_event(s, fsts_reg);
472}
473
474/* Set the IWC field and try to generate an invalidation completion interrupt */
475static void vtd_generate_completion_event(IntelIOMMUState *s)
476{
ed7b8fbc 477 if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) {
bc535e59 478 trace_vtd_inv_desc_wait_irq("One pending, skip current");
ed7b8fbc
LT		 479 return;
480 }
481 vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC);
482 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP);
483 if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) {
bc535e59
PX		 484 trace_vtd_inv_desc_wait_irq("IM in IECTL_REG is set, "
485 "new event not generated");
ed7b8fbc
LT		 486 return;
487 } else {
488 /* Generate the interrupt event */
bc535e59 489 trace_vtd_inv_desc_wait_irq("Generating complete event");
ed7b8fbc
LT		 490 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
491 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
492 }
493}
494
1da12ec4
LT		 495static inline bool vtd_root_entry_present(VTDRootEntry *root)
496{
497 return root->val & VTD_ROOT_ENTRY_P;
498}
499
500static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index,
501 VTDRootEntry *re)
502{
503 dma_addr_t addr;
504
505 addr = s->root + index * sizeof(*re);
506 if (dma_memory_read(&address_space_memory, addr, re, sizeof(*re))) {
6c441e1d 507 trace_vtd_re_invalid(re->rsvd, re->val);
1da12ec4
LT		 508 re->val = 0;
509 return -VTD_FR_ROOT_TABLE_INV;
510 }
511 re->val = le64_to_cpu(re->val);
512 return 0;
513}
514
515static inline bool vtd_context_entry_present(VTDContextEntry *context)
516{
517 return context->lo & VTD_CONTEXT_ENTRY_P;
518}
519
520static int vtd_get_context_entry_from_root(VTDRootEntry *root, uint8_t index,
521 VTDContextEntry *ce)
522{
523 dma_addr_t addr;
524
6c441e1d 525 /* we have checked that root entry is present */
1da12ec4
LT		 526 addr = (root->val & VTD_ROOT_ENTRY_CTP) + index * sizeof(*ce);
527 if (dma_memory_read(&address_space_memory, addr, ce, sizeof(*ce))) {
6c441e1d 528 trace_vtd_re_invalid(root->rsvd, root->val);
1da12ec4
LT		 529 return -VTD_FR_CONTEXT_TABLE_INV;
530 }
531 ce->lo = le64_to_cpu(ce->lo);
532 ce->hi = le64_to_cpu(ce->hi);
533 return 0;
534}
535
536static inline dma_addr_t vtd_get_slpt_base_from_context(VTDContextEntry *ce)
537{
538 return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR;
539}
540
1da12ec4
LT		 541static inline uint64_t vtd_get_slpte_addr(uint64_t slpte)
542{
543 return slpte & VTD_SL_PT_BASE_ADDR_MASK;
544}
545
546/* Whether the pte indicates the address of the page frame */
547static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level)
548{
549 return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK);
550}
551
552/* Get the content of a spte located in @base_addr[@index] */
553static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index)
554{
555 uint64_t slpte;
556
557 assert(index < VTD_SL_PT_ENTRY_NR);
558
559 if (dma_memory_read(&address_space_memory,
560 base_addr + index * sizeof(slpte), &slpte,
561 sizeof(slpte))) {
562 slpte = (uint64_t)-1;
563 return slpte;
564 }
565 slpte = le64_to_cpu(slpte);
566 return slpte;
567}
568
6e905564
PX		 569/* Given an iova and the level of paging structure, return the offset
570 * of current level.
1da12ec4 571 */
6e905564 572static inline uint32_t vtd_iova_level_offset(uint64_t iova, uint32_t level)
1da12ec4 573{
6e905564 574 return (iova >> vtd_slpt_level_shift(level)) &
1da12ec4
LT		 575 ((1ULL << VTD_SL_LEVEL_BITS) - 1);
576}
577
578/* Check Capability Register to see if the @level of page-table is supported */
579static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level)
580{
581 return VTD_CAP_SAGAW_MASK & s->cap &
582 (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT));
583}
584
585/* Get the page-table level that hardware should use for the second-level
586 * page-table walk from the Address Width field of context-entry.
587 */
588static inline uint32_t vtd_get_level_from_context_entry(VTDContextEntry *ce)
589{
590 return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW);
591}
592
593static inline uint32_t vtd_get_agaw_from_context_entry(VTDContextEntry *ce)
594{
595 return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9;
596}
597
598static const uint64_t vtd_paging_entry_rsvd_field[] = {
599 [0] = ~0ULL,
600 /* For not large page */
601 [1] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
602 [2] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
603 [3] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
604 [4] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
605 /* For large page */
606 [5] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
607 [6] = 0x1ff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
608 [7] = 0x3ffff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
609 [8] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
610};
611
612static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level)
613{
614 if (slpte & VTD_SL_PT_PAGE_SIZE_MASK) {
615 /* Maybe large page */
616 return slpte & vtd_paging_entry_rsvd_field[level + 4];
617 } else {
618 return slpte & vtd_paging_entry_rsvd_field[level];
619 }
620}
621
6e905564 622/* Given the @iova, get relevant @slptep. @slpte_level will be the last level
1da12ec4
LT		 623 * of the translation, can be used for deciding the size of large page.
624 */
6e905564
PX		 625static int vtd_iova_to_slpte(VTDContextEntry *ce, uint64_t iova, bool is_write,
626 uint64_t *slptep, uint32_t *slpte_level,
627 bool *reads, bool *writes)
1da12ec4
LT		 628{
629 dma_addr_t addr = vtd_get_slpt_base_from_context(ce);
630 uint32_t level = vtd_get_level_from_context_entry(ce);
631 uint32_t offset;
632 uint64_t slpte;
633 uint32_t ce_agaw = vtd_get_agaw_from_context_entry(ce);
634 uint64_t access_right_check;
635
6e905564
PX		 636 /* Check if @iova is above 2^X-1, where X is the minimum of MGAW
637 * in CAP_REG and AW in context-entry.
1da12ec4 638 */
6e905564
PX		 639 if (iova & ~((1ULL << MIN(ce_agaw, VTD_MGAW)) - 1)) {
640 VTD_DPRINTF(GENERAL, "error: iova 0x%"PRIx64 " exceeds limits", iova);
1da12ec4
LT		 641 return -VTD_FR_ADDR_BEYOND_MGAW;
642 }
643
644 /* FIXME: what is the Atomics request here? */
645 access_right_check = is_write ? VTD_SL_W : VTD_SL_R;
646
647 while (true) {
6e905564 648 offset = vtd_iova_level_offset(iova, level);
1da12ec4
LT		 649 slpte = vtd_get_slpte(addr, offset);
650
651 if (slpte == (uint64_t)-1) {
652 VTD_DPRINTF(GENERAL, "error: fail to access second-level paging "
6e905564
PX		 653 "entry at level %"PRIu32 " for iova 0x%"PRIx64,
654 level, iova);
1da12ec4
LT		 655 if (level == vtd_get_level_from_context_entry(ce)) {
656 /* Invalid programming of context-entry */
657 return -VTD_FR_CONTEXT_ENTRY_INV;
658 } else {
659 return -VTD_FR_PAGING_ENTRY_INV;
660 }
661 }
662 *reads = (*reads) && (slpte & VTD_SL_R);
663 *writes = (*writes) && (slpte & VTD_SL_W);
664 if (!(slpte & access_right_check)) {
665 VTD_DPRINTF(GENERAL, "error: lack of %s permission for "
6e905564
PX		 666 "iova 0x%"PRIx64 " slpte 0x%"PRIx64,
667 (is_write ? "write" : "read"), iova, slpte);
1da12ec4
LT		 668 return is_write ? -VTD_FR_WRITE : -VTD_FR_READ;
669 }
670 if (vtd_slpte_nonzero_rsvd(slpte, level)) {
671 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in second "
672 "level paging entry level %"PRIu32 " slpte 0x%"PRIx64,
673 level, slpte);
674 return -VTD_FR_PAGING_ENTRY_RSVD;
675 }
676
677 if (vtd_is_last_slpte(slpte, level)) {
678 *slptep = slpte;
679 *slpte_level = level;
680 return 0;
681 }
682 addr = vtd_get_slpte_addr(slpte);
683 level--;
684 }
685}
686
687/* Map a device to its corresponding domain (context-entry) */
688static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num,
689 uint8_t devfn, VTDContextEntry *ce)
690{
691 VTDRootEntry re;
692 int ret_fr;
693
694 ret_fr = vtd_get_root_entry(s, bus_num, &re);
695 if (ret_fr) {
696 return ret_fr;
697 }
698
699 if (!vtd_root_entry_present(&re)) {
6c441e1d
PX		 700 /* Not error - it's okay we don't have root entry. */
701 trace_vtd_re_not_present(bus_num);
1da12ec4
LT		 702 return -VTD_FR_ROOT_ENTRY_P;
703 } else if (re.rsvd || (re.val & VTD_ROOT_ENTRY_RSVD)) {
6c441e1d 704 trace_vtd_re_invalid(re.rsvd, re.val);
1da12ec4
LT		 705 return -VTD_FR_ROOT_ENTRY_RSVD;
706 }
707
708 ret_fr = vtd_get_context_entry_from_root(&re, devfn, ce);
709 if (ret_fr) {
710 return ret_fr;
711 }
712
713 if (!vtd_context_entry_present(ce)) {
6c441e1d
PX		 714 /* Not error - it's okay we don't have context entry. */
715 trace_vtd_ce_not_present(bus_num, devfn);
1da12ec4
LT		 716 return -VTD_FR_CONTEXT_ENTRY_P;
717 } else if ((ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI) ||
718 (ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO)) {
6c441e1d 719 trace_vtd_ce_invalid(ce->hi, ce->lo);
1da12ec4
LT		 720 return -VTD_FR_CONTEXT_ENTRY_RSVD;
721 }
722 /* Check if the programming of context-entry is valid */
723 if (!vtd_is_level_supported(s, vtd_get_level_from_context_entry(ce))) {
6c441e1d 724 trace_vtd_ce_invalid(ce->hi, ce->lo);
1da12ec4 725 return -VTD_FR_CONTEXT_ENTRY_INV;
554f5e16
JW		 726 } else {
727 switch (ce->lo & VTD_CONTEXT_ENTRY_TT) {
728 case VTD_CONTEXT_TT_MULTI_LEVEL:
729 /* fall through */
730 case VTD_CONTEXT_TT_DEV_IOTLB:
731 break;
732 default:
6c441e1d 733 trace_vtd_ce_invalid(ce->hi, ce->lo);
554f5e16
JW		 734 return -VTD_FR_CONTEXT_ENTRY_INV;
735 }
1da12ec4
LT		 736 }
737 return 0;
738}
739
740static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn)
741{
742 return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL);
743}
744
745static const bool vtd_qualified_faults[] = {
746 [VTD_FR_RESERVED] = false,
747 [VTD_FR_ROOT_ENTRY_P] = false,
748 [VTD_FR_CONTEXT_ENTRY_P] = true,
749 [VTD_FR_CONTEXT_ENTRY_INV] = true,
750 [VTD_FR_ADDR_BEYOND_MGAW] = true,
751 [VTD_FR_WRITE] = true,
752 [VTD_FR_READ] = true,
753 [VTD_FR_PAGING_ENTRY_INV] = true,
754 [VTD_FR_ROOT_TABLE_INV] = false,
755 [VTD_FR_CONTEXT_TABLE_INV] = false,
756 [VTD_FR_ROOT_ENTRY_RSVD] = false,
757 [VTD_FR_PAGING_ENTRY_RSVD] = true,
758 [VTD_FR_CONTEXT_ENTRY_TT] = true,
759 [VTD_FR_RESERVED_ERR] = false,
760 [VTD_FR_MAX] = false,
761};
762
763/* To see if a fault condition is "qualified", which is reported to software
764 * only if the FPD field in the context-entry used to process the faulting
765 * request is 0.
766 */
767static inline bool vtd_is_qualified_fault(VTDFaultReason fault)
768{
769 return vtd_qualified_faults[fault];
770}
771
772static inline bool vtd_is_interrupt_addr(hwaddr addr)
773{
774 return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST;
775}
776
777/* Map dev to context-entry then do a paging-structures walk to do a iommu
778 * translation.
79e2b9ae
PB		 779 *
780 * Called from RCU critical section.
781 *
1da12ec4
LT		 782 * @bus_num: The bus number
783 * @devfn: The devfn, which is the combined of device and function number
784 * @is_write: The access is a write operation
785 * @entry: IOMMUTLBEntry that contain the addr to be translated and result
786 */
7df953bd 787static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus,
1da12ec4
LT		 788 uint8_t devfn, hwaddr addr, bool is_write,
789 IOMMUTLBEntry *entry)
790{
d92fa2dc 791 IntelIOMMUState *s = vtd_as->iommu_state;
1da12ec4 792 VTDContextEntry ce;
7df953bd 793 uint8_t bus_num = pci_bus_num(bus);
d92fa2dc 794 VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry;
d66b969b 795 uint64_t slpte, page_mask;
1da12ec4
LT		 796 uint32_t level;
797 uint16_t source_id = vtd_make_source_id(bus_num, devfn);
798 int ret_fr;
799 bool is_fpd_set = false;
800 bool reads = true;
801 bool writes = true;
b5a280c0 802 VTDIOTLBEntry *iotlb_entry;
1da12ec4 803
046ab7e9
PX		 804 /*
805 * We have standalone memory region for interrupt addresses, we
806 * should never receive translation requests in this region.
807 */
808 assert(!vtd_is_interrupt_addr(addr));
809
b5a280c0
LT		 810 /* Try to fetch slpte form IOTLB */
811 iotlb_entry = vtd_lookup_iotlb(s, source_id, addr);
812 if (iotlb_entry) {
6c441e1d
PX		 813 trace_vtd_iotlb_page_hit(source_id, addr, iotlb_entry->slpte,
814 iotlb_entry->domain_id);
b5a280c0
LT		 815 slpte = iotlb_entry->slpte;
816 reads = iotlb_entry->read_flags;
817 writes = iotlb_entry->write_flags;
d66b969b 818 page_mask = iotlb_entry->mask;
b5a280c0
LT		 819 goto out;
820 }
d92fa2dc
LT		 821 /* Try to fetch context-entry from cache first */
822 if (cc_entry->context_cache_gen == s->context_cache_gen) {
6c441e1d
PX		 823 trace_vtd_iotlb_cc_hit(bus_num, devfn, cc_entry->context_entry.hi,
824 cc_entry->context_entry.lo,
825 cc_entry->context_cache_gen);
d92fa2dc
LT		 826 ce = cc_entry->context_entry;
827 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
828 } else {
829 ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce);
830 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
831 if (ret_fr) {
832 ret_fr = -ret_fr;
833 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {
6c441e1d 834 trace_vtd_fault_disabled();
d92fa2dc
LT		 835 } else {
836 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);
837 }
838 return;
1da12ec4 839 }
d92fa2dc 840 /* Update context-cache */
6c441e1d
PX		 841 trace_vtd_iotlb_cc_update(bus_num, devfn, ce.hi, ce.lo,
842 cc_entry->context_cache_gen,
843 s->context_cache_gen);
d92fa2dc
LT		 844 cc_entry->context_entry = ce;
845 cc_entry->context_cache_gen = s->context_cache_gen;
1da12ec4
LT		 846 }
847
6e905564
PX		 848 ret_fr = vtd_iova_to_slpte(&ce, addr, is_write, &slpte, &level,
849 &reads, &writes);
1da12ec4
LT		 850 if (ret_fr) {
851 ret_fr = -ret_fr;
852 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {
6c441e1d 853 trace_vtd_fault_disabled();
1da12ec4
LT		 854 } else {
855 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);
856 }
857 return;
858 }
859
d66b969b 860 page_mask = vtd_slpt_level_page_mask(level);
b5a280c0 861 vtd_update_iotlb(s, source_id, VTD_CONTEXT_ENTRY_DID(ce.hi), addr, slpte,
d66b969b 862 reads, writes, level);
b5a280c0 863out:
d66b969b
JW		 864 entry->iova = addr & page_mask;
865 entry->translated_addr = vtd_get_slpte_addr(slpte) & page_mask;
866 entry->addr_mask = ~page_mask;
1da12ec4
LT		 867 entry->perm = (writes ? 2 : 0) + (reads ? 1 : 0);
868}
869
870static void vtd_root_table_setup(IntelIOMMUState *s)
871{
872 s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG);
873 s->root_extended = s->root & VTD_RTADDR_RTT;
874 s->root &= VTD_RTADDR_ADDR_MASK;
875
876 VTD_DPRINTF(CSR, "root_table addr 0x%"PRIx64 " %s", s->root,
877 (s->root_extended ? "(extended)" : ""));
878}
879
02a2cbc8
PX		 880static void vtd_iec_notify_all(IntelIOMMUState *s, bool global,
881 uint32_t index, uint32_t mask)
882{
883 x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask);
884}
885
a5861439
PX		 886static void vtd_interrupt_remap_table_setup(IntelIOMMUState *s)
887{
888 uint64_t value = 0;
889 value = vtd_get_quad_raw(s, DMAR_IRTA_REG);
890 s->intr_size = 1UL << ((value & VTD_IRTA_SIZE_MASK) + 1);
891 s->intr_root = value & VTD_IRTA_ADDR_MASK;
28589311 892 s->intr_eime = value & VTD_IRTA_EIME;
a5861439 893
02a2cbc8
PX		 894 /* Notify global invalidation */
895 vtd_iec_notify_all(s, true, 0, 0);
a5861439
PX		 896
897 VTD_DPRINTF(CSR, "int remap table addr 0x%"PRIx64 " size %"PRIu32,
898 s->intr_root, s->intr_size);
899}
900
d92fa2dc
LT		 901static void vtd_context_global_invalidate(IntelIOMMUState *s)
902{
bc535e59 903 trace_vtd_inv_desc_cc_global();
d92fa2dc
LT		 904 s->context_cache_gen++;
905 if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) {
906 vtd_reset_context_cache(s);
907 }
908}
909
7df953bd
KO		 910
911/* Find the VTD address space currently associated with a given bus number,
912 */
913static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num)
914{
915 VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num];
916 if (!vtd_bus) {
917 /* Iterate over the registered buses to find the one
918 * which currently hold this bus number, and update the bus_num lookup table:
919 */
920 GHashTableIter iter;
921
922 g_hash_table_iter_init(&iter, s->vtd_as_by_busptr);
923 while (g_hash_table_iter_next (&iter, NULL, (void**)&vtd_bus)) {
924 if (pci_bus_num(vtd_bus->bus) == bus_num) {
925 s->vtd_as_by_bus_num[bus_num] = vtd_bus;
926 return vtd_bus;
927 }
928 }
929 }
930 return vtd_bus;
931}
932
d92fa2dc
LT		 933/* Do a context-cache device-selective invalidation.
934 * @func_mask: FM field after shifting
935 */
936static void vtd_context_device_invalidate(IntelIOMMUState *s,
937 uint16_t source_id,
938 uint16_t func_mask)
939{
940 uint16_t mask;
7df953bd 941 VTDBus *vtd_bus;
d92fa2dc 942 VTDAddressSpace *vtd_as;
bc535e59 943 uint8_t bus_n, devfn;
d92fa2dc
LT		 944 uint16_t devfn_it;
945
bc535e59
PX		 946 trace_vtd_inv_desc_cc_devices(source_id, func_mask);
947
d92fa2dc
LT		 948 switch (func_mask & 3) {
949 case 0:
950 mask = 0; /* No bits in the SID field masked */
951 break;
952 case 1:
953 mask = 4; /* Mask bit 2 in the SID field */
954 break;
955 case 2:
956 mask = 6; /* Mask bit 2:1 in the SID field */
957 break;
958 case 3:
959 mask = 7; /* Mask bit 2:0 in the SID field */
960 break;
961 }
6cb99acc 962 mask = ~mask;
bc535e59
PX		 963
964 bus_n = VTD_SID_TO_BUS(source_id);
965 vtd_bus = vtd_find_as_from_bus_num(s, bus_n);
7df953bd 966 if (vtd_bus) {
d92fa2dc 967 devfn = VTD_SID_TO_DEVFN(source_id);
04af0e18 968 for (devfn_it = 0; devfn_it < X86_IOMMU_PCI_DEVFN_MAX; ++devfn_it) {
7df953bd 969 vtd_as = vtd_bus->dev_as[devfn_it];
d92fa2dc 970 if (vtd_as && ((devfn_it & mask) == (devfn & mask))) {
bc535e59
PX		 971 trace_vtd_inv_desc_cc_device(bus_n, VTD_PCI_SLOT(devfn_it),
972 VTD_PCI_FUNC(devfn_it));
d92fa2dc
LT		 973 vtd_as->context_cache_entry.context_cache_gen = 0;
974 }
975 }
976 }
977}
978
1da12ec4
LT		 979/* Context-cache invalidation
980 * Returns the Context Actual Invalidation Granularity.
981 * @val: the content of the CCMD_REG
982 */
983static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val)
984{
985 uint64_t caig;
986 uint64_t type = val & VTD_CCMD_CIRG_MASK;
987
988 switch (type) {
d92fa2dc
LT		 989 case VTD_CCMD_DOMAIN_INVL:
990 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
991 (uint16_t)VTD_CCMD_DID(val));
992 /* Fall through */
1da12ec4 993 case VTD_CCMD_GLOBAL_INVL:
d92fa2dc 994 VTD_DPRINTF(INV, "global invalidation");
1da12ec4 995 caig = VTD_CCMD_GLOBAL_INVL_A;
d92fa2dc 996 vtd_context_global_invalidate(s);
1da12ec4
LT		 997 break;
998
999 case VTD_CCMD_DEVICE_INVL:
1da12ec4 1000 caig = VTD_CCMD_DEVICE_INVL_A;
d92fa2dc 1001 vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val));
1da12ec4
LT		 1002 break;
1003
1004 default:
d92fa2dc 1005 VTD_DPRINTF(GENERAL, "error: invalid granularity");
1da12ec4
LT		 1006 caig = 0;
1007 }
1008 return caig;
1009}
1010
b5a280c0
LT		 1011static void vtd_iotlb_global_invalidate(IntelIOMMUState *s)
1012{
6c441e1d 1013 trace_vtd_iotlb_reset("global invalidation recved");
b5a280c0
LT		 1014 vtd_reset_iotlb(s);
1015}
1016
1017static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id)
1018{
1019 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain,
1020 &domain_id);
1021}
1022
1023static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id,
1024 hwaddr addr, uint8_t am)
1025{
1026 VTDIOTLBPageInvInfo info;
1027
1028 assert(am <= VTD_MAMV);
1029 info.domain_id = domain_id;
d66b969b 1030 info.addr = addr;
b5a280c0
LT		 1031 info.mask = ~((1 << am) - 1);
1032 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info);
1033}
1034
1da12ec4
LT		 1035/* Flush IOTLB
1036 * Returns the IOTLB Actual Invalidation Granularity.
1037 * @val: the content of the IOTLB_REG
1038 */
1039static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val)
1040{
1041 uint64_t iaig;
1042 uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK;
b5a280c0
LT		 1043 uint16_t domain_id;
1044 hwaddr addr;
1045 uint8_t am;
1da12ec4
LT		 1046
1047 switch (type) {
1048 case VTD_TLB_GLOBAL_FLUSH:
b5a280c0 1049 VTD_DPRINTF(INV, "global invalidation");
1da12ec4 1050 iaig = VTD_TLB_GLOBAL_FLUSH_A;
b5a280c0 1051 vtd_iotlb_global_invalidate(s);
1da12ec4
LT		 1052 break;
1053
1054 case VTD_TLB_DSI_FLUSH:
b5a280c0
LT		 1055 domain_id = VTD_TLB_DID(val);
1056 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
1057 domain_id);
1da12ec4 1058 iaig = VTD_TLB_DSI_FLUSH_A;
b5a280c0 1059 vtd_iotlb_domain_invalidate(s, domain_id);
1da12ec4
LT		 1060 break;
1061
1062 case VTD_TLB_PSI_FLUSH:
b5a280c0
LT		 1063 domain_id = VTD_TLB_DID(val);
1064 addr = vtd_get_quad_raw(s, DMAR_IVA_REG);
1065 am = VTD_IVA_AM(addr);
1066 addr = VTD_IVA_ADDR(addr);
1067 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16
1068 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am);
1069 if (am > VTD_MAMV) {
1070 VTD_DPRINTF(GENERAL, "error: supported max address mask value is "
1071 "%"PRIu8, (uint8_t)VTD_MAMV);
1072 iaig = 0;
1073 break;
1074 }
1da12ec4 1075 iaig = VTD_TLB_PSI_FLUSH_A;
b5a280c0 1076 vtd_iotlb_page_invalidate(s, domain_id, addr, am);
1da12ec4
LT		 1077 break;
1078
1079 default:
b5a280c0 1080 VTD_DPRINTF(GENERAL, "error: invalid granularity");
1da12ec4
LT		 1081 iaig = 0;
1082 }
1083 return iaig;
1084}
1085
ed7b8fbc
LT		 1086static inline bool vtd_queued_inv_enable_check(IntelIOMMUState *s)
1087{
1088 return s->iq_tail == 0;
1089}
1090
1091static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s)
1092{
1093 return s->qi_enabled && (s->iq_tail == s->iq_head) &&
1094 (s->iq_last_desc_type == VTD_INV_DESC_WAIT);
1095}
1096
1097static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en)
1098{
1099 uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG);
1100
1101 VTD_DPRINTF(INV, "Queued Invalidation Enable %s", (en ? "on" : "off"));
1102 if (en) {
1103 if (vtd_queued_inv_enable_check(s)) {
1104 s->iq = iqa_val & VTD_IQA_IQA_MASK;
1105 /* 2^(x+8) entries */
1106 s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8);
1107 s->qi_enabled = true;
1108 VTD_DPRINTF(INV, "DMAR_IQA_REG 0x%"PRIx64, iqa_val);
1109 VTD_DPRINTF(INV, "Invalidation Queue addr 0x%"PRIx64 " size %d",
1110 s->iq, s->iq_size);
1111 /* Ok - report back to driver */
1112 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES);
1113 } else {
1114 VTD_DPRINTF(GENERAL, "error: can't enable Queued Invalidation: "
1115 "tail %"PRIu16, s->iq_tail);
1116 }
1117 } else {
1118 if (vtd_queued_inv_disable_check(s)) {
1119 /* disable Queued Invalidation */
1120 vtd_set_quad_raw(s, DMAR_IQH_REG, 0);
1121 s->iq_head = 0;
1122 s->qi_enabled = false;
1123 /* Ok - report back to driver */
1124 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0);
1125 } else {
1126 VTD_DPRINTF(GENERAL, "error: can't disable Queued Invalidation: "
1127 "head %"PRIu16 ", tail %"PRIu16
1128 ", last_descriptor %"PRIu8,
1129 s->iq_head, s->iq_tail, s->iq_last_desc_type);
1130 }
1131 }
1132}
1133
1da12ec4
LT		 1134/* Set Root Table Pointer */
1135static void vtd_handle_gcmd_srtp(IntelIOMMUState *s)
1136{
1137 VTD_DPRINTF(CSR, "set Root Table Pointer");
1138
1139 vtd_root_table_setup(s);
1140 /* Ok - report back to driver */
1141 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS);
1142}
1143
a5861439
PX		 1144/* Set Interrupt Remap Table Pointer */
1145static void vtd_handle_gcmd_sirtp(IntelIOMMUState *s)
1146{
1147 VTD_DPRINTF(CSR, "set Interrupt Remap Table Pointer");
1148
1149 vtd_interrupt_remap_table_setup(s);
1150 /* Ok - report back to driver */
1151 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRTPS);
1152}
1153
1da12ec4
LT		 1154/* Handle Translation Enable/Disable */
1155static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en)
1156{
1157 VTD_DPRINTF(CSR, "Translation Enable %s", (en ? "on" : "off"));
1158
1159 if (en) {
1160 s->dmar_enabled = true;
1161 /* Ok - report back to driver */
1162 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES);
1163 } else {
1164 s->dmar_enabled = false;
1165
1166 /* Clear the index of Fault Recording Register */
1167 s->next_frcd_reg = 0;
1168 /* Ok - report back to driver */
1169 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0);
1170 }
1171}
1172
80de52ba
PX		 1173/* Handle Interrupt Remap Enable/Disable */
1174static void vtd_handle_gcmd_ire(IntelIOMMUState *s, bool en)
1175{
1176 VTD_DPRINTF(CSR, "Interrupt Remap Enable %s", (en ? "on" : "off"));
1177
1178 if (en) {
1179 s->intr_enabled = true;
1180 /* Ok - report back to driver */
1181 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRES);
1182 } else {
1183 s->intr_enabled = false;
1184 /* Ok - report back to driver */
1185 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_IRES, 0);
1186 }
1187}
1188
1da12ec4
LT		 1189/* Handle write to Global Command Register */
1190static void vtd_handle_gcmd_write(IntelIOMMUState *s)
1191{
1192 uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG);
1193 uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG);
1194 uint32_t changed = status ^ val;
1195
1196 VTD_DPRINTF(CSR, "value 0x%"PRIx32 " status 0x%"PRIx32, val, status);
1197 if (changed & VTD_GCMD_TE) {
1198 /* Translation enable/disable */
1199 vtd_handle_gcmd_te(s, val & VTD_GCMD_TE);
1200 }
1201 if (val & VTD_GCMD_SRTP) {
1202 /* Set/update the root-table pointer */
1203 vtd_handle_gcmd_srtp(s);
1204 }
ed7b8fbc
LT		 1205 if (changed & VTD_GCMD_QIE) {
1206 /* Queued Invalidation Enable */
1207 vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE);
1208 }
a5861439
PX		 1209 if (val & VTD_GCMD_SIRTP) {
1210 /* Set/update the interrupt remapping root-table pointer */
1211 vtd_handle_gcmd_sirtp(s);
1212 }
80de52ba
PX		 1213 if (changed & VTD_GCMD_IRE) {
1214 /* Interrupt remap enable/disable */
1215 vtd_handle_gcmd_ire(s, val & VTD_GCMD_IRE);
1216 }
1da12ec4
LT		 1217}
1218
1219/* Handle write to Context Command Register */
1220static void vtd_handle_ccmd_write(IntelIOMMUState *s)
1221{
1222 uint64_t ret;
1223 uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG);
1224
1225 /* Context-cache invalidation request */
1226 if (val & VTD_CCMD_ICC) {
ed7b8fbc
LT		 1227 if (s->qi_enabled) {
1228 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, "
1229 "should not use register-based invalidation");
1230 return;
1231 }
1da12ec4
LT		 1232 ret = vtd_context_cache_invalidate(s, val);
1233 /* Invalidation completed. Change something to show */
1234 vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL);
1235 ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK,
1236 ret);
1237 VTD_DPRINTF(INV, "CCMD_REG write-back val: 0x%"PRIx64, ret);
1238 }
1239}
1240
1241/* Handle write to IOTLB Invalidation Register */
1242static void vtd_handle_iotlb_write(IntelIOMMUState *s)
1243{
1244 uint64_t ret;
1245 uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG);
1246
1247 /* IOTLB invalidation request */
1248 if (val & VTD_TLB_IVT) {
ed7b8fbc
LT		 1249 if (s->qi_enabled) {
1250 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, "
1251 "should not use register-based invalidation");
1252 return;
1253 }
1da12ec4
LT		 1254 ret = vtd_iotlb_flush(s, val);
1255 /* Invalidation completed. Change something to show */
1256 vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL);
1257 ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG,
1258 VTD_TLB_FLUSH_GRANU_MASK_A, ret);
1259 VTD_DPRINTF(INV, "IOTLB_REG write-back val: 0x%"PRIx64, ret);
1260 }
1261}
1262
ed7b8fbc
LT		 1263/* Fetch an Invalidation Descriptor from the Invalidation Queue */
1264static bool vtd_get_inv_desc(dma_addr_t base_addr, uint32_t offset,
1265 VTDInvDesc *inv_desc)
1266{
1267 dma_addr_t addr = base_addr + offset * sizeof(*inv_desc);
1268 if (dma_memory_read(&address_space_memory, addr, inv_desc,
1269 sizeof(*inv_desc))) {
1270 VTD_DPRINTF(GENERAL, "error: fail to fetch Invalidation Descriptor "
1271 "base_addr 0x%"PRIx64 " offset %"PRIu32, base_addr, offset);
1272 inv_desc->lo = 0;
1273 inv_desc->hi = 0;
1274
1275 return false;
1276 }
1277 inv_desc->lo = le64_to_cpu(inv_desc->lo);
1278 inv_desc->hi = le64_to_cpu(inv_desc->hi);
1279 return true;
1280}
1281
1282static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
1283{
1284 if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) ||
1285 (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) {
bc535e59 1286 trace_vtd_inv_desc_wait_invalid(inv_desc->hi, inv_desc->lo);
ed7b8fbc
LT		 1287 return false;
1288 }
1289 if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) {
1290 /* Status Write */
1291 uint32_t status_data = (uint32_t)(inv_desc->lo >>
1292 VTD_INV_DESC_WAIT_DATA_SHIFT);
1293
1294 assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF));
1295
1296 /* FIXME: need to be masked with HAW? */
1297 dma_addr_t status_addr = inv_desc->hi;
bc535e59 1298 trace_vtd_inv_desc_wait_sw(status_addr, status_data);
ed7b8fbc
LT		 1299 status_data = cpu_to_le32(status_data);
1300 if (dma_memory_write(&address_space_memory, status_addr, &status_data,
1301 sizeof(status_data))) {
bc535e59 1302 trace_vtd_inv_desc_wait_write_fail(inv_desc->hi, inv_desc->lo);
ed7b8fbc
LT		 1303 return false;
1304 }
1305 } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) {
1306 /* Interrupt flag */
ed7b8fbc
LT		 1307 vtd_generate_completion_event(s);
1308 } else {
bc535e59 1309 trace_vtd_inv_desc_wait_invalid(inv_desc->hi, inv_desc->lo);
ed7b8fbc
LT		 1310 return false;
1311 }
1312 return true;
1313}
1314
d92fa2dc
LT		 1315static bool vtd_process_context_cache_desc(IntelIOMMUState *s,
1316 VTDInvDesc *inv_desc)
1317{
bc535e59
PX		 1318 uint16_t sid, fmask;
1319
d92fa2dc 1320 if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) {
bc535e59 1321 trace_vtd_inv_desc_cc_invalid(inv_desc->hi, inv_desc->lo);
d92fa2dc
LT		 1322 return false;
1323 }
1324 switch (inv_desc->lo & VTD_INV_DESC_CC_G) {
1325 case VTD_INV_DESC_CC_DOMAIN:
bc535e59
PX		 1326 trace_vtd_inv_desc_cc_domain(
1327 (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo));
d92fa2dc
LT		 1328 /* Fall through */
1329 case VTD_INV_DESC_CC_GLOBAL:
d92fa2dc
LT		 1330 vtd_context_global_invalidate(s);
1331 break;
1332
1333 case VTD_INV_DESC_CC_DEVICE:
bc535e59
PX		 1334 sid = VTD_INV_DESC_CC_SID(inv_desc->lo);
1335 fmask = VTD_INV_DESC_CC_FM(inv_desc->lo);
1336 vtd_context_device_invalidate(s, sid, fmask);
d92fa2dc
LT		 1337 break;
1338
1339 default:
bc535e59 1340 trace_vtd_inv_desc_cc_invalid(inv_desc->hi, inv_desc->lo);
d92fa2dc
LT		 1341 return false;
1342 }
1343 return true;
1344}
1345
b5a280c0
LT		 1346static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
1347{
1348 uint16_t domain_id;
1349 uint8_t am;
1350 hwaddr addr;
1351
1352 if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) ||
1353 (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) {
bc535e59 1354 trace_vtd_inv_desc_iotlb_invalid(inv_desc->hi, inv_desc->lo);
b5a280c0
LT		 1355 return false;
1356 }
1357
1358 switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) {
1359 case VTD_INV_DESC_IOTLB_GLOBAL:
bc535e59 1360 trace_vtd_inv_desc_iotlb_global();
b5a280c0
LT		 1361 vtd_iotlb_global_invalidate(s);
1362 break;
1363
1364 case VTD_INV_DESC_IOTLB_DOMAIN:
1365 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
bc535e59 1366 trace_vtd_inv_desc_iotlb_domain(domain_id);
b5a280c0
LT		 1367 vtd_iotlb_domain_invalidate(s, domain_id);
1368 break;
1369
1370 case VTD_INV_DESC_IOTLB_PAGE:
1371 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
1372 addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi);
1373 am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi);
bc535e59 1374 trace_vtd_inv_desc_iotlb_pages(domain_id, addr, am);
b5a280c0 1375 if (am > VTD_MAMV) {
bc535e59 1376 trace_vtd_inv_desc_iotlb_invalid(inv_desc->hi, inv_desc->lo);
b5a280c0
LT		 1377 return false;
1378 }
1379 vtd_iotlb_page_invalidate(s, domain_id, addr, am);
1380 break;
1381
1382 default:
bc535e59 1383 trace_vtd_inv_desc_iotlb_invalid(inv_desc->hi, inv_desc->lo);
b5a280c0
LT		 1384 return false;
1385 }
1386 return true;
1387}
1388
02a2cbc8
PX		 1389static bool vtd_process_inv_iec_desc(IntelIOMMUState *s,
1390 VTDInvDesc *inv_desc)
1391{
1392 VTD_DPRINTF(INV, "inv ir glob %d index %d mask %d",
1393 inv_desc->iec.granularity,
1394 inv_desc->iec.index,
1395 inv_desc->iec.index_mask);
1396
1397 vtd_iec_notify_all(s, !inv_desc->iec.granularity,
1398 inv_desc->iec.index,
1399 inv_desc->iec.index_mask);
554f5e16
JW		 1400 return true;
1401}
1402
1403static bool vtd_process_device_iotlb_desc(IntelIOMMUState *s,
1404 VTDInvDesc *inv_desc)
1405{
1406 VTDAddressSpace *vtd_dev_as;
1407 IOMMUTLBEntry entry;
1408 struct VTDBus *vtd_bus;
1409 hwaddr addr;
1410 uint64_t sz;
1411 uint16_t sid;
1412 uint8_t devfn;
1413 bool size;
1414 uint8_t bus_num;
1415
1416 addr = VTD_INV_DESC_DEVICE_IOTLB_ADDR(inv_desc->hi);
1417 sid = VTD_INV_DESC_DEVICE_IOTLB_SID(inv_desc->lo);
1418 devfn = sid & 0xff;
1419 bus_num = sid >> 8;
1420 size = VTD_INV_DESC_DEVICE_IOTLB_SIZE(inv_desc->hi);
1421
1422 if ((inv_desc->lo & VTD_INV_DESC_DEVICE_IOTLB_RSVD_LO) ||
1423 (inv_desc->hi & VTD_INV_DESC_DEVICE_IOTLB_RSVD_HI)) {
1424 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Device "
1425 "IOTLB Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1426 inv_desc->hi, inv_desc->lo);
1427 return false;
1428 }
1429
1430 vtd_bus = vtd_find_as_from_bus_num(s, bus_num);
1431 if (!vtd_bus) {
1432 goto done;
1433 }
1434
1435 vtd_dev_as = vtd_bus->dev_as[devfn];
1436 if (!vtd_dev_as) {
1437 goto done;
1438 }
1439
04eb6247
JW		 1440 /* According to ATS spec table 2.4:
1441 * S = 0, bits 15:12 = xxxx range size: 4K
1442 * S = 1, bits 15:12 = xxx0 range size: 8K
1443 * S = 1, bits 15:12 = xx01 range size: 16K
1444 * S = 1, bits 15:12 = x011 range size: 32K
1445 * S = 1, bits 15:12 = 0111 range size: 64K
1446 * ...
1447 */
554f5e16 1448 if (size) {
04eb6247 1449 sz = (VTD_PAGE_SIZE * 2) << cto64(addr >> VTD_PAGE_SHIFT);
554f5e16
JW		 1450 addr &= ~(sz - 1);
1451 } else {
1452 sz = VTD_PAGE_SIZE;
1453 }
02a2cbc8 1454
554f5e16
JW		 1455 entry.target_as = &vtd_dev_as->as;
1456 entry.addr_mask = sz - 1;
1457 entry.iova = addr;
1458 entry.perm = IOMMU_NONE;
1459 entry.translated_addr = 0;
10315b9b 1460 memory_region_notify_iommu(&vtd_dev_as->iommu, entry);
554f5e16
JW		 1461
1462done:
02a2cbc8
PX		 1463 return true;
1464}
1465
ed7b8fbc
LT		 1466static bool vtd_process_inv_desc(IntelIOMMUState *s)
1467{
1468 VTDInvDesc inv_desc;
1469 uint8_t desc_type;
1470
1471 VTD_DPRINTF(INV, "iq head %"PRIu16, s->iq_head);
1472 if (!vtd_get_inv_desc(s->iq, s->iq_head, &inv_desc)) {
1473 s->iq_last_desc_type = VTD_INV_DESC_NONE;
1474 return false;
1475 }
1476 desc_type = inv_desc.lo & VTD_INV_DESC_TYPE;
1477 /* FIXME: should update at first or at last? */
1478 s->iq_last_desc_type = desc_type;
1479
1480 switch (desc_type) {
1481 case VTD_INV_DESC_CC:
bc535e59 1482 trace_vtd_inv_desc("context-cache", inv_desc.hi, inv_desc.lo);
d92fa2dc
LT		 1483 if (!vtd_process_context_cache_desc(s, &inv_desc)) {
1484 return false;
1485 }
ed7b8fbc
LT		 1486 break;
1487
1488 case VTD_INV_DESC_IOTLB:
bc535e59 1489 trace_vtd_inv_desc("iotlb", inv_desc.hi, inv_desc.lo);
b5a280c0
LT		 1490 if (!vtd_process_iotlb_desc(s, &inv_desc)) {
1491 return false;
1492 }
ed7b8fbc
LT		 1493 break;
1494
1495 case VTD_INV_DESC_WAIT:
bc535e59 1496 trace_vtd_inv_desc("wait", inv_desc.hi, inv_desc.lo);
ed7b8fbc
LT		 1497 if (!vtd_process_wait_desc(s, &inv_desc)) {
1498 return false;
1499 }
1500 break;
1501
b7910472 1502 case VTD_INV_DESC_IEC:
bc535e59 1503 trace_vtd_inv_desc("iec", inv_desc.hi, inv_desc.lo);
02a2cbc8
PX		 1504 if (!vtd_process_inv_iec_desc(s, &inv_desc)) {
1505 return false;
1506 }
b7910472
PX		 1507 break;
1508
554f5e16
JW		 1509 case VTD_INV_DESC_DEVICE:
1510 VTD_DPRINTF(INV, "Device IOTLB Invalidation Descriptor hi 0x%"PRIx64
1511 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
1512 if (!vtd_process_device_iotlb_desc(s, &inv_desc)) {
1513 return false;
1514 }
1515 break;
1516
ed7b8fbc 1517 default:
bc535e59 1518 trace_vtd_inv_desc_invalid(inv_desc.hi, inv_desc.lo);
ed7b8fbc
LT		 1519 return false;
1520 }
1521 s->iq_head++;
1522 if (s->iq_head == s->iq_size) {
1523 s->iq_head = 0;
1524 }
1525 return true;
1526}
1527
1528/* Try to fetch and process more Invalidation Descriptors */
1529static void vtd_fetch_inv_desc(IntelIOMMUState *s)
1530{
1531 VTD_DPRINTF(INV, "fetch Invalidation Descriptors");
1532 if (s->iq_tail >= s->iq_size) {
1533 /* Detects an invalid Tail pointer */
1534 VTD_DPRINTF(GENERAL, "error: iq_tail is %"PRIu16
1535 " while iq_size is %"PRIu16, s->iq_tail, s->iq_size);
1536 vtd_handle_inv_queue_error(s);
1537 return;
1538 }
1539 while (s->iq_head != s->iq_tail) {
1540 if (!vtd_process_inv_desc(s)) {
1541 /* Invalidation Queue Errors */
1542 vtd_handle_inv_queue_error(s);
1543 break;
1544 }
1545 /* Must update the IQH_REG in time */
1546 vtd_set_quad_raw(s, DMAR_IQH_REG,
1547 (((uint64_t)(s->iq_head)) << VTD_IQH_QH_SHIFT) &
1548 VTD_IQH_QH_MASK);
1549 }
1550}
1551
1552/* Handle write to Invalidation Queue Tail Register */
1553static void vtd_handle_iqt_write(IntelIOMMUState *s)
1554{
1555 uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG);
1556
1557 s->iq_tail = VTD_IQT_QT(val);
1558 VTD_DPRINTF(INV, "set iq tail %"PRIu16, s->iq_tail);
1559 if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) {
1560 /* Process Invalidation Queue here */
1561 vtd_fetch_inv_desc(s);
1562 }
1563}
1564
1da12ec4
LT		 1565static void vtd_handle_fsts_write(IntelIOMMUState *s)
1566{
1567 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
1568 uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
1569 uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE;
1570
1571 if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) {
1572 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
1573 VTD_DPRINTF(FLOG, "all pending interrupt conditions serviced, clear "
1574 "IP field of FECTL_REG");
1575 }
ed7b8fbc
LT		 1576 /* FIXME: when IQE is Clear, should we try to fetch some Invalidation
1577 * Descriptors if there are any when Queued Invalidation is enabled?
1578 */
1da12ec4
LT		 1579}
1580
1581static void vtd_handle_fectl_write(IntelIOMMUState *s)
1582{
1583 uint32_t fectl_reg;
1584 /* FIXME: when software clears the IM field, check the IP field. But do we
1585 * need to compare the old value and the new value to conclude that
1586 * software clears the IM field? Or just check if the IM field is zero?
1587 */
1588 fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
1589 if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) {
1590 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
1591 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
1592 VTD_DPRINTF(FLOG, "IM field is cleared, generate "
1593 "fault event interrupt");
1594 }
1595}
1596
ed7b8fbc
LT		 1597static void vtd_handle_ics_write(IntelIOMMUState *s)
1598{
1599 uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG);
1600 uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
1601
1602 if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) {
1603 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
1604 VTD_DPRINTF(INV, "pending completion interrupt condition serviced, "
1605 "clear IP field of IECTL_REG");
1606 }
1607}
1608
1609static void vtd_handle_iectl_write(IntelIOMMUState *s)
1610{
1611 uint32_t iectl_reg;
1612 /* FIXME: when software clears the IM field, check the IP field. But do we
1613 * need to compare the old value and the new value to conclude that
1614 * software clears the IM field? Or just check if the IM field is zero?
1615 */
1616 iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
1617 if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) {
1618 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
1619 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
1620 VTD_DPRINTF(INV, "IM field is cleared, generate "
1621 "invalidation event interrupt");
1622 }
1623}
1624
1da12ec4
LT		 1625static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size)
1626{
1627 IntelIOMMUState *s = opaque;
1628 uint64_t val;
1629
1630 if (addr + size > DMAR_REG_SIZE) {
1631 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64
1632 ", got 0x%"PRIx64 " %d",
1633 (uint64_t)DMAR_REG_SIZE, addr, size);
1634 return (uint64_t)-1;
1635 }
1636
1637 switch (addr) {
1638 /* Root Table Address Register, 64-bit */
1639 case DMAR_RTADDR_REG:
1640 if (size == 4) {
1641 val = s->root & ((1ULL << 32) - 1);
1642 } else {
1643 val = s->root;
1644 }
1645 break;
1646
1647 case DMAR_RTADDR_REG_HI:
1648 assert(size == 4);
1649 val = s->root >> 32;
1650 break;
1651
ed7b8fbc
LT		 1652 /* Invalidation Queue Address Register, 64-bit */
1653 case DMAR_IQA_REG:
1654 val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS);
1655 if (size == 4) {
1656 val = val & ((1ULL << 32) - 1);
1657 }
1658 break;
1659
1660 case DMAR_IQA_REG_HI:
1661 assert(size == 4);
1662 val = s->iq >> 32;
1663 break;
1664
1da12ec4
LT		 1665 default:
1666 if (size == 4) {
1667 val = vtd_get_long(s, addr);
1668 } else {
1669 val = vtd_get_quad(s, addr);
1670 }
1671 }
1672 VTD_DPRINTF(CSR, "addr 0x%"PRIx64 " size %d val 0x%"PRIx64,
1673 addr, size, val);
1674 return val;
1675}
1676
1677static void vtd_mem_write(void *opaque, hwaddr addr,
1678 uint64_t val, unsigned size)
1679{
1680 IntelIOMMUState *s = opaque;
1681
1682 if (addr + size > DMAR_REG_SIZE) {
1683 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64
1684 ", got 0x%"PRIx64 " %d",
1685 (uint64_t)DMAR_REG_SIZE, addr, size);
1686 return;
1687 }
1688
1689 switch (addr) {
1690 /* Global Command Register, 32-bit */
1691 case DMAR_GCMD_REG:
1692 VTD_DPRINTF(CSR, "DMAR_GCMD_REG write addr 0x%"PRIx64
1693 ", size %d, val 0x%"PRIx64, addr, size, val);
1694 vtd_set_long(s, addr, val);
1695 vtd_handle_gcmd_write(s);
1696 break;
1697
1698 /* Context Command Register, 64-bit */
1699 case DMAR_CCMD_REG:
1700 VTD_DPRINTF(CSR, "DMAR_CCMD_REG write addr 0x%"PRIx64
1701 ", size %d, val 0x%"PRIx64, addr, size, val);
1702 if (size == 4) {
1703 vtd_set_long(s, addr, val);
1704 } else {
1705 vtd_set_quad(s, addr, val);
1706 vtd_handle_ccmd_write(s);
1707 }
1708 break;
1709
1710 case DMAR_CCMD_REG_HI:
1711 VTD_DPRINTF(CSR, "DMAR_CCMD_REG_HI write addr 0x%"PRIx64
1712 ", size %d, val 0x%"PRIx64, addr, size, val);
1713 assert(size == 4);
1714 vtd_set_long(s, addr, val);
1715 vtd_handle_ccmd_write(s);
1716 break;
1717
1718 /* IOTLB Invalidation Register, 64-bit */
1719 case DMAR_IOTLB_REG:
1720 VTD_DPRINTF(INV, "DMAR_IOTLB_REG write addr 0x%"PRIx64
1721 ", size %d, val 0x%"PRIx64, addr, size, val);
1722 if (size == 4) {
1723 vtd_set_long(s, addr, val);
1724 } else {
1725 vtd_set_quad(s, addr, val);
1726 vtd_handle_iotlb_write(s);
1727 }
1728 break;
1729
1730 case DMAR_IOTLB_REG_HI:
1731 VTD_DPRINTF(INV, "DMAR_IOTLB_REG_HI write addr 0x%"PRIx64
1732 ", size %d, val 0x%"PRIx64, addr, size, val);
1733 assert(size == 4);
1734 vtd_set_long(s, addr, val);
1735 vtd_handle_iotlb_write(s);
1736 break;
1737
b5a280c0
LT		 1738 /* Invalidate Address Register, 64-bit */
1739 case DMAR_IVA_REG:
1740 VTD_DPRINTF(INV, "DMAR_IVA_REG write addr 0x%"PRIx64
1741 ", size %d, val 0x%"PRIx64, addr, size, val);
1742 if (size == 4) {
1743 vtd_set_long(s, addr, val);
1744 } else {
1745 vtd_set_quad(s, addr, val);
1746 }
1747 break;
1748
1749 case DMAR_IVA_REG_HI:
1750 VTD_DPRINTF(INV, "DMAR_IVA_REG_HI write addr 0x%"PRIx64
1751 ", size %d, val 0x%"PRIx64, addr, size, val);
1752 assert(size == 4);
1753 vtd_set_long(s, addr, val);
1754 break;
1755
1da12ec4
LT		 1756 /* Fault Status Register, 32-bit */
1757 case DMAR_FSTS_REG:
1758 VTD_DPRINTF(FLOG, "DMAR_FSTS_REG write addr 0x%"PRIx64
1759 ", size %d, val 0x%"PRIx64, addr, size, val);
1760 assert(size == 4);
1761 vtd_set_long(s, addr, val);
1762 vtd_handle_fsts_write(s);
1763 break;
1764
1765 /* Fault Event Control Register, 32-bit */
1766 case DMAR_FECTL_REG:
1767 VTD_DPRINTF(FLOG, "DMAR_FECTL_REG write addr 0x%"PRIx64
1768 ", size %d, val 0x%"PRIx64, addr, size, val);
1769 assert(size == 4);
1770 vtd_set_long(s, addr, val);
1771 vtd_handle_fectl_write(s);
1772 break;
1773
1774 /* Fault Event Data Register, 32-bit */
1775 case DMAR_FEDATA_REG:
1776 VTD_DPRINTF(FLOG, "DMAR_FEDATA_REG write addr 0x%"PRIx64
1777 ", size %d, val 0x%"PRIx64, addr, size, val);
1778 assert(size == 4);
1779 vtd_set_long(s, addr, val);
1780 break;
1781
1782 /* Fault Event Address Register, 32-bit */
1783 case DMAR_FEADDR_REG:
1784 VTD_DPRINTF(FLOG, "DMAR_FEADDR_REG write addr 0x%"PRIx64
1785 ", size %d, val 0x%"PRIx64, addr, size, val);
1786 assert(size == 4);
1787 vtd_set_long(s, addr, val);
1788 break;
1789
1790 /* Fault Event Upper Address Register, 32-bit */
1791 case DMAR_FEUADDR_REG:
1792 VTD_DPRINTF(FLOG, "DMAR_FEUADDR_REG write addr 0x%"PRIx64
1793 ", size %d, val 0x%"PRIx64, addr, size, val);
1794 assert(size == 4);
1795 vtd_set_long(s, addr, val);
1796 break;
1797
1798 /* Protected Memory Enable Register, 32-bit */
1799 case DMAR_PMEN_REG:
1800 VTD_DPRINTF(CSR, "DMAR_PMEN_REG write addr 0x%"PRIx64
1801 ", size %d, val 0x%"PRIx64, addr, size, val);
1802 assert(size == 4);
1803 vtd_set_long(s, addr, val);
1804 break;
1805
1806 /* Root Table Address Register, 64-bit */
1807 case DMAR_RTADDR_REG:
1808 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG write addr 0x%"PRIx64
1809 ", size %d, val 0x%"PRIx64, addr, size, val);
1810 if (size == 4) {
1811 vtd_set_long(s, addr, val);
1812 } else {
1813 vtd_set_quad(s, addr, val);
1814 }
1815 break;
1816
1817 case DMAR_RTADDR_REG_HI:
1818 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG_HI write addr 0x%"PRIx64
1819 ", size %d, val 0x%"PRIx64, addr, size, val);
1820 assert(size == 4);
1821 vtd_set_long(s, addr, val);
1822 break;
1823
ed7b8fbc
LT		 1824 /* Invalidation Queue Tail Register, 64-bit */
1825 case DMAR_IQT_REG:
1826 VTD_DPRINTF(INV, "DMAR_IQT_REG write addr 0x%"PRIx64
1827 ", size %d, val 0x%"PRIx64, addr, size, val);
1828 if (size == 4) {
1829 vtd_set_long(s, addr, val);
1830 } else {
1831 vtd_set_quad(s, addr, val);
1832 }
1833 vtd_handle_iqt_write(s);
1834 break;
1835
1836 case DMAR_IQT_REG_HI:
1837 VTD_DPRINTF(INV, "DMAR_IQT_REG_HI write addr 0x%"PRIx64
1838 ", size %d, val 0x%"PRIx64, addr, size, val);
1839 assert(size == 4);
1840 vtd_set_long(s, addr, val);
1841 /* 19:63 of IQT_REG is RsvdZ, do nothing here */
1842 break;
1843
1844 /* Invalidation Queue Address Register, 64-bit */
1845 case DMAR_IQA_REG:
1846 VTD_DPRINTF(INV, "DMAR_IQA_REG write addr 0x%"PRIx64
1847 ", size %d, val 0x%"PRIx64, addr, size, val);
1848 if (size == 4) {
1849 vtd_set_long(s, addr, val);
1850 } else {
1851 vtd_set_quad(s, addr, val);
1852 }
1853 break;
1854
1855 case DMAR_IQA_REG_HI:
1856 VTD_DPRINTF(INV, "DMAR_IQA_REG_HI write addr 0x%"PRIx64
1857 ", size %d, val 0x%"PRIx64, addr, size, val);
1858 assert(size == 4);
1859 vtd_set_long(s, addr, val);
1860 break;
1861
1862 /* Invalidation Completion Status Register, 32-bit */
1863 case DMAR_ICS_REG:
1864 VTD_DPRINTF(INV, "DMAR_ICS_REG write addr 0x%"PRIx64
1865 ", size %d, val 0x%"PRIx64, addr, size, val);
1866 assert(size == 4);
1867 vtd_set_long(s, addr, val);
1868 vtd_handle_ics_write(s);
1869 break;
1870
1871 /* Invalidation Event Control Register, 32-bit */
1872 case DMAR_IECTL_REG:
1873 VTD_DPRINTF(INV, "DMAR_IECTL_REG write addr 0x%"PRIx64
1874 ", size %d, val 0x%"PRIx64, addr, size, val);
1875 assert(size == 4);
1876 vtd_set_long(s, addr, val);
1877 vtd_handle_iectl_write(s);
1878 break;
1879
1880 /* Invalidation Event Data Register, 32-bit */
1881 case DMAR_IEDATA_REG:
1882 VTD_DPRINTF(INV, "DMAR_IEDATA_REG write addr 0x%"PRIx64
1883 ", size %d, val 0x%"PRIx64, addr, size, val);
1884 assert(size == 4);
1885 vtd_set_long(s, addr, val);
1886 break;
1887
1888 /* Invalidation Event Address Register, 32-bit */
1889 case DMAR_IEADDR_REG:
1890 VTD_DPRINTF(INV, "DMAR_IEADDR_REG write addr 0x%"PRIx64
1891 ", size %d, val 0x%"PRIx64, addr, size, val);
1892 assert(size == 4);
1893 vtd_set_long(s, addr, val);
1894 break;
1895
1896 /* Invalidation Event Upper Address Register, 32-bit */
1897 case DMAR_IEUADDR_REG:
1898 VTD_DPRINTF(INV, "DMAR_IEUADDR_REG write addr 0x%"PRIx64
1899 ", size %d, val 0x%"PRIx64, addr, size, val);
1900 assert(size == 4);
1901 vtd_set_long(s, addr, val);
1902 break;
1903
1da12ec4
LT		 1904 /* Fault Recording Registers, 128-bit */
1905 case DMAR_FRCD_REG_0_0:
1906 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_0 write addr 0x%"PRIx64
1907 ", size %d, val 0x%"PRIx64, addr, size, val);
1908 if (size == 4) {
1909 vtd_set_long(s, addr, val);
1910 } else {
1911 vtd_set_quad(s, addr, val);
1912 }
1913 break;
1914
1915 case DMAR_FRCD_REG_0_1:
1916 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_1 write addr 0x%"PRIx64
1917 ", size %d, val 0x%"PRIx64, addr, size, val);
1918 assert(size == 4);
1919 vtd_set_long(s, addr, val);
1920 break;
1921
1922 case DMAR_FRCD_REG_0_2:
1923 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_2 write addr 0x%"PRIx64
1924 ", size %d, val 0x%"PRIx64, addr, size, val);
1925 if (size == 4) {
1926 vtd_set_long(s, addr, val);
1927 } else {
1928 vtd_set_quad(s, addr, val);
1929 /* May clear bit 127 (Fault), update PPF */
1930 vtd_update_fsts_ppf(s);
1931 }
1932 break;
1933
1934 case DMAR_FRCD_REG_0_3:
1935 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_3 write addr 0x%"PRIx64
1936 ", size %d, val 0x%"PRIx64, addr, size, val);
1937 assert(size == 4);
1938 vtd_set_long(s, addr, val);
1939 /* May clear bit 127 (Fault), update PPF */
1940 vtd_update_fsts_ppf(s);
1941 break;
1942
a5861439
PX		 1943 case DMAR_IRTA_REG:
1944 VTD_DPRINTF(IR, "DMAR_IRTA_REG write addr 0x%"PRIx64
1945 ", size %d, val 0x%"PRIx64, addr, size, val);
1946 if (size == 4) {
1947 vtd_set_long(s, addr, val);
1948 } else {
1949 vtd_set_quad(s, addr, val);
1950 }
1951 break;
1952
1953 case DMAR_IRTA_REG_HI:
1954 VTD_DPRINTF(IR, "DMAR_IRTA_REG_HI write addr 0x%"PRIx64
1955 ", size %d, val 0x%"PRIx64, addr, size, val);
1956 assert(size == 4);
1957 vtd_set_long(s, addr, val);
1958 break;
1959
1da12ec4
LT		 1960 default:
1961 VTD_DPRINTF(GENERAL, "error: unhandled reg write addr 0x%"PRIx64
1962 ", size %d, val 0x%"PRIx64, addr, size, val);
1963 if (size == 4) {
1964 vtd_set_long(s, addr, val);
1965 } else {
1966 vtd_set_quad(s, addr, val);
1967 }
1968 }
1969}
1970
1971static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr,
1972 bool is_write)
1973{
1974 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
1975 IntelIOMMUState *s = vtd_as->iommu_state;
1da12ec4
LT		 1976 IOMMUTLBEntry ret = {
1977 .target_as = &address_space_memory,
1978 .iova = addr,
1979 .translated_addr = 0,
1980 .addr_mask = ~(hwaddr)0,
1981 .perm = IOMMU_NONE,
1982 };
1983
1984 if (!s->dmar_enabled) {
1985 /* DMAR disabled, passthrough, use 4k-page*/
1986 ret.iova = addr & VTD_PAGE_MASK_4K;
1987 ret.translated_addr = addr & VTD_PAGE_MASK_4K;
1988 ret.addr_mask = ~VTD_PAGE_MASK_4K;
1989 ret.perm = IOMMU_RW;
1990 return ret;
1991 }
1992
7df953bd 1993 vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, addr,
d92fa2dc 1994 is_write, &ret);
1da12ec4
LT		 1995 VTD_DPRINTF(MMU,
1996 "bus %"PRIu8 " slot %"PRIu8 " func %"PRIu8 " devfn %"PRIu8
6e905564 1997 " iova 0x%"PRIx64 " hpa 0x%"PRIx64, pci_bus_num(vtd_as->bus),
d92fa2dc
LT		 1998 VTD_PCI_SLOT(vtd_as->devfn), VTD_PCI_FUNC(vtd_as->devfn),
1999 vtd_as->devfn, addr, ret.translated_addr);
1da12ec4
LT		 2000 return ret;
2001}
2002
5bf3d319
PX		 2003static void vtd_iommu_notify_flag_changed(MemoryRegion *iommu,
2004 IOMMUNotifierFlag old,
2005 IOMMUNotifierFlag new)
3cb3b154
AW		 2006{
2007 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
2008
a3276f78
PX		 2009 if (new & IOMMU_NOTIFIER_MAP) {
2010 error_report("Device at bus %s addr %02x.%d requires iommu "
2011 "notifier which is currently not supported by "
2012 "intel-iommu emulation",
2013 vtd_as->bus->qbus.name, PCI_SLOT(vtd_as->devfn),
2014 PCI_FUNC(vtd_as->devfn));
2015 exit(1);
2016 }
3cb3b154
AW		 2017}
2018
1da12ec4
LT		 2019static const VMStateDescription vtd_vmstate = {
2020 .name = "iommu-intel",
8cdcf3c1
PX		 2021 .version_id = 1,
2022 .minimum_version_id = 1,
2023 .priority = MIG_PRI_IOMMU,
2024 .fields = (VMStateField[]) {
2025 VMSTATE_UINT64(root, IntelIOMMUState),
2026 VMSTATE_UINT64(intr_root, IntelIOMMUState),
2027 VMSTATE_UINT64(iq, IntelIOMMUState),
2028 VMSTATE_UINT32(intr_size, IntelIOMMUState),
2029 VMSTATE_UINT16(iq_head, IntelIOMMUState),
2030 VMSTATE_UINT16(iq_tail, IntelIOMMUState),
2031 VMSTATE_UINT16(iq_size, IntelIOMMUState),
2032 VMSTATE_UINT16(next_frcd_reg, IntelIOMMUState),
2033 VMSTATE_UINT8_ARRAY(csr, IntelIOMMUState, DMAR_REG_SIZE),
2034 VMSTATE_UINT8(iq_last_desc_type, IntelIOMMUState),
2035 VMSTATE_BOOL(root_extended, IntelIOMMUState),
2036 VMSTATE_BOOL(dmar_enabled, IntelIOMMUState),
2037 VMSTATE_BOOL(qi_enabled, IntelIOMMUState),
2038 VMSTATE_BOOL(intr_enabled, IntelIOMMUState),
2039 VMSTATE_BOOL(intr_eime, IntelIOMMUState),
2040 VMSTATE_END_OF_LIST()
2041 }
1da12ec4
LT		 2042};
2043
2044static const MemoryRegionOps vtd_mem_ops = {
2045 .read = vtd_mem_read,
2046 .write = vtd_mem_write,
2047 .endianness = DEVICE_LITTLE_ENDIAN,
2048 .impl = {
2049 .min_access_size = 4,
2050 .max_access_size = 8,
2051 },
2052 .valid = {
2053 .min_access_size = 4,
2054 .max_access_size = 8,
2055 },
2056};
2057
2058static Property vtd_properties[] = {
2059 DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0),
e6b6af05
RK		 2060 DEFINE_PROP_ON_OFF_AUTO("eim", IntelIOMMUState, intr_eim,
2061 ON_OFF_AUTO_AUTO),
fb506e70 2062 DEFINE_PROP_BOOL("x-buggy-eim", IntelIOMMUState, buggy_eim, false),
3b40f0e5 2063 DEFINE_PROP_BOOL("caching-mode", IntelIOMMUState, caching_mode, FALSE),
1da12ec4
LT		 2064 DEFINE_PROP_END_OF_LIST(),
2065};
2066
651e4cef
PX		 2067/* Read IRTE entry with specific index */
2068static int vtd_irte_get(IntelIOMMUState *iommu, uint16_t index,
bc38ee10 2069 VTD_IR_TableEntry *entry, uint16_t sid)
651e4cef 2070{
ede9c94a
PX		 2071 static const uint16_t vtd_svt_mask[VTD_SQ_MAX] = \
2072 {0xffff, 0xfffb, 0xfff9, 0xfff8};
651e4cef 2073 dma_addr_t addr = 0x00;
ede9c94a
PX		 2074 uint16_t mask, source_id;
2075 uint8_t bus, bus_max, bus_min;
651e4cef
PX		 2076
2077 addr = iommu->intr_root + index * sizeof(*entry);
2078 if (dma_memory_read(&address_space_memory, addr, entry,
2079 sizeof(*entry))) {
2080 VTD_DPRINTF(GENERAL, "error: fail to access IR root at 0x%"PRIx64
2081 " + %"PRIu16, iommu->intr_root, index);
2082 return -VTD_FR_IR_ROOT_INVAL;
2083 }
2084
bc38ee10 2085 if (!entry->irte.present) {
651e4cef
PX		 2086 VTD_DPRINTF(GENERAL, "error: present flag not set in IRTE"
2087 " entry index %u value 0x%"PRIx64 " 0x%"PRIx64,
2088 index, le64_to_cpu(entry->data[1]),
2089 le64_to_cpu(entry->data[0]));
2090 return -VTD_FR_IR_ENTRY_P;
2091 }
2092
bc38ee10
MT		 2093 if (entry->irte.__reserved_0 || entry->irte.__reserved_1 ||
2094 entry->irte.__reserved_2) {
651e4cef
PX		 2095 VTD_DPRINTF(GENERAL, "error: IRTE entry index %"PRIu16
2096 " reserved fields non-zero: 0x%"PRIx64 " 0x%"PRIx64,
2097 index, le64_to_cpu(entry->data[1]),
2098 le64_to_cpu(entry->data[0]));
2099 return -VTD_FR_IR_IRTE_RSVD;
2100 }
2101
ede9c94a
PX		 2102 if (sid != X86_IOMMU_SID_INVALID) {
2103 /* Validate IRTE SID */
bc38ee10
MT		 2104 source_id = le32_to_cpu(entry->irte.source_id);
2105 switch (entry->irte.sid_vtype) {
ede9c94a
PX		 2106 case VTD_SVT_NONE:
2107 VTD_DPRINTF(IR, "No SID validation for IRTE index %d", index);
2108 break;
2109
2110 case VTD_SVT_ALL:
bc38ee10 2111 mask = vtd_svt_mask[entry->irte.sid_q];
ede9c94a
PX		 2112 if ((source_id & mask) != (sid & mask)) {
2113 VTD_DPRINTF(GENERAL, "SID validation for IRTE index "
2114 "%d failed (reqid 0x%04x sid 0x%04x)", index,
2115 sid, source_id);
2116 return -VTD_FR_IR_SID_ERR;
2117 }
2118 break;
2119
2120 case VTD_SVT_BUS:
2121 bus_max = source_id >> 8;
2122 bus_min = source_id & 0xff;
2123 bus = sid >> 8;
2124 if (bus > bus_max || bus < bus_min) {
2125 VTD_DPRINTF(GENERAL, "SID validation for IRTE index %d "
2126 "failed (bus %d outside %d-%d)", index, bus,
2127 bus_min, bus_max);
2128 return -VTD_FR_IR_SID_ERR;
2129 }
2130 break;
2131
2132 default:
2133 VTD_DPRINTF(GENERAL, "Invalid SVT bits (0x%x) in IRTE index "
bc38ee10 2134 "%d", entry->irte.sid_vtype, index);
ede9c94a
PX		 2135 /* Take this as verification failure. */
2136 return -VTD_FR_IR_SID_ERR;
2137 break;
2138 }
2139 }
651e4cef
PX		 2140
2141 return 0;
2142}
2143
2144/* Fetch IRQ information of specific IR index */
ede9c94a
PX		 2145static int vtd_remap_irq_get(IntelIOMMUState *iommu, uint16_t index,
2146 VTDIrq *irq, uint16_t sid)
651e4cef 2147{
bc38ee10 2148 VTD_IR_TableEntry irte = {};
651e4cef
PX		 2149 int ret = 0;
2150
ede9c94a 2151 ret = vtd_irte_get(iommu, index, &irte, sid);
651e4cef
PX		 2152 if (ret) {
2153 return ret;
2154 }
2155
bc38ee10
MT		 2156 irq->trigger_mode = irte.irte.trigger_mode;
2157 irq->vector = irte.irte.vector;
2158 irq->delivery_mode = irte.irte.delivery_mode;
2159 irq->dest = le32_to_cpu(irte.irte.dest_id);
28589311 2160 if (!iommu->intr_eime) {
651e4cef
PX		 2161#define VTD_IR_APIC_DEST_MASK (0xff00ULL)
2162#define VTD_IR_APIC_DEST_SHIFT (8)
28589311
JK		 2163 irq->dest = (irq->dest & VTD_IR_APIC_DEST_MASK) >>
2164 VTD_IR_APIC_DEST_SHIFT;
2165 }
bc38ee10
MT		 2166 irq->dest_mode = irte.irte.dest_mode;
2167 irq->redir_hint = irte.irte.redir_hint;
651e4cef
PX		 2168
2169 VTD_DPRINTF(IR, "remapping interrupt index %d: trig:%u,vec:%u,"
2170 "deliver:%u,dest:%u,dest_mode:%u", index,
2171 irq->trigger_mode, irq->vector, irq->delivery_mode,
2172 irq->dest, irq->dest_mode);
2173
2174 return 0;
2175}
2176
2177/* Generate one MSI message from VTDIrq info */
2178static void vtd_generate_msi_message(VTDIrq *irq, MSIMessage *msg_out)
2179{
2180 VTD_MSIMessage msg = {};
2181
2182 /* Generate address bits */
2183 msg.dest_mode = irq->dest_mode;
2184 msg.redir_hint = irq->redir_hint;
2185 msg.dest = irq->dest;
32946019 2186 msg.__addr_hi = irq->dest & 0xffffff00;
651e4cef
PX		 2187 msg.__addr_head = cpu_to_le32(0xfee);
2188 /* Keep this from original MSI address bits */
2189 msg.__not_used = irq->msi_addr_last_bits;
2190
2191 /* Generate data bits */
2192 msg.vector = irq->vector;
2193 msg.delivery_mode = irq->delivery_mode;
2194 msg.level = 1;
2195 msg.trigger_mode = irq->trigger_mode;
2196
2197 msg_out->address = msg.msi_addr;
2198 msg_out->data = msg.msi_data;
2199}
2200
2201/* Interrupt remapping for MSI/MSI-X entry */
2202static int vtd_interrupt_remap_msi(IntelIOMMUState *iommu,
2203 MSIMessage *origin,
ede9c94a
PX		 2204 MSIMessage *translated,
2205 uint16_t sid)
651e4cef
PX		 2206{
2207 int ret = 0;
2208 VTD_IR_MSIAddress addr;
2209 uint16_t index;
09cd058a 2210 VTDIrq irq = {};
651e4cef
PX		 2211
2212 assert(origin && translated);
2213
2214 if (!iommu || !iommu->intr_enabled) {
2215 goto do_not_translate;
2216 }
2217
2218 if (origin->address & VTD_MSI_ADDR_HI_MASK) {
2219 VTD_DPRINTF(GENERAL, "error: MSI addr high 32 bits nonzero"
2220 " during interrupt remapping: 0x%"PRIx32,
2221 (uint32_t)((origin->address & VTD_MSI_ADDR_HI_MASK) >> \
2222 VTD_MSI_ADDR_HI_SHIFT));
2223 return -VTD_FR_IR_REQ_RSVD;
2224 }
2225
2226 addr.data = origin->address & VTD_MSI_ADDR_LO_MASK;
1a43713b 2227 if (addr.addr.__head != 0xfee) {
651e4cef
PX		 2228 VTD_DPRINTF(GENERAL, "error: MSI addr low 32 bits invalid: "
2229 "0x%"PRIx32, addr.data);
2230 return -VTD_FR_IR_REQ_RSVD;
2231 }
2232
2233 /* This is compatible mode. */
bc38ee10 2234 if (addr.addr.int_mode != VTD_IR_INT_FORMAT_REMAP) {
651e4cef
PX		 2235 goto do_not_translate;
2236 }
2237
bc38ee10 2238 index = addr.addr.index_h << 15 | le16_to_cpu(addr.addr.index_l);
651e4cef
PX		 2239
2240#define VTD_IR_MSI_DATA_SUBHANDLE (0x0000ffff)
2241#define VTD_IR_MSI_DATA_RESERVED (0xffff0000)
2242
bc38ee10 2243 if (addr.addr.sub_valid) {
651e4cef
PX		 2244 /* See VT-d spec 5.1.2.2 and 5.1.3 on subhandle */
2245 index += origin->data & VTD_IR_MSI_DATA_SUBHANDLE;
2246 }
2247
ede9c94a 2248 ret = vtd_remap_irq_get(iommu, index, &irq, sid);
651e4cef
PX		 2249 if (ret) {
2250 return ret;
2251 }
2252
bc38ee10 2253 if (addr.addr.sub_valid) {
651e4cef
PX		 2254 VTD_DPRINTF(IR, "received MSI interrupt");
2255 if (origin->data & VTD_IR_MSI_DATA_RESERVED) {
2256 VTD_DPRINTF(GENERAL, "error: MSI data bits non-zero for "
2257 "interrupt remappable entry: 0x%"PRIx32,
2258 origin->data);
2259 return -VTD_FR_IR_REQ_RSVD;
2260 }
2261 } else {
2262 uint8_t vector = origin->data & 0xff;
dea651a9
FW		 2263 uint8_t trigger_mode = (origin->data >> MSI_DATA_TRIGGER_SHIFT) & 0x1;
2264
651e4cef
PX		 2265 VTD_DPRINTF(IR, "received IOAPIC interrupt");
2266 /* IOAPIC entry vector should be aligned with IRTE vector
2267 * (see vt-d spec 5.1.5.1). */
2268 if (vector != irq.vector) {
2269 VTD_DPRINTF(GENERAL, "IOAPIC vector inconsistent: "
2270 "entry: %d, IRTE: %d, index: %d",
2271 vector, irq.vector, index);
2272 }
dea651a9
FW		 2273
2274 /* The Trigger Mode field must match the Trigger Mode in the IRTE.
2275 * (see vt-d spec 5.1.5.1). */
2276 if (trigger_mode != irq.trigger_mode) {
2277 VTD_DPRINTF(GENERAL, "IOAPIC trigger mode inconsistent: "
2278 "entry: %u, IRTE: %u, index: %d",
2279 trigger_mode, irq.trigger_mode, index);
2280 }
2281
651e4cef
PX		 2282 }
2283
2284 /*
2285 * We'd better keep the last two bits, assuming that guest OS
2286 * might modify it. Keep it does not hurt after all.
2287 */
bc38ee10 2288 irq.msi_addr_last_bits = addr.addr.__not_care;
651e4cef
PX		 2289
2290 /* Translate VTDIrq to MSI message */
2291 vtd_generate_msi_message(&irq, translated);
2292
2293 VTD_DPRINTF(IR, "mapping MSI 0x%"PRIx64":0x%"PRIx32 " -> "
2294 "0x%"PRIx64":0x%"PRIx32, origin->address, origin->data,
2295 translated->address, translated->data);
2296 return 0;
2297
2298do_not_translate:
2299 memcpy(translated, origin, sizeof(*origin));
2300 return 0;
2301}
2302
8b5ed7df
PX		 2303static int vtd_int_remap(X86IOMMUState *iommu, MSIMessage *src,
2304 MSIMessage *dst, uint16_t sid)
2305{
ede9c94a
PX		 2306 return vtd_interrupt_remap_msi(INTEL_IOMMU_DEVICE(iommu),
2307 src, dst, sid);
8b5ed7df
PX		 2308}
2309
651e4cef
PX		 2310static MemTxResult vtd_mem_ir_read(void *opaque, hwaddr addr,
2311 uint64_t *data, unsigned size,
2312 MemTxAttrs attrs)
2313{
2314 return MEMTX_OK;
2315}
2316
2317static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr,
2318 uint64_t value, unsigned size,
2319 MemTxAttrs attrs)
2320{
2321 int ret = 0;
09cd058a 2322 MSIMessage from = {}, to = {};
ede9c94a 2323 uint16_t sid = X86_IOMMU_SID_INVALID;
651e4cef
PX		 2324
2325 from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST;
2326 from.data = (uint32_t) value;
2327
ede9c94a
PX		 2328 if (!attrs.unspecified) {
2329 /* We have explicit Source ID */
2330 sid = attrs.requester_id;
2331 }
2332
2333 ret = vtd_interrupt_remap_msi(opaque, &from, &to, sid);
651e4cef
PX		 2334 if (ret) {
2335 /* TODO: report error */
2336 VTD_DPRINTF(GENERAL, "int remap fail for addr 0x%"PRIx64
2337 " data 0x%"PRIx32, from.address, from.data);
2338 /* Drop this interrupt */
2339 return MEMTX_ERROR;
2340 }
2341
2342 VTD_DPRINTF(IR, "delivering MSI 0x%"PRIx64":0x%"PRIx32
2343 " for device sid 0x%04x",
2344 to.address, to.data, sid);
2345
32946019 2346 apic_get_class()->send_msi(&to);
651e4cef
PX		 2347
2348 return MEMTX_OK;
2349}
2350
2351static const MemoryRegionOps vtd_mem_ir_ops = {
2352 .read_with_attrs = vtd_mem_ir_read,
2353 .write_with_attrs = vtd_mem_ir_write,
2354 .endianness = DEVICE_LITTLE_ENDIAN,
2355 .impl = {
2356 .min_access_size = 4,
2357 .max_access_size = 4,
2358 },
2359 .valid = {
2360 .min_access_size = 4,
2361 .max_access_size = 4,
2362 },
2363};
7df953bd
KO		 2364
2365VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn)
2366{
2367 uintptr_t key = (uintptr_t)bus;
2368 VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key);
2369 VTDAddressSpace *vtd_dev_as;
e0a3c8cc 2370 char name[128];
7df953bd
KO		 2371
2372 if (!vtd_bus) {
2d3fc581
JW		 2373 uintptr_t *new_key = g_malloc(sizeof(*new_key));
2374 *new_key = (uintptr_t)bus;
7df953bd 2375 /* No corresponding free() */
04af0e18
PX		 2376 vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * \
2377 X86_IOMMU_PCI_DEVFN_MAX);
7df953bd 2378 vtd_bus->bus = bus;
2d3fc581 2379 g_hash_table_insert(s->vtd_as_by_busptr, new_key, vtd_bus);
7df953bd
KO		 2380 }
2381
2382 vtd_dev_as = vtd_bus->dev_as[devfn];
2383
2384 if (!vtd_dev_as) {
e0a3c8cc 2385 snprintf(name, sizeof(name), "intel_iommu_devfn_%d", devfn);
7df953bd
KO		 2386 vtd_bus->dev_as[devfn] = vtd_dev_as = g_malloc0(sizeof(VTDAddressSpace));
2387
2388 vtd_dev_as->bus = bus;
2389 vtd_dev_as->devfn = (uint8_t)devfn;
2390 vtd_dev_as->iommu_state = s;
2391 vtd_dev_as->context_cache_entry.context_cache_gen = 0;
2392 memory_region_init_iommu(&vtd_dev_as->iommu, OBJECT(s),
2393 &s->iommu_ops, "intel_iommu", UINT64_MAX);
651e4cef
PX		 2394 memory_region_init_io(&vtd_dev_as->iommu_ir, OBJECT(s),
2395 &vtd_mem_ir_ops, s, "intel_iommu_ir",
2396 VTD_INTERRUPT_ADDR_SIZE);
2397 memory_region_add_subregion(&vtd_dev_as->iommu, VTD_INTERRUPT_ADDR_FIRST,
2398 &vtd_dev_as->iommu_ir);
7df953bd 2399 address_space_init(&vtd_dev_as->as,
e0a3c8cc 2400 &vtd_dev_as->iommu, name);
7df953bd
KO		 2401 }
2402 return vtd_dev_as;
2403}
2404
1da12ec4
LT		 2405/* Do the initialization. It will also be called when reset, so pay
2406 * attention when adding new initialization stuff.
2407 */
2408static void vtd_init(IntelIOMMUState *s)
2409{
d54bd7f8
PX		 2410 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
2411
1da12ec4
LT		 2412 memset(s->csr, 0, DMAR_REG_SIZE);
2413 memset(s->wmask, 0, DMAR_REG_SIZE);
2414 memset(s->w1cmask, 0, DMAR_REG_SIZE);
2415 memset(s->womask, 0, DMAR_REG_SIZE);
2416
2417 s->iommu_ops.translate = vtd_iommu_translate;
5bf3d319 2418 s->iommu_ops.notify_flag_changed = vtd_iommu_notify_flag_changed;
1da12ec4
LT		 2419 s->root = 0;
2420 s->root_extended = false;
2421 s->dmar_enabled = false;
2422 s->iq_head = 0;
2423 s->iq_tail = 0;
2424 s->iq = 0;
2425 s->iq_size = 0;
2426 s->qi_enabled = false;
2427 s->iq_last_desc_type = VTD_INV_DESC_NONE;
2428 s->next_frcd_reg = 0;
2429 s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | VTD_CAP_MGAW |
d66b969b 2430 VTD_CAP_SAGAW | VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS;
ed7b8fbc 2431 s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO;
1da12ec4 2432
d54bd7f8 2433 if (x86_iommu->intr_supported) {
e6b6af05
RK		 2434 s->ecap |= VTD_ECAP_IR | VTD_ECAP_MHMV;
2435 if (s->intr_eim == ON_OFF_AUTO_ON) {
2436 s->ecap |= VTD_ECAP_EIM;
2437 }
2438 assert(s->intr_eim != ON_OFF_AUTO_AUTO);
d54bd7f8
PX		 2439 }
2440
554f5e16
JW		 2441 if (x86_iommu->dt_supported) {
2442 s->ecap |= VTD_ECAP_DT;
2443 }
2444
3b40f0e5
ABD		 2445 if (s->caching_mode) {
2446 s->cap |= VTD_CAP_CM;
2447 }
2448
d92fa2dc 2449 vtd_reset_context_cache(s);
b5a280c0 2450 vtd_reset_iotlb(s);
d92fa2dc 2451
1da12ec4
LT		 2452 /* Define registers with default values and bit semantics */
2453 vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0);
2454 vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0);
2455 vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0);
2456 vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0);
2457 vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL);
2458 vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0);
2459 vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffff000ULL, 0);
2460 vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0);
2461 vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL);
2462
2463 /* Advanced Fault Logging not supported */
2464 vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL);
2465 vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0);
2466 vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0);
2467 vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0);
2468
2469 /* Treated as RsvdZ when EIM in ECAP_REG is not supported
2470 * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0);
2471 */
2472 vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0);
2473
2474 /* Treated as RO for implementations that PLMR and PHMR fields reported
2475 * as Clear in the CAP_REG.
2476 * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0);
2477 */
2478 vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0);
2479
ed7b8fbc
LT		 2480 vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0);
2481 vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0);
2482 vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff007ULL, 0);
2483 vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL);
2484 vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0);
2485 vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0);
2486 vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0);
2487 /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */
2488 vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0);
2489
1da12ec4
LT		 2490 /* IOTLB registers */
2491 vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0);
2492 vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0);
2493 vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL);
2494
2495 /* Fault Recording Registers, 128-bit */
2496 vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0);
2497 vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL);
a5861439
PX		 2498
2499 /*
28589311 2500 * Interrupt remapping registers.
a5861439 2501 */
28589311 2502 vtd_define_quad(s, DMAR_IRTA_REG, 0, 0xfffffffffffff80fULL, 0);
1da12ec4
LT		 2503}
2504
2505/* Should not reset address_spaces when reset because devices will still use
2506 * the address space they got at first (won't ask the bus again).
2507 */
2508static void vtd_reset(DeviceState *dev)
2509{
2510 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
2511
2512 VTD_DPRINTF(GENERAL, "");
2513 vtd_init(s);
2514}
2515
621d983a
MA		 2516static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
2517{
2518 IntelIOMMUState *s = opaque;
2519 VTDAddressSpace *vtd_as;
2520
8e7a0a16 2521 assert(0 <= devfn && devfn < X86_IOMMU_PCI_DEVFN_MAX);
621d983a
MA		 2522
2523 vtd_as = vtd_find_add_as(s, bus, devfn);
2524 return &vtd_as->as;
2525}
2526
e6b6af05 2527static bool vtd_decide_config(IntelIOMMUState *s, Error **errp)
6333e93c 2528{
e6b6af05
RK		 2529 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
2530
6333e93c
RK		 2531 /* Currently Intel IOMMU IR only support "kernel-irqchip={off|split}" */
2532 if (x86_iommu->intr_supported && kvm_irqchip_in_kernel() &&
2533 !kvm_irqchip_is_split()) {
2534 error_setg(errp, "Intel Interrupt Remapping cannot work with "
2535 "kernel-irqchip=on, please use 'split|off'.");
2536 return false;
2537 }
e6b6af05
RK		 2538 if (s->intr_eim == ON_OFF_AUTO_ON && !x86_iommu->intr_supported) {
2539 error_setg(errp, "eim=on cannot be selected without intremap=on");
2540 return false;
2541 }
2542
2543 if (s->intr_eim == ON_OFF_AUTO_AUTO) {
fb506e70
RK		 2544 s->intr_eim = (kvm_irqchip_in_kernel() || s->buggy_eim)
2545 && x86_iommu->intr_supported ?
e6b6af05
RK		 2546 ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
2547 }
fb506e70
RK		 2548 if (s->intr_eim == ON_OFF_AUTO_ON && !s->buggy_eim) {
2549 if (!kvm_irqchip_in_kernel()) {
2550 error_setg(errp, "eim=on requires accel=kvm,kernel-irqchip=split");
2551 return false;
2552 }
2553 if (!kvm_enable_x2apic()) {
2554 error_setg(errp, "eim=on requires support on the KVM side"
2555 "(X2APIC_API, first shipped in v4.7)");
2556 return false;
2557 }
2558 }
e6b6af05 2559
6333e93c
RK		 2560 return true;
2561}
2562
1da12ec4
LT		 2563static void vtd_realize(DeviceState *dev, Error **errp)
2564{
cb135f59
PX		 2565 PCMachineState *pcms = PC_MACHINE(qdev_get_machine());
2566 PCIBus *bus = pcms->bus;
1da12ec4 2567 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
4684a204 2568 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(dev);
1da12ec4
LT		 2569
2570 VTD_DPRINTF(GENERAL, "");
fb9f5926 2571 x86_iommu->type = TYPE_INTEL;
6333e93c 2572
e6b6af05 2573 if (!vtd_decide_config(s, errp)) {
6333e93c
RK		 2574 return;
2575 }
2576
7df953bd 2577 memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num));
1da12ec4
LT		 2578 memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s,
2579 "intel_iommu", DMAR_REG_SIZE);
2580 sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem);
b5a280c0
LT		 2581 /* No corresponding destroy */
2582 s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
2583 g_free, g_free);
7df953bd
KO		 2584 s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
2585 g_free, g_free);
1da12ec4 2586 vtd_init(s);
621d983a
MA		 2587 sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, Q35_HOST_BRIDGE_IOMMU_ADDR);
2588 pci_setup_iommu(bus, vtd_host_dma_iommu, dev);
cb135f59
PX		 2589 /* Pseudo address space under root PCI bus. */
2590 pcms->ioapic_as = vtd_host_dma_iommu(bus, s, Q35_PSEUDO_DEVFN_IOAPIC);
1da12ec4
LT		 2591}
2592
2593static void vtd_class_init(ObjectClass *klass, void *data)
2594{
2595 DeviceClass *dc = DEVICE_CLASS(klass);
1c7955c4 2596 X86IOMMUClass *x86_class = X86_IOMMU_CLASS(klass);
1da12ec4
LT		 2597
2598 dc->reset = vtd_reset;
1da12ec4
LT		 2599 dc->vmsd = &vtd_vmstate;
2600 dc->props = vtd_properties;
621d983a 2601 dc->hotpluggable = false;
1c7955c4 2602 x86_class->realize = vtd_realize;
8b5ed7df 2603 x86_class->int_remap = vtd_int_remap;
1da12ec4
LT		 2604}
2605
2606static const TypeInfo vtd_info = {
2607 .name = TYPE_INTEL_IOMMU_DEVICE,
1c7955c4 2608 .parent = TYPE_X86_IOMMU_DEVICE,
1da12ec4
LT		 2609 .instance_size = sizeof(IntelIOMMUState),
2610 .class_init = vtd_class_init,
2611};
2612
2613static void vtd_register_types(void)
2614{
2615 VTD_DPRINTF(GENERAL, "");
2616 type_register_static(&vtd_info);
2617}
2618
2619type_init(vtd_register_types)
QEMU mirror