]> git.proxmox.com Git - mirror_qemu.git/blame - hw/i386/intel_iommu.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
intel_iommu: renaming gpa to iova where proper
[mirror_qemu.git] / hw / i386 / intel_iommu.c
CommitLineData
1da12ec4
LT		 1/*
2 * QEMU emulation of an Intel IOMMU (VT-d)
3 * (DMA Remapping device)
4 *
5 * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com>
6 * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, see <http://www.gnu.org/licenses/>.
20 */
21
b6a0aa05 22#include "qemu/osdep.h"
4684a204 23#include "qemu/error-report.h"
6333e93c 24#include "qapi/error.h"
1da12ec4
LT		 25#include "hw/sysbus.h"
26#include "exec/address-spaces.h"
27#include "intel_iommu_internal.h"
7df953bd 28#include "hw/pci/pci.h"
3cb3b154 29#include "hw/pci/pci_bus.h"
621d983a 30#include "hw/i386/pc.h"
dea651a9 31#include "hw/i386/apic-msidef.h"
04af0e18
PX		 32#include "hw/boards.h"
33#include "hw/i386/x86-iommu.h"
cb135f59 34#include "hw/pci-host/q35.h"
4684a204 35#include "sysemu/kvm.h"
32946019 36#include "hw/i386/apic_internal.h"
fb506e70 37#include "kvm_i386.h"
1da12ec4
LT		 38
39/*#define DEBUG_INTEL_IOMMU*/
40#ifdef DEBUG_INTEL_IOMMU
41enum {
42 DEBUG_GENERAL, DEBUG_CSR, DEBUG_INV, DEBUG_MMU, DEBUG_FLOG,
a5861439 43 DEBUG_CACHE, DEBUG_IR,
1da12ec4
LT		 44};
45#define VTD_DBGBIT(x) (1 << DEBUG_##x)
46static int vtd_dbgflags = VTD_DBGBIT(GENERAL) | VTD_DBGBIT(CSR);
47
48#define VTD_DPRINTF(what, fmt, ...) do { \
49 if (vtd_dbgflags & VTD_DBGBIT(what)) { \
50 fprintf(stderr, "(vtd)%s: " fmt "\n", __func__, \
51 ## __VA_ARGS__); } \
52 } while (0)
53#else
54#define VTD_DPRINTF(what, fmt, ...) do {} while (0)
55#endif
56
57static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val,
58 uint64_t wmask, uint64_t w1cmask)
59{
60 stq_le_p(&s->csr[addr], val);
61 stq_le_p(&s->wmask[addr], wmask);
62 stq_le_p(&s->w1cmask[addr], w1cmask);
63}
64
65static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask)
66{
67 stq_le_p(&s->womask[addr], mask);
68}
69
70static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val,
71 uint32_t wmask, uint32_t w1cmask)
72{
73 stl_le_p(&s->csr[addr], val);
74 stl_le_p(&s->wmask[addr], wmask);
75 stl_le_p(&s->w1cmask[addr], w1cmask);
76}
77
78static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask)
79{
80 stl_le_p(&s->womask[addr], mask);
81}
82
83/* "External" get/set operations */
84static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val)
85{
86 uint64_t oldval = ldq_le_p(&s->csr[addr]);
87 uint64_t wmask = ldq_le_p(&s->wmask[addr]);
88 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]);
89 stq_le_p(&s->csr[addr],
90 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val));
91}
92
93static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val)
94{
95 uint32_t oldval = ldl_le_p(&s->csr[addr]);
96 uint32_t wmask = ldl_le_p(&s->wmask[addr]);
97 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]);
98 stl_le_p(&s->csr[addr],
99 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val));
100}
101
102static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr)
103{
104 uint64_t val = ldq_le_p(&s->csr[addr]);
105 uint64_t womask = ldq_le_p(&s->womask[addr]);
106 return val & ~womask;
107}
108
109static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr)
110{
111 uint32_t val = ldl_le_p(&s->csr[addr]);
112 uint32_t womask = ldl_le_p(&s->womask[addr]);
113 return val & ~womask;
114}
115
116/* "Internal" get/set operations */
117static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr)
118{
119 return ldq_le_p(&s->csr[addr]);
120}
121
122static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr)
123{
124 return ldl_le_p(&s->csr[addr]);
125}
126
127static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val)
128{
129 stq_le_p(&s->csr[addr], val);
130}
131
132static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr,
133 uint32_t clear, uint32_t mask)
134{
135 uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask;
136 stl_le_p(&s->csr[addr], new_val);
137 return new_val;
138}
139
140static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr,
141 uint64_t clear, uint64_t mask)
142{
143 uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask;
144 stq_le_p(&s->csr[addr], new_val);
145 return new_val;
146}
147
b5a280c0
LT		 148/* GHashTable functions */
149static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2)
150{
151 return *((const uint64_t *)v1) == *((const uint64_t *)v2);
152}
153
154static guint vtd_uint64_hash(gconstpointer v)
155{
156 return (guint)*(const uint64_t *)v;
157}
158
159static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value,
160 gpointer user_data)
161{
162 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
163 uint16_t domain_id = *(uint16_t *)user_data;
164 return entry->domain_id == domain_id;
165}
166
d66b969b
JW		 167/* The shift of an addr for a certain level of paging structure */
168static inline uint32_t vtd_slpt_level_shift(uint32_t level)
169{
170 return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS;
171}
172
173static inline uint64_t vtd_slpt_level_page_mask(uint32_t level)
174{
175 return ~((1ULL << vtd_slpt_level_shift(level)) - 1);
176}
177
b5a280c0
LT		 178static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value,
179 gpointer user_data)
180{
181 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
182 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data;
d66b969b
JW		 183 uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask;
184 uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K;
b5a280c0 185 return (entry->domain_id == info->domain_id) &&
d66b969b
JW		 186 (((entry->gfn & info->mask) == gfn) ||
187 (entry->gfn == gfn_tlb));
b5a280c0
LT		 188}
189
d92fa2dc
LT		 190/* Reset all the gen of VTDAddressSpace to zero and set the gen of
191 * IntelIOMMUState to 1.
192 */
193static void vtd_reset_context_cache(IntelIOMMUState *s)
194{
d92fa2dc 195 VTDAddressSpace *vtd_as;
7df953bd
KO		 196 VTDBus *vtd_bus;
197 GHashTableIter bus_it;
d92fa2dc
LT		 198 uint32_t devfn_it;
199
7df953bd
KO		 200 g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr);
201
d92fa2dc 202 VTD_DPRINTF(CACHE, "global context_cache_gen=1");
7df953bd 203 while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) {
04af0e18 204 for (devfn_it = 0; devfn_it < X86_IOMMU_PCI_DEVFN_MAX; ++devfn_it) {
7df953bd 205 vtd_as = vtd_bus->dev_as[devfn_it];
d92fa2dc
LT		 206 if (!vtd_as) {
207 continue;
208 }
209 vtd_as->context_cache_entry.context_cache_gen = 0;
210 }
211 }
212 s->context_cache_gen = 1;
213}
214
b5a280c0
LT		 215static void vtd_reset_iotlb(IntelIOMMUState *s)
216{
217 assert(s->iotlb);
218 g_hash_table_remove_all(s->iotlb);
219}
220
bacabb0a 221static uint64_t vtd_get_iotlb_key(uint64_t gfn, uint16_t source_id,
d66b969b
JW		 222 uint32_t level)
223{
224 return gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT) |
225 ((uint64_t)(level) << VTD_IOTLB_LVL_SHIFT);
226}
227
228static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level)
229{
230 return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K;
231}
232
b5a280c0
LT		 233static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id,
234 hwaddr addr)
235{
d66b969b 236 VTDIOTLBEntry *entry;
b5a280c0 237 uint64_t key;
d66b969b
JW		 238 int level;
239
240 for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) {
241 key = vtd_get_iotlb_key(vtd_get_iotlb_gfn(addr, level),
242 source_id, level);
243 entry = g_hash_table_lookup(s->iotlb, &key);
244 if (entry) {
245 goto out;
246 }
247 }
b5a280c0 248
d66b969b
JW		 249out:
250 return entry;
b5a280c0
LT		 251}
252
253static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id,
254 uint16_t domain_id, hwaddr addr, uint64_t slpte,
d66b969b
JW		 255 bool read_flags, bool write_flags,
256 uint32_t level)
b5a280c0
LT		 257{
258 VTDIOTLBEntry *entry = g_malloc(sizeof(*entry));
259 uint64_t *key = g_malloc(sizeof(*key));
d66b969b 260 uint64_t gfn = vtd_get_iotlb_gfn(addr, level);
b5a280c0 261
6e905564 262 VTD_DPRINTF(CACHE, "update iotlb sid 0x%"PRIx16 " iova 0x%"PRIx64
b5a280c0
LT		 263 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr, slpte,
264 domain_id);
265 if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) {
266 VTD_DPRINTF(CACHE, "iotlb exceeds size limit, forced to reset");
267 vtd_reset_iotlb(s);
268 }
269
270 entry->gfn = gfn;
271 entry->domain_id = domain_id;
272 entry->slpte = slpte;
273 entry->read_flags = read_flags;
274 entry->write_flags = write_flags;
d66b969b
JW		 275 entry->mask = vtd_slpt_level_page_mask(level);
276 *key = vtd_get_iotlb_key(gfn, source_id, level);
b5a280c0
LT		 277 g_hash_table_replace(s->iotlb, key, entry);
278}
279
1da12ec4
LT		 280/* Given the reg addr of both the message data and address, generate an
281 * interrupt via MSI.
282 */
283static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg,
284 hwaddr mesg_data_reg)
285{
32946019 286 MSIMessage msi;
1da12ec4
LT		 287
288 assert(mesg_data_reg < DMAR_REG_SIZE);
289 assert(mesg_addr_reg < DMAR_REG_SIZE);
290
32946019
RK		 291 msi.address = vtd_get_long_raw(s, mesg_addr_reg);
292 msi.data = vtd_get_long_raw(s, mesg_data_reg);
1da12ec4 293
32946019
RK		 294 VTD_DPRINTF(FLOG, "msi: addr 0x%"PRIx64 " data 0x%"PRIx32,
295 msi.address, msi.data);
296 apic_get_class()->send_msi(&msi);
1da12ec4
LT		 297}
298
299/* Generate a fault event to software via MSI if conditions are met.
300 * Notice that the value of FSTS_REG being passed to it should be the one
301 * before any update.
302 */
303static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts)
304{
305 if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO ||
306 pre_fsts & VTD_FSTS_IQE) {
307 VTD_DPRINTF(FLOG, "there are previous interrupt conditions "
308 "to be serviced by software, fault event is not generated "
309 "(FSTS_REG 0x%"PRIx32 ")", pre_fsts);
310 return;
311 }
312 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP);
313 if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) {
314 VTD_DPRINTF(FLOG, "Interrupt Mask set, fault event is not generated");
315 } else {
316 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
317 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
318 }
319}
320
321/* Check if the Fault (F) field of the Fault Recording Register referenced by
322 * @index is Set.
323 */
324static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index)
325{
326 /* Each reg is 128-bit */
327 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
328 addr += 8; /* Access the high 64-bit half */
329
330 assert(index < DMAR_FRCD_REG_NR);
331
332 return vtd_get_quad_raw(s, addr) & VTD_FRCD_F;
333}
334
335/* Update the PPF field of Fault Status Register.
336 * Should be called whenever change the F field of any fault recording
337 * registers.
338 */
339static void vtd_update_fsts_ppf(IntelIOMMUState *s)
340{
341 uint32_t i;
342 uint32_t ppf_mask = 0;
343
344 for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
345 if (vtd_is_frcd_set(s, i)) {
346 ppf_mask = VTD_FSTS_PPF;
347 break;
348 }
349 }
350 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask);
351 VTD_DPRINTF(FLOG, "set PPF of FSTS_REG to %d", ppf_mask ? 1 : 0);
352}
353
354static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index)
355{
356 /* Each reg is 128-bit */
357 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
358 addr += 8; /* Access the high 64-bit half */
359
360 assert(index < DMAR_FRCD_REG_NR);
361
362 vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F);
363 vtd_update_fsts_ppf(s);
364}
365
366/* Must not update F field now, should be done later */
367static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index,
368 uint16_t source_id, hwaddr addr,
369 VTDFaultReason fault, bool is_write)
370{
371 uint64_t hi = 0, lo;
372 hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
373
374 assert(index < DMAR_FRCD_REG_NR);
375
376 lo = VTD_FRCD_FI(addr);
377 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault);
378 if (!is_write) {
379 hi |= VTD_FRCD_T;
380 }
381 vtd_set_quad_raw(s, frcd_reg_addr, lo);
382 vtd_set_quad_raw(s, frcd_reg_addr + 8, hi);
383 VTD_DPRINTF(FLOG, "record to FRCD_REG #%"PRIu16 ": hi 0x%"PRIx64
384 ", lo 0x%"PRIx64, index, hi, lo);
385}
386
387/* Try to collapse multiple pending faults from the same requester */
388static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id)
389{
390 uint32_t i;
391 uint64_t frcd_reg;
392 hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */
393
394 for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
395 frcd_reg = vtd_get_quad_raw(s, addr);
396 VTD_DPRINTF(FLOG, "frcd_reg #%d 0x%"PRIx64, i, frcd_reg);
397 if ((frcd_reg & VTD_FRCD_F) &&
398 ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) {
399 return true;
400 }
401 addr += 16; /* 128-bit for each */
402 }
403 return false;
404}
405
406/* Log and report an DMAR (address translation) fault to software */
407static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id,
408 hwaddr addr, VTDFaultReason fault,
409 bool is_write)
410{
411 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
412
413 assert(fault < VTD_FR_MAX);
414
415 if (fault == VTD_FR_RESERVED_ERR) {
416 /* This is not a normal fault reason case. Drop it. */
417 return;
418 }
419 VTD_DPRINTF(FLOG, "sid 0x%"PRIx16 ", fault %d, addr 0x%"PRIx64
420 ", is_write %d", source_id, fault, addr, is_write);
421 if (fsts_reg & VTD_FSTS_PFO) {
422 VTD_DPRINTF(FLOG, "new fault is not recorded due to "
423 "Primary Fault Overflow");
424 return;
425 }
426 if (vtd_try_collapse_fault(s, source_id)) {
427 VTD_DPRINTF(FLOG, "new fault is not recorded due to "
428 "compression of faults");
429 return;
430 }
431 if (vtd_is_frcd_set(s, s->next_frcd_reg)) {
432 VTD_DPRINTF(FLOG, "Primary Fault Overflow and "
433 "new fault is not recorded, set PFO field");
434 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO);
435 return;
436 }
437
438 vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write);
439
440 if (fsts_reg & VTD_FSTS_PPF) {
441 VTD_DPRINTF(FLOG, "there are pending faults already, "
442 "fault event is not generated");
443 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg);
444 s->next_frcd_reg++;
445 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
446 s->next_frcd_reg = 0;
447 }
448 } else {
449 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK,
450 VTD_FSTS_FRI(s->next_frcd_reg));
451 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */
452 s->next_frcd_reg++;
453 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
454 s->next_frcd_reg = 0;
455 }
456 /* This case actually cause the PPF to be Set.
457 * So generate fault event (interrupt).
458 */
459 vtd_generate_fault_event(s, fsts_reg);
460 }
461}
462
ed7b8fbc
LT		 463/* Handle Invalidation Queue Errors of queued invalidation interface error
464 * conditions.
465 */
466static void vtd_handle_inv_queue_error(IntelIOMMUState *s)
467{
468 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
469
470 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE);
471 vtd_generate_fault_event(s, fsts_reg);
472}
473
474/* Set the IWC field and try to generate an invalidation completion interrupt */
475static void vtd_generate_completion_event(IntelIOMMUState *s)
476{
477 VTD_DPRINTF(INV, "completes an invalidation wait command with "
478 "Interrupt Flag");
479 if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) {
480 VTD_DPRINTF(INV, "there is a previous interrupt condition to be "
481 "serviced by software, "
482 "new invalidation event is not generated");
483 return;
484 }
485 vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC);
486 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP);
487 if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) {
488 VTD_DPRINTF(INV, "IM filed in IECTL_REG is set, new invalidation "
489 "event is not generated");
490 return;
491 } else {
492 /* Generate the interrupt event */
493 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
494 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
495 }
496}
497
1da12ec4
LT		 498static inline bool vtd_root_entry_present(VTDRootEntry *root)
499{
500 return root->val & VTD_ROOT_ENTRY_P;
501}
502
503static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index,
504 VTDRootEntry *re)
505{
506 dma_addr_t addr;
507
508 addr = s->root + index * sizeof(*re);
509 if (dma_memory_read(&address_space_memory, addr, re, sizeof(*re))) {
510 VTD_DPRINTF(GENERAL, "error: fail to access root-entry at 0x%"PRIx64
511 " + %"PRIu8, s->root, index);
512 re->val = 0;
513 return -VTD_FR_ROOT_TABLE_INV;
514 }
515 re->val = le64_to_cpu(re->val);
516 return 0;
517}
518
519static inline bool vtd_context_entry_present(VTDContextEntry *context)
520{
521 return context->lo & VTD_CONTEXT_ENTRY_P;
522}
523
524static int vtd_get_context_entry_from_root(VTDRootEntry *root, uint8_t index,
525 VTDContextEntry *ce)
526{
527 dma_addr_t addr;
528
529 if (!vtd_root_entry_present(root)) {
530 VTD_DPRINTF(GENERAL, "error: root-entry is not present");
531 return -VTD_FR_ROOT_ENTRY_P;
532 }
533 addr = (root->val & VTD_ROOT_ENTRY_CTP) + index * sizeof(*ce);
534 if (dma_memory_read(&address_space_memory, addr, ce, sizeof(*ce))) {
535 VTD_DPRINTF(GENERAL, "error: fail to access context-entry at 0x%"PRIx64
536 " + %"PRIu8,
537 (uint64_t)(root->val & VTD_ROOT_ENTRY_CTP), index);
538 return -VTD_FR_CONTEXT_TABLE_INV;
539 }
540 ce->lo = le64_to_cpu(ce->lo);
541 ce->hi = le64_to_cpu(ce->hi);
542 return 0;
543}
544
545static inline dma_addr_t vtd_get_slpt_base_from_context(VTDContextEntry *ce)
546{
547 return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR;
548}
549
1da12ec4
LT		 550static inline uint64_t vtd_get_slpte_addr(uint64_t slpte)
551{
552 return slpte & VTD_SL_PT_BASE_ADDR_MASK;
553}
554
555/* Whether the pte indicates the address of the page frame */
556static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level)
557{
558 return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK);
559}
560
561/* Get the content of a spte located in @base_addr[@index] */
562static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index)
563{
564 uint64_t slpte;
565
566 assert(index < VTD_SL_PT_ENTRY_NR);
567
568 if (dma_memory_read(&address_space_memory,
569 base_addr + index * sizeof(slpte), &slpte,
570 sizeof(slpte))) {
571 slpte = (uint64_t)-1;
572 return slpte;
573 }
574 slpte = le64_to_cpu(slpte);
575 return slpte;
576}
577
6e905564
PX		 578/* Given an iova and the level of paging structure, return the offset
579 * of current level.
1da12ec4 580 */
6e905564 581static inline uint32_t vtd_iova_level_offset(uint64_t iova, uint32_t level)
1da12ec4 582{
6e905564 583 return (iova >> vtd_slpt_level_shift(level)) &
1da12ec4
LT		 584 ((1ULL << VTD_SL_LEVEL_BITS) - 1);
585}
586
587/* Check Capability Register to see if the @level of page-table is supported */
588static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level)
589{
590 return VTD_CAP_SAGAW_MASK & s->cap &
591 (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT));
592}
593
594/* Get the page-table level that hardware should use for the second-level
595 * page-table walk from the Address Width field of context-entry.
596 */
597static inline uint32_t vtd_get_level_from_context_entry(VTDContextEntry *ce)
598{
599 return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW);
600}
601
602static inline uint32_t vtd_get_agaw_from_context_entry(VTDContextEntry *ce)
603{
604 return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9;
605}
606
607static const uint64_t vtd_paging_entry_rsvd_field[] = {
608 [0] = ~0ULL,
609 /* For not large page */
610 [1] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
611 [2] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
612 [3] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
613 [4] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
614 /* For large page */
615 [5] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
616 [6] = 0x1ff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
617 [7] = 0x3ffff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
618 [8] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
619};
620
621static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level)
622{
623 if (slpte & VTD_SL_PT_PAGE_SIZE_MASK) {
624 /* Maybe large page */
625 return slpte & vtd_paging_entry_rsvd_field[level + 4];
626 } else {
627 return slpte & vtd_paging_entry_rsvd_field[level];
628 }
629}
630
6e905564 631/* Given the @iova, get relevant @slptep. @slpte_level will be the last level
1da12ec4
LT		 632 * of the translation, can be used for deciding the size of large page.
633 */
6e905564
PX		 634static int vtd_iova_to_slpte(VTDContextEntry *ce, uint64_t iova, bool is_write,
635 uint64_t *slptep, uint32_t *slpte_level,
636 bool *reads, bool *writes)
1da12ec4
LT		 637{
638 dma_addr_t addr = vtd_get_slpt_base_from_context(ce);
639 uint32_t level = vtd_get_level_from_context_entry(ce);
640 uint32_t offset;
641 uint64_t slpte;
642 uint32_t ce_agaw = vtd_get_agaw_from_context_entry(ce);
643 uint64_t access_right_check;
644
6e905564
PX		 645 /* Check if @iova is above 2^X-1, where X is the minimum of MGAW
646 * in CAP_REG and AW in context-entry.
1da12ec4 647 */
6e905564
PX		 648 if (iova & ~((1ULL << MIN(ce_agaw, VTD_MGAW)) - 1)) {
649 VTD_DPRINTF(GENERAL, "error: iova 0x%"PRIx64 " exceeds limits", iova);
1da12ec4
LT		 650 return -VTD_FR_ADDR_BEYOND_MGAW;
651 }
652
653 /* FIXME: what is the Atomics request here? */
654 access_right_check = is_write ? VTD_SL_W : VTD_SL_R;
655
656 while (true) {
6e905564 657 offset = vtd_iova_level_offset(iova, level);
1da12ec4
LT		 658 slpte = vtd_get_slpte(addr, offset);
659
660 if (slpte == (uint64_t)-1) {
661 VTD_DPRINTF(GENERAL, "error: fail to access second-level paging "
6e905564
PX		 662 "entry at level %"PRIu32 " for iova 0x%"PRIx64,
663 level, iova);
1da12ec4
LT		 664 if (level == vtd_get_level_from_context_entry(ce)) {
665 /* Invalid programming of context-entry */
666 return -VTD_FR_CONTEXT_ENTRY_INV;
667 } else {
668 return -VTD_FR_PAGING_ENTRY_INV;
669 }
670 }
671 *reads = (*reads) && (slpte & VTD_SL_R);
672 *writes = (*writes) && (slpte & VTD_SL_W);
673 if (!(slpte & access_right_check)) {
674 VTD_DPRINTF(GENERAL, "error: lack of %s permission for "
6e905564
PX		 675 "iova 0x%"PRIx64 " slpte 0x%"PRIx64,
676 (is_write ? "write" : "read"), iova, slpte);
1da12ec4
LT		 677 return is_write ? -VTD_FR_WRITE : -VTD_FR_READ;
678 }
679 if (vtd_slpte_nonzero_rsvd(slpte, level)) {
680 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in second "
681 "level paging entry level %"PRIu32 " slpte 0x%"PRIx64,
682 level, slpte);
683 return -VTD_FR_PAGING_ENTRY_RSVD;
684 }
685
686 if (vtd_is_last_slpte(slpte, level)) {
687 *slptep = slpte;
688 *slpte_level = level;
689 return 0;
690 }
691 addr = vtd_get_slpte_addr(slpte);
692 level--;
693 }
694}
695
696/* Map a device to its corresponding domain (context-entry) */
697static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num,
698 uint8_t devfn, VTDContextEntry *ce)
699{
700 VTDRootEntry re;
701 int ret_fr;
702
703 ret_fr = vtd_get_root_entry(s, bus_num, &re);
704 if (ret_fr) {
705 return ret_fr;
706 }
707
708 if (!vtd_root_entry_present(&re)) {
709 VTD_DPRINTF(GENERAL, "error: root-entry #%"PRIu8 " is not present",
710 bus_num);
711 return -VTD_FR_ROOT_ENTRY_P;
712 } else if (re.rsvd || (re.val & VTD_ROOT_ENTRY_RSVD)) {
713 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in root-entry "
714 "hi 0x%"PRIx64 " lo 0x%"PRIx64, re.rsvd, re.val);
715 return -VTD_FR_ROOT_ENTRY_RSVD;
716 }
717
718 ret_fr = vtd_get_context_entry_from_root(&re, devfn, ce);
719 if (ret_fr) {
720 return ret_fr;
721 }
722
723 if (!vtd_context_entry_present(ce)) {
724 VTD_DPRINTF(GENERAL,
725 "error: context-entry #%"PRIu8 "(bus #%"PRIu8 ") "
726 "is not present", devfn, bus_num);
727 return -VTD_FR_CONTEXT_ENTRY_P;
728 } else if ((ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI) ||
729 (ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO)) {
730 VTD_DPRINTF(GENERAL,
731 "error: non-zero reserved field in context-entry "
732 "hi 0x%"PRIx64 " lo 0x%"PRIx64, ce->hi, ce->lo);
733 return -VTD_FR_CONTEXT_ENTRY_RSVD;
734 }
735 /* Check if the programming of context-entry is valid */
736 if (!vtd_is_level_supported(s, vtd_get_level_from_context_entry(ce))) {
737 VTD_DPRINTF(GENERAL, "error: unsupported Address Width value in "
738 "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64,
739 ce->hi, ce->lo);
740 return -VTD_FR_CONTEXT_ENTRY_INV;
554f5e16
JW		 741 } else {
742 switch (ce->lo & VTD_CONTEXT_ENTRY_TT) {
743 case VTD_CONTEXT_TT_MULTI_LEVEL:
744 /* fall through */
745 case VTD_CONTEXT_TT_DEV_IOTLB:
746 break;
747 default:
748 VTD_DPRINTF(GENERAL, "error: unsupported Translation Type in "
749 "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64,
750 ce->hi, ce->lo);
751 return -VTD_FR_CONTEXT_ENTRY_INV;
752 }
1da12ec4
LT		 753 }
754 return 0;
755}
756
757static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn)
758{
759 return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL);
760}
761
762static const bool vtd_qualified_faults[] = {
763 [VTD_FR_RESERVED] = false,
764 [VTD_FR_ROOT_ENTRY_P] = false,
765 [VTD_FR_CONTEXT_ENTRY_P] = true,
766 [VTD_FR_CONTEXT_ENTRY_INV] = true,
767 [VTD_FR_ADDR_BEYOND_MGAW] = true,
768 [VTD_FR_WRITE] = true,
769 [VTD_FR_READ] = true,
770 [VTD_FR_PAGING_ENTRY_INV] = true,
771 [VTD_FR_ROOT_TABLE_INV] = false,
772 [VTD_FR_CONTEXT_TABLE_INV] = false,
773 [VTD_FR_ROOT_ENTRY_RSVD] = false,
774 [VTD_FR_PAGING_ENTRY_RSVD] = true,
775 [VTD_FR_CONTEXT_ENTRY_TT] = true,
776 [VTD_FR_RESERVED_ERR] = false,
777 [VTD_FR_MAX] = false,
778};
779
780/* To see if a fault condition is "qualified", which is reported to software
781 * only if the FPD field in the context-entry used to process the faulting
782 * request is 0.
783 */
784static inline bool vtd_is_qualified_fault(VTDFaultReason fault)
785{
786 return vtd_qualified_faults[fault];
787}
788
789static inline bool vtd_is_interrupt_addr(hwaddr addr)
790{
791 return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST;
792}
793
794/* Map dev to context-entry then do a paging-structures walk to do a iommu
795 * translation.
79e2b9ae
PB		 796 *
797 * Called from RCU critical section.
798 *
1da12ec4
LT		 799 * @bus_num: The bus number
800 * @devfn: The devfn, which is the combined of device and function number
801 * @is_write: The access is a write operation
802 * @entry: IOMMUTLBEntry that contain the addr to be translated and result
803 */
7df953bd 804static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus,
1da12ec4
LT		 805 uint8_t devfn, hwaddr addr, bool is_write,
806 IOMMUTLBEntry *entry)
807{
d92fa2dc 808 IntelIOMMUState *s = vtd_as->iommu_state;
1da12ec4 809 VTDContextEntry ce;
7df953bd 810 uint8_t bus_num = pci_bus_num(bus);
d92fa2dc 811 VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry;
d66b969b 812 uint64_t slpte, page_mask;
1da12ec4
LT		 813 uint32_t level;
814 uint16_t source_id = vtd_make_source_id(bus_num, devfn);
815 int ret_fr;
816 bool is_fpd_set = false;
817 bool reads = true;
818 bool writes = true;
b5a280c0 819 VTDIOTLBEntry *iotlb_entry;
1da12ec4 820
046ab7e9
PX		 821 /*
822 * We have standalone memory region for interrupt addresses, we
823 * should never receive translation requests in this region.
824 */
825 assert(!vtd_is_interrupt_addr(addr));
826
b5a280c0
LT		 827 /* Try to fetch slpte form IOTLB */
828 iotlb_entry = vtd_lookup_iotlb(s, source_id, addr);
829 if (iotlb_entry) {
6e905564 830 VTD_DPRINTF(CACHE, "hit iotlb sid 0x%"PRIx16 " iova 0x%"PRIx64
b5a280c0
LT		 831 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr,
832 iotlb_entry->slpte, iotlb_entry->domain_id);
833 slpte = iotlb_entry->slpte;
834 reads = iotlb_entry->read_flags;
835 writes = iotlb_entry->write_flags;
d66b969b 836 page_mask = iotlb_entry->mask;
b5a280c0
LT		 837 goto out;
838 }
d92fa2dc
LT		 839 /* Try to fetch context-entry from cache first */
840 if (cc_entry->context_cache_gen == s->context_cache_gen) {
841 VTD_DPRINTF(CACHE, "hit context-cache bus %d devfn %d "
842 "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 ")",
843 bus_num, devfn, cc_entry->context_entry.hi,
844 cc_entry->context_entry.lo, cc_entry->context_cache_gen);
845 ce = cc_entry->context_entry;
846 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
847 } else {
848 ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce);
849 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
850 if (ret_fr) {
851 ret_fr = -ret_fr;
852 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {
853 VTD_DPRINTF(FLOG, "fault processing is disabled for DMA "
854 "requests through this context-entry "
855 "(with FPD Set)");
856 } else {
857 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);
858 }
859 return;
1da12ec4 860 }
d92fa2dc
LT		 861 /* Update context-cache */
862 VTD_DPRINTF(CACHE, "update context-cache bus %d devfn %d "
863 "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 "->%"PRIu32 ")",
864 bus_num, devfn, ce.hi, ce.lo,
865 cc_entry->context_cache_gen, s->context_cache_gen);
866 cc_entry->context_entry = ce;
867 cc_entry->context_cache_gen = s->context_cache_gen;
1da12ec4
LT		 868 }
869
6e905564
PX		 870 ret_fr = vtd_iova_to_slpte(&ce, addr, is_write, &slpte, &level,
871 &reads, &writes);
1da12ec4
LT		 872 if (ret_fr) {
873 ret_fr = -ret_fr;
874 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {
875 VTD_DPRINTF(FLOG, "fault processing is disabled for DMA requests "
876 "through this context-entry (with FPD Set)");
877 } else {
878 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);
879 }
880 return;
881 }
882
d66b969b 883 page_mask = vtd_slpt_level_page_mask(level);
b5a280c0 884 vtd_update_iotlb(s, source_id, VTD_CONTEXT_ENTRY_DID(ce.hi), addr, slpte,
d66b969b 885 reads, writes, level);
b5a280c0 886out:
d66b969b
JW		 887 entry->iova = addr & page_mask;
888 entry->translated_addr = vtd_get_slpte_addr(slpte) & page_mask;
889 entry->addr_mask = ~page_mask;
1da12ec4
LT		 890 entry->perm = (writes ? 2 : 0) + (reads ? 1 : 0);
891}
892
893static void vtd_root_table_setup(IntelIOMMUState *s)
894{
895 s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG);
896 s->root_extended = s->root & VTD_RTADDR_RTT;
897 s->root &= VTD_RTADDR_ADDR_MASK;
898
899 VTD_DPRINTF(CSR, "root_table addr 0x%"PRIx64 " %s", s->root,
900 (s->root_extended ? "(extended)" : ""));
901}
902
02a2cbc8
PX		 903static void vtd_iec_notify_all(IntelIOMMUState *s, bool global,
904 uint32_t index, uint32_t mask)
905{
906 x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask);
907}
908
a5861439
PX		 909static void vtd_interrupt_remap_table_setup(IntelIOMMUState *s)
910{
911 uint64_t value = 0;
912 value = vtd_get_quad_raw(s, DMAR_IRTA_REG);
913 s->intr_size = 1UL << ((value & VTD_IRTA_SIZE_MASK) + 1);
914 s->intr_root = value & VTD_IRTA_ADDR_MASK;
28589311 915 s->intr_eime = value & VTD_IRTA_EIME;
a5861439 916
02a2cbc8
PX		 917 /* Notify global invalidation */
918 vtd_iec_notify_all(s, true, 0, 0);
a5861439
PX		 919
920 VTD_DPRINTF(CSR, "int remap table addr 0x%"PRIx64 " size %"PRIu32,
921 s->intr_root, s->intr_size);
922}
923
d92fa2dc
LT		 924static void vtd_context_global_invalidate(IntelIOMMUState *s)
925{
926 s->context_cache_gen++;
927 if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) {
928 vtd_reset_context_cache(s);
929 }
930}
931
7df953bd
KO		 932
933/* Find the VTD address space currently associated with a given bus number,
934 */
935static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num)
936{
937 VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num];
938 if (!vtd_bus) {
939 /* Iterate over the registered buses to find the one
940 * which currently hold this bus number, and update the bus_num lookup table:
941 */
942 GHashTableIter iter;
943
944 g_hash_table_iter_init(&iter, s->vtd_as_by_busptr);
945 while (g_hash_table_iter_next (&iter, NULL, (void**)&vtd_bus)) {
946 if (pci_bus_num(vtd_bus->bus) == bus_num) {
947 s->vtd_as_by_bus_num[bus_num] = vtd_bus;
948 return vtd_bus;
949 }
950 }
951 }
952 return vtd_bus;
953}
954
d92fa2dc
LT		 955/* Do a context-cache device-selective invalidation.
956 * @func_mask: FM field after shifting
957 */
958static void vtd_context_device_invalidate(IntelIOMMUState *s,
959 uint16_t source_id,
960 uint16_t func_mask)
961{
962 uint16_t mask;
7df953bd 963 VTDBus *vtd_bus;
d92fa2dc
LT		 964 VTDAddressSpace *vtd_as;
965 uint16_t devfn;
966 uint16_t devfn_it;
967
968 switch (func_mask & 3) {
969 case 0:
970 mask = 0; /* No bits in the SID field masked */
971 break;
972 case 1:
973 mask = 4; /* Mask bit 2 in the SID field */
974 break;
975 case 2:
976 mask = 6; /* Mask bit 2:1 in the SID field */
977 break;
978 case 3:
979 mask = 7; /* Mask bit 2:0 in the SID field */
980 break;
981 }
6cb99acc 982 mask = ~mask;
d92fa2dc
LT		 983 VTD_DPRINTF(INV, "device-selective invalidation source 0x%"PRIx16
984 " mask %"PRIu16, source_id, mask);
7df953bd
KO		 985 vtd_bus = vtd_find_as_from_bus_num(s, VTD_SID_TO_BUS(source_id));
986 if (vtd_bus) {
d92fa2dc 987 devfn = VTD_SID_TO_DEVFN(source_id);
04af0e18 988 for (devfn_it = 0; devfn_it < X86_IOMMU_PCI_DEVFN_MAX; ++devfn_it) {
7df953bd 989 vtd_as = vtd_bus->dev_as[devfn_it];
d92fa2dc
LT		 990 if (vtd_as && ((devfn_it & mask) == (devfn & mask))) {
991 VTD_DPRINTF(INV, "invalidate context-cahce of devfn 0x%"PRIx16,
992 devfn_it);
993 vtd_as->context_cache_entry.context_cache_gen = 0;
994 }
995 }
996 }
997}
998
1da12ec4
LT		 999/* Context-cache invalidation
1000 * Returns the Context Actual Invalidation Granularity.
1001 * @val: the content of the CCMD_REG
1002 */
1003static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val)
1004{
1005 uint64_t caig;
1006 uint64_t type = val & VTD_CCMD_CIRG_MASK;
1007
1008 switch (type) {
d92fa2dc
LT		 1009 case VTD_CCMD_DOMAIN_INVL:
1010 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
1011 (uint16_t)VTD_CCMD_DID(val));
1012 /* Fall through */
1da12ec4 1013 case VTD_CCMD_GLOBAL_INVL:
d92fa2dc 1014 VTD_DPRINTF(INV, "global invalidation");
1da12ec4 1015 caig = VTD_CCMD_GLOBAL_INVL_A;
d92fa2dc 1016 vtd_context_global_invalidate(s);
1da12ec4
LT		 1017 break;
1018
1019 case VTD_CCMD_DEVICE_INVL:
1da12ec4 1020 caig = VTD_CCMD_DEVICE_INVL_A;
d92fa2dc 1021 vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val));
1da12ec4
LT		 1022 break;
1023
1024 default:
d92fa2dc 1025 VTD_DPRINTF(GENERAL, "error: invalid granularity");
1da12ec4
LT		 1026 caig = 0;
1027 }
1028 return caig;
1029}
1030
b5a280c0
LT		 1031static void vtd_iotlb_global_invalidate(IntelIOMMUState *s)
1032{
1033 vtd_reset_iotlb(s);
1034}
1035
1036static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id)
1037{
1038 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain,
1039 &domain_id);
1040}
1041
1042static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id,
1043 hwaddr addr, uint8_t am)
1044{
1045 VTDIOTLBPageInvInfo info;
1046
1047 assert(am <= VTD_MAMV);
1048 info.domain_id = domain_id;
d66b969b 1049 info.addr = addr;
b5a280c0
LT		 1050 info.mask = ~((1 << am) - 1);
1051 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info);
1052}
1053
1da12ec4
LT		 1054/* Flush IOTLB
1055 * Returns the IOTLB Actual Invalidation Granularity.
1056 * @val: the content of the IOTLB_REG
1057 */
1058static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val)
1059{
1060 uint64_t iaig;
1061 uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK;
b5a280c0
LT		 1062 uint16_t domain_id;
1063 hwaddr addr;
1064 uint8_t am;
1da12ec4
LT		 1065
1066 switch (type) {
1067 case VTD_TLB_GLOBAL_FLUSH:
b5a280c0 1068 VTD_DPRINTF(INV, "global invalidation");
1da12ec4 1069 iaig = VTD_TLB_GLOBAL_FLUSH_A;
b5a280c0 1070 vtd_iotlb_global_invalidate(s);
1da12ec4
LT		 1071 break;
1072
1073 case VTD_TLB_DSI_FLUSH:
b5a280c0
LT		 1074 domain_id = VTD_TLB_DID(val);
1075 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
1076 domain_id);
1da12ec4 1077 iaig = VTD_TLB_DSI_FLUSH_A;
b5a280c0 1078 vtd_iotlb_domain_invalidate(s, domain_id);
1da12ec4
LT		 1079 break;
1080
1081 case VTD_TLB_PSI_FLUSH:
b5a280c0
LT		 1082 domain_id = VTD_TLB_DID(val);
1083 addr = vtd_get_quad_raw(s, DMAR_IVA_REG);
1084 am = VTD_IVA_AM(addr);
1085 addr = VTD_IVA_ADDR(addr);
1086 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16
1087 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am);
1088 if (am > VTD_MAMV) {
1089 VTD_DPRINTF(GENERAL, "error: supported max address mask value is "
1090 "%"PRIu8, (uint8_t)VTD_MAMV);
1091 iaig = 0;
1092 break;
1093 }
1da12ec4 1094 iaig = VTD_TLB_PSI_FLUSH_A;
b5a280c0 1095 vtd_iotlb_page_invalidate(s, domain_id, addr, am);
1da12ec4
LT		 1096 break;
1097
1098 default:
b5a280c0 1099 VTD_DPRINTF(GENERAL, "error: invalid granularity");
1da12ec4
LT		 1100 iaig = 0;
1101 }
1102 return iaig;
1103}
1104
ed7b8fbc
LT		 1105static inline bool vtd_queued_inv_enable_check(IntelIOMMUState *s)
1106{
1107 return s->iq_tail == 0;
1108}
1109
1110static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s)
1111{
1112 return s->qi_enabled && (s->iq_tail == s->iq_head) &&
1113 (s->iq_last_desc_type == VTD_INV_DESC_WAIT);
1114}
1115
1116static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en)
1117{
1118 uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG);
1119
1120 VTD_DPRINTF(INV, "Queued Invalidation Enable %s", (en ? "on" : "off"));
1121 if (en) {
1122 if (vtd_queued_inv_enable_check(s)) {
1123 s->iq = iqa_val & VTD_IQA_IQA_MASK;
1124 /* 2^(x+8) entries */
1125 s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8);
1126 s->qi_enabled = true;
1127 VTD_DPRINTF(INV, "DMAR_IQA_REG 0x%"PRIx64, iqa_val);
1128 VTD_DPRINTF(INV, "Invalidation Queue addr 0x%"PRIx64 " size %d",
1129 s->iq, s->iq_size);
1130 /* Ok - report back to driver */
1131 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES);
1132 } else {
1133 VTD_DPRINTF(GENERAL, "error: can't enable Queued Invalidation: "
1134 "tail %"PRIu16, s->iq_tail);
1135 }
1136 } else {
1137 if (vtd_queued_inv_disable_check(s)) {
1138 /* disable Queued Invalidation */
1139 vtd_set_quad_raw(s, DMAR_IQH_REG, 0);
1140 s->iq_head = 0;
1141 s->qi_enabled = false;
1142 /* Ok - report back to driver */
1143 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0);
1144 } else {
1145 VTD_DPRINTF(GENERAL, "error: can't disable Queued Invalidation: "
1146 "head %"PRIu16 ", tail %"PRIu16
1147 ", last_descriptor %"PRIu8,
1148 s->iq_head, s->iq_tail, s->iq_last_desc_type);
1149 }
1150 }
1151}
1152
1da12ec4
LT		 1153/* Set Root Table Pointer */
1154static void vtd_handle_gcmd_srtp(IntelIOMMUState *s)
1155{
1156 VTD_DPRINTF(CSR, "set Root Table Pointer");
1157
1158 vtd_root_table_setup(s);
1159 /* Ok - report back to driver */
1160 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS);
1161}
1162
a5861439
PX		 1163/* Set Interrupt Remap Table Pointer */
1164static void vtd_handle_gcmd_sirtp(IntelIOMMUState *s)
1165{
1166 VTD_DPRINTF(CSR, "set Interrupt Remap Table Pointer");
1167
1168 vtd_interrupt_remap_table_setup(s);
1169 /* Ok - report back to driver */
1170 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRTPS);
1171}
1172
1da12ec4
LT		 1173/* Handle Translation Enable/Disable */
1174static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en)
1175{
1176 VTD_DPRINTF(CSR, "Translation Enable %s", (en ? "on" : "off"));
1177
1178 if (en) {
1179 s->dmar_enabled = true;
1180 /* Ok - report back to driver */
1181 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES);
1182 } else {
1183 s->dmar_enabled = false;
1184
1185 /* Clear the index of Fault Recording Register */
1186 s->next_frcd_reg = 0;
1187 /* Ok - report back to driver */
1188 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0);
1189 }
1190}
1191
80de52ba
PX		 1192/* Handle Interrupt Remap Enable/Disable */
1193static void vtd_handle_gcmd_ire(IntelIOMMUState *s, bool en)
1194{
1195 VTD_DPRINTF(CSR, "Interrupt Remap Enable %s", (en ? "on" : "off"));
1196
1197 if (en) {
1198 s->intr_enabled = true;
1199 /* Ok - report back to driver */
1200 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRES);
1201 } else {
1202 s->intr_enabled = false;
1203 /* Ok - report back to driver */
1204 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_IRES, 0);
1205 }
1206}
1207
1da12ec4
LT		 1208/* Handle write to Global Command Register */
1209static void vtd_handle_gcmd_write(IntelIOMMUState *s)
1210{
1211 uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG);
1212 uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG);
1213 uint32_t changed = status ^ val;
1214
1215 VTD_DPRINTF(CSR, "value 0x%"PRIx32 " status 0x%"PRIx32, val, status);
1216 if (changed & VTD_GCMD_TE) {
1217 /* Translation enable/disable */
1218 vtd_handle_gcmd_te(s, val & VTD_GCMD_TE);
1219 }
1220 if (val & VTD_GCMD_SRTP) {
1221 /* Set/update the root-table pointer */
1222 vtd_handle_gcmd_srtp(s);
1223 }
ed7b8fbc
LT		 1224 if (changed & VTD_GCMD_QIE) {
1225 /* Queued Invalidation Enable */
1226 vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE);
1227 }
a5861439
PX		 1228 if (val & VTD_GCMD_SIRTP) {
1229 /* Set/update the interrupt remapping root-table pointer */
1230 vtd_handle_gcmd_sirtp(s);
1231 }
80de52ba
PX		 1232 if (changed & VTD_GCMD_IRE) {
1233 /* Interrupt remap enable/disable */
1234 vtd_handle_gcmd_ire(s, val & VTD_GCMD_IRE);
1235 }
1da12ec4
LT		 1236}
1237
1238/* Handle write to Context Command Register */
1239static void vtd_handle_ccmd_write(IntelIOMMUState *s)
1240{
1241 uint64_t ret;
1242 uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG);
1243
1244 /* Context-cache invalidation request */
1245 if (val & VTD_CCMD_ICC) {
ed7b8fbc
LT		 1246 if (s->qi_enabled) {
1247 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, "
1248 "should not use register-based invalidation");
1249 return;
1250 }
1da12ec4
LT		 1251 ret = vtd_context_cache_invalidate(s, val);
1252 /* Invalidation completed. Change something to show */
1253 vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL);
1254 ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK,
1255 ret);
1256 VTD_DPRINTF(INV, "CCMD_REG write-back val: 0x%"PRIx64, ret);
1257 }
1258}
1259
1260/* Handle write to IOTLB Invalidation Register */
1261static void vtd_handle_iotlb_write(IntelIOMMUState *s)
1262{
1263 uint64_t ret;
1264 uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG);
1265
1266 /* IOTLB invalidation request */
1267 if (val & VTD_TLB_IVT) {
ed7b8fbc
LT		 1268 if (s->qi_enabled) {
1269 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, "
1270 "should not use register-based invalidation");
1271 return;
1272 }
1da12ec4
LT		 1273 ret = vtd_iotlb_flush(s, val);
1274 /* Invalidation completed. Change something to show */
1275 vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL);
1276 ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG,
1277 VTD_TLB_FLUSH_GRANU_MASK_A, ret);
1278 VTD_DPRINTF(INV, "IOTLB_REG write-back val: 0x%"PRIx64, ret);
1279 }
1280}
1281
ed7b8fbc
LT		 1282/* Fetch an Invalidation Descriptor from the Invalidation Queue */
1283static bool vtd_get_inv_desc(dma_addr_t base_addr, uint32_t offset,
1284 VTDInvDesc *inv_desc)
1285{
1286 dma_addr_t addr = base_addr + offset * sizeof(*inv_desc);
1287 if (dma_memory_read(&address_space_memory, addr, inv_desc,
1288 sizeof(*inv_desc))) {
1289 VTD_DPRINTF(GENERAL, "error: fail to fetch Invalidation Descriptor "
1290 "base_addr 0x%"PRIx64 " offset %"PRIu32, base_addr, offset);
1291 inv_desc->lo = 0;
1292 inv_desc->hi = 0;
1293
1294 return false;
1295 }
1296 inv_desc->lo = le64_to_cpu(inv_desc->lo);
1297 inv_desc->hi = le64_to_cpu(inv_desc->hi);
1298 return true;
1299}
1300
1301static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
1302{
1303 if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) ||
1304 (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) {
1305 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Invalidation "
1306 "Wait Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1307 inv_desc->hi, inv_desc->lo);
1308 return false;
1309 }
1310 if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) {
1311 /* Status Write */
1312 uint32_t status_data = (uint32_t)(inv_desc->lo >>
1313 VTD_INV_DESC_WAIT_DATA_SHIFT);
1314
1315 assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF));
1316
1317 /* FIXME: need to be masked with HAW? */
1318 dma_addr_t status_addr = inv_desc->hi;
1319 VTD_DPRINTF(INV, "status data 0x%x, status addr 0x%"PRIx64,
1320 status_data, status_addr);
1321 status_data = cpu_to_le32(status_data);
1322 if (dma_memory_write(&address_space_memory, status_addr, &status_data,
1323 sizeof(status_data))) {
1324 VTD_DPRINTF(GENERAL, "error: fail to perform a coherent write");
1325 return false;
1326 }
1327 } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) {
1328 /* Interrupt flag */
1329 VTD_DPRINTF(INV, "Invalidation Wait Descriptor interrupt completion");
1330 vtd_generate_completion_event(s);
1331 } else {
1332 VTD_DPRINTF(GENERAL, "error: invalid Invalidation Wait Descriptor: "
1333 "hi 0x%"PRIx64 " lo 0x%"PRIx64, inv_desc->hi, inv_desc->lo);
1334 return false;
1335 }
1336 return true;
1337}
1338
d92fa2dc
LT		 1339static bool vtd_process_context_cache_desc(IntelIOMMUState *s,
1340 VTDInvDesc *inv_desc)
1341{
1342 if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) {
1343 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Context-cache "
1344 "Invalidate Descriptor");
1345 return false;
1346 }
1347 switch (inv_desc->lo & VTD_INV_DESC_CC_G) {
1348 case VTD_INV_DESC_CC_DOMAIN:
1349 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
1350 (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo));
1351 /* Fall through */
1352 case VTD_INV_DESC_CC_GLOBAL:
1353 VTD_DPRINTF(INV, "global invalidation");
1354 vtd_context_global_invalidate(s);
1355 break;
1356
1357 case VTD_INV_DESC_CC_DEVICE:
1358 vtd_context_device_invalidate(s, VTD_INV_DESC_CC_SID(inv_desc->lo),
1359 VTD_INV_DESC_CC_FM(inv_desc->lo));
1360 break;
1361
1362 default:
1363 VTD_DPRINTF(GENERAL, "error: invalid granularity in Context-cache "
1364 "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1365 inv_desc->hi, inv_desc->lo);
1366 return false;
1367 }
1368 return true;
1369}
1370
b5a280c0
LT		 1371static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
1372{
1373 uint16_t domain_id;
1374 uint8_t am;
1375 hwaddr addr;
1376
1377 if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) ||
1378 (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) {
1379 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in IOTLB "
1380 "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1381 inv_desc->hi, inv_desc->lo);
1382 return false;
1383 }
1384
1385 switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) {
1386 case VTD_INV_DESC_IOTLB_GLOBAL:
1387 VTD_DPRINTF(INV, "global invalidation");
1388 vtd_iotlb_global_invalidate(s);
1389 break;
1390
1391 case VTD_INV_DESC_IOTLB_DOMAIN:
1392 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
1393 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
1394 domain_id);
1395 vtd_iotlb_domain_invalidate(s, domain_id);
1396 break;
1397
1398 case VTD_INV_DESC_IOTLB_PAGE:
1399 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
1400 addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi);
1401 am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi);
1402 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16
1403 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am);
1404 if (am > VTD_MAMV) {
1405 VTD_DPRINTF(GENERAL, "error: supported max address mask value is "
1406 "%"PRIu8, (uint8_t)VTD_MAMV);
1407 return false;
1408 }
1409 vtd_iotlb_page_invalidate(s, domain_id, addr, am);
1410 break;
1411
1412 default:
1413 VTD_DPRINTF(GENERAL, "error: invalid granularity in IOTLB Invalidate "
1414 "Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1415 inv_desc->hi, inv_desc->lo);
1416 return false;
1417 }
1418 return true;
1419}
1420
02a2cbc8
PX		 1421static bool vtd_process_inv_iec_desc(IntelIOMMUState *s,
1422 VTDInvDesc *inv_desc)
1423{
1424 VTD_DPRINTF(INV, "inv ir glob %d index %d mask %d",
1425 inv_desc->iec.granularity,
1426 inv_desc->iec.index,
1427 inv_desc->iec.index_mask);
1428
1429 vtd_iec_notify_all(s, !inv_desc->iec.granularity,
1430 inv_desc->iec.index,
1431 inv_desc->iec.index_mask);
554f5e16
JW		 1432 return true;
1433}
1434
1435static bool vtd_process_device_iotlb_desc(IntelIOMMUState *s,
1436 VTDInvDesc *inv_desc)
1437{
1438 VTDAddressSpace *vtd_dev_as;
1439 IOMMUTLBEntry entry;
1440 struct VTDBus *vtd_bus;
1441 hwaddr addr;
1442 uint64_t sz;
1443 uint16_t sid;
1444 uint8_t devfn;
1445 bool size;
1446 uint8_t bus_num;
1447
1448 addr = VTD_INV_DESC_DEVICE_IOTLB_ADDR(inv_desc->hi);
1449 sid = VTD_INV_DESC_DEVICE_IOTLB_SID(inv_desc->lo);
1450 devfn = sid & 0xff;
1451 bus_num = sid >> 8;
1452 size = VTD_INV_DESC_DEVICE_IOTLB_SIZE(inv_desc->hi);
1453
1454 if ((inv_desc->lo & VTD_INV_DESC_DEVICE_IOTLB_RSVD_LO) ||
1455 (inv_desc->hi & VTD_INV_DESC_DEVICE_IOTLB_RSVD_HI)) {
1456 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Device "
1457 "IOTLB Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1458 inv_desc->hi, inv_desc->lo);
1459 return false;
1460 }
1461
1462 vtd_bus = vtd_find_as_from_bus_num(s, bus_num);
1463 if (!vtd_bus) {
1464 goto done;
1465 }
1466
1467 vtd_dev_as = vtd_bus->dev_as[devfn];
1468 if (!vtd_dev_as) {
1469 goto done;
1470 }
1471
04eb6247
JW		 1472 /* According to ATS spec table 2.4:
1473 * S = 0, bits 15:12 = xxxx range size: 4K
1474 * S = 1, bits 15:12 = xxx0 range size: 8K
1475 * S = 1, bits 15:12 = xx01 range size: 16K
1476 * S = 1, bits 15:12 = x011 range size: 32K
1477 * S = 1, bits 15:12 = 0111 range size: 64K
1478 * ...
1479 */
554f5e16 1480 if (size) {
04eb6247 1481 sz = (VTD_PAGE_SIZE * 2) << cto64(addr >> VTD_PAGE_SHIFT);
554f5e16
JW		 1482 addr &= ~(sz - 1);
1483 } else {
1484 sz = VTD_PAGE_SIZE;
1485 }
02a2cbc8 1486
554f5e16
JW		 1487 entry.target_as = &vtd_dev_as->as;
1488 entry.addr_mask = sz - 1;
1489 entry.iova = addr;
1490 entry.perm = IOMMU_NONE;
1491 entry.translated_addr = 0;
1492 memory_region_notify_iommu(entry.target_as->root, entry);
1493
1494done:
02a2cbc8
PX		 1495 return true;
1496}
1497
ed7b8fbc
LT		 1498static bool vtd_process_inv_desc(IntelIOMMUState *s)
1499{
1500 VTDInvDesc inv_desc;
1501 uint8_t desc_type;
1502
1503 VTD_DPRINTF(INV, "iq head %"PRIu16, s->iq_head);
1504 if (!vtd_get_inv_desc(s->iq, s->iq_head, &inv_desc)) {
1505 s->iq_last_desc_type = VTD_INV_DESC_NONE;
1506 return false;
1507 }
1508 desc_type = inv_desc.lo & VTD_INV_DESC_TYPE;
1509 /* FIXME: should update at first or at last? */
1510 s->iq_last_desc_type = desc_type;
1511
1512 switch (desc_type) {
1513 case VTD_INV_DESC_CC:
1514 VTD_DPRINTF(INV, "Context-cache Invalidate Descriptor hi 0x%"PRIx64
1515 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
d92fa2dc
LT		 1516 if (!vtd_process_context_cache_desc(s, &inv_desc)) {
1517 return false;
1518 }
ed7b8fbc
LT		 1519 break;
1520
1521 case VTD_INV_DESC_IOTLB:
1522 VTD_DPRINTF(INV, "IOTLB Invalidate Descriptor hi 0x%"PRIx64
1523 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
b5a280c0
LT		 1524 if (!vtd_process_iotlb_desc(s, &inv_desc)) {
1525 return false;
1526 }
ed7b8fbc
LT		 1527 break;
1528
1529 case VTD_INV_DESC_WAIT:
1530 VTD_DPRINTF(INV, "Invalidation Wait Descriptor hi 0x%"PRIx64
1531 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
1532 if (!vtd_process_wait_desc(s, &inv_desc)) {
1533 return false;
1534 }
1535 break;
1536
b7910472 1537 case VTD_INV_DESC_IEC:
02a2cbc8
PX		 1538 VTD_DPRINTF(INV, "Invalidation Interrupt Entry Cache "
1539 "Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1540 inv_desc.hi, inv_desc.lo);
1541 if (!vtd_process_inv_iec_desc(s, &inv_desc)) {
1542 return false;
1543 }
b7910472
PX		 1544 break;
1545
554f5e16
JW		 1546 case VTD_INV_DESC_DEVICE:
1547 VTD_DPRINTF(INV, "Device IOTLB Invalidation Descriptor hi 0x%"PRIx64
1548 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
1549 if (!vtd_process_device_iotlb_desc(s, &inv_desc)) {
1550 return false;
1551 }
1552 break;
1553
ed7b8fbc
LT		 1554 default:
1555 VTD_DPRINTF(GENERAL, "error: unkonw Invalidation Descriptor type "
1556 "hi 0x%"PRIx64 " lo 0x%"PRIx64 " type %"PRIu8,
1557 inv_desc.hi, inv_desc.lo, desc_type);
1558 return false;
1559 }
1560 s->iq_head++;
1561 if (s->iq_head == s->iq_size) {
1562 s->iq_head = 0;
1563 }
1564 return true;
1565}
1566
1567/* Try to fetch and process more Invalidation Descriptors */
1568static void vtd_fetch_inv_desc(IntelIOMMUState *s)
1569{
1570 VTD_DPRINTF(INV, "fetch Invalidation Descriptors");
1571 if (s->iq_tail >= s->iq_size) {
1572 /* Detects an invalid Tail pointer */
1573 VTD_DPRINTF(GENERAL, "error: iq_tail is %"PRIu16
1574 " while iq_size is %"PRIu16, s->iq_tail, s->iq_size);
1575 vtd_handle_inv_queue_error(s);
1576 return;
1577 }
1578 while (s->iq_head != s->iq_tail) {
1579 if (!vtd_process_inv_desc(s)) {
1580 /* Invalidation Queue Errors */
1581 vtd_handle_inv_queue_error(s);
1582 break;
1583 }
1584 /* Must update the IQH_REG in time */
1585 vtd_set_quad_raw(s, DMAR_IQH_REG,
1586 (((uint64_t)(s->iq_head)) << VTD_IQH_QH_SHIFT) &
1587 VTD_IQH_QH_MASK);
1588 }
1589}
1590
1591/* Handle write to Invalidation Queue Tail Register */
1592static void vtd_handle_iqt_write(IntelIOMMUState *s)
1593{
1594 uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG);
1595
1596 s->iq_tail = VTD_IQT_QT(val);
1597 VTD_DPRINTF(INV, "set iq tail %"PRIu16, s->iq_tail);
1598 if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) {
1599 /* Process Invalidation Queue here */
1600 vtd_fetch_inv_desc(s);
1601 }
1602}
1603
1da12ec4
LT		 1604static void vtd_handle_fsts_write(IntelIOMMUState *s)
1605{
1606 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
1607 uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
1608 uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE;
1609
1610 if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) {
1611 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
1612 VTD_DPRINTF(FLOG, "all pending interrupt conditions serviced, clear "
1613 "IP field of FECTL_REG");
1614 }
ed7b8fbc
LT		 1615 /* FIXME: when IQE is Clear, should we try to fetch some Invalidation
1616 * Descriptors if there are any when Queued Invalidation is enabled?
1617 */
1da12ec4
LT		 1618}
1619
1620static void vtd_handle_fectl_write(IntelIOMMUState *s)
1621{
1622 uint32_t fectl_reg;
1623 /* FIXME: when software clears the IM field, check the IP field. But do we
1624 * need to compare the old value and the new value to conclude that
1625 * software clears the IM field? Or just check if the IM field is zero?
1626 */
1627 fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
1628 if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) {
1629 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
1630 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
1631 VTD_DPRINTF(FLOG, "IM field is cleared, generate "
1632 "fault event interrupt");
1633 }
1634}
1635
ed7b8fbc
LT		 1636static void vtd_handle_ics_write(IntelIOMMUState *s)
1637{
1638 uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG);
1639 uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
1640
1641 if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) {
1642 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
1643 VTD_DPRINTF(INV, "pending completion interrupt condition serviced, "
1644 "clear IP field of IECTL_REG");
1645 }
1646}
1647
1648static void vtd_handle_iectl_write(IntelIOMMUState *s)
1649{
1650 uint32_t iectl_reg;
1651 /* FIXME: when software clears the IM field, check the IP field. But do we
1652 * need to compare the old value and the new value to conclude that
1653 * software clears the IM field? Or just check if the IM field is zero?
1654 */
1655 iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
1656 if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) {
1657 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
1658 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
1659 VTD_DPRINTF(INV, "IM field is cleared, generate "
1660 "invalidation event interrupt");
1661 }
1662}
1663
1da12ec4
LT		 1664static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size)
1665{
1666 IntelIOMMUState *s = opaque;
1667 uint64_t val;
1668
1669 if (addr + size > DMAR_REG_SIZE) {
1670 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64
1671 ", got 0x%"PRIx64 " %d",
1672 (uint64_t)DMAR_REG_SIZE, addr, size);
1673 return (uint64_t)-1;
1674 }
1675
1676 switch (addr) {
1677 /* Root Table Address Register, 64-bit */
1678 case DMAR_RTADDR_REG:
1679 if (size == 4) {
1680 val = s->root & ((1ULL << 32) - 1);
1681 } else {
1682 val = s->root;
1683 }
1684 break;
1685
1686 case DMAR_RTADDR_REG_HI:
1687 assert(size == 4);
1688 val = s->root >> 32;
1689 break;
1690
ed7b8fbc
LT		 1691 /* Invalidation Queue Address Register, 64-bit */
1692 case DMAR_IQA_REG:
1693 val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS);
1694 if (size == 4) {
1695 val = val & ((1ULL << 32) - 1);
1696 }
1697 break;
1698
1699 case DMAR_IQA_REG_HI:
1700 assert(size == 4);
1701 val = s->iq >> 32;
1702 break;
1703
1da12ec4
LT		 1704 default:
1705 if (size == 4) {
1706 val = vtd_get_long(s, addr);
1707 } else {
1708 val = vtd_get_quad(s, addr);
1709 }
1710 }
1711 VTD_DPRINTF(CSR, "addr 0x%"PRIx64 " size %d val 0x%"PRIx64,
1712 addr, size, val);
1713 return val;
1714}
1715
1716static void vtd_mem_write(void *opaque, hwaddr addr,
1717 uint64_t val, unsigned size)
1718{
1719 IntelIOMMUState *s = opaque;
1720
1721 if (addr + size > DMAR_REG_SIZE) {
1722 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64
1723 ", got 0x%"PRIx64 " %d",
1724 (uint64_t)DMAR_REG_SIZE, addr, size);
1725 return;
1726 }
1727
1728 switch (addr) {
1729 /* Global Command Register, 32-bit */
1730 case DMAR_GCMD_REG:
1731 VTD_DPRINTF(CSR, "DMAR_GCMD_REG write addr 0x%"PRIx64
1732 ", size %d, val 0x%"PRIx64, addr, size, val);
1733 vtd_set_long(s, addr, val);
1734 vtd_handle_gcmd_write(s);
1735 break;
1736
1737 /* Context Command Register, 64-bit */
1738 case DMAR_CCMD_REG:
1739 VTD_DPRINTF(CSR, "DMAR_CCMD_REG write addr 0x%"PRIx64
1740 ", size %d, val 0x%"PRIx64, addr, size, val);
1741 if (size == 4) {
1742 vtd_set_long(s, addr, val);
1743 } else {
1744 vtd_set_quad(s, addr, val);
1745 vtd_handle_ccmd_write(s);
1746 }
1747 break;
1748
1749 case DMAR_CCMD_REG_HI:
1750 VTD_DPRINTF(CSR, "DMAR_CCMD_REG_HI write addr 0x%"PRIx64
1751 ", size %d, val 0x%"PRIx64, addr, size, val);
1752 assert(size == 4);
1753 vtd_set_long(s, addr, val);
1754 vtd_handle_ccmd_write(s);
1755 break;
1756
1757 /* IOTLB Invalidation Register, 64-bit */
1758 case DMAR_IOTLB_REG:
1759 VTD_DPRINTF(INV, "DMAR_IOTLB_REG write addr 0x%"PRIx64
1760 ", size %d, val 0x%"PRIx64, addr, size, val);
1761 if (size == 4) {
1762 vtd_set_long(s, addr, val);
1763 } else {
1764 vtd_set_quad(s, addr, val);
1765 vtd_handle_iotlb_write(s);
1766 }
1767 break;
1768
1769 case DMAR_IOTLB_REG_HI:
1770 VTD_DPRINTF(INV, "DMAR_IOTLB_REG_HI write addr 0x%"PRIx64
1771 ", size %d, val 0x%"PRIx64, addr, size, val);
1772 assert(size == 4);
1773 vtd_set_long(s, addr, val);
1774 vtd_handle_iotlb_write(s);
1775 break;
1776
b5a280c0
LT		 1777 /* Invalidate Address Register, 64-bit */
1778 case DMAR_IVA_REG:
1779 VTD_DPRINTF(INV, "DMAR_IVA_REG write addr 0x%"PRIx64
1780 ", size %d, val 0x%"PRIx64, addr, size, val);
1781 if (size == 4) {
1782 vtd_set_long(s, addr, val);
1783 } else {
1784 vtd_set_quad(s, addr, val);
1785 }
1786 break;
1787
1788 case DMAR_IVA_REG_HI:
1789 VTD_DPRINTF(INV, "DMAR_IVA_REG_HI write addr 0x%"PRIx64
1790 ", size %d, val 0x%"PRIx64, addr, size, val);
1791 assert(size == 4);
1792 vtd_set_long(s, addr, val);
1793 break;
1794
1da12ec4
LT		 1795 /* Fault Status Register, 32-bit */
1796 case DMAR_FSTS_REG:
1797 VTD_DPRINTF(FLOG, "DMAR_FSTS_REG write addr 0x%"PRIx64
1798 ", size %d, val 0x%"PRIx64, addr, size, val);
1799 assert(size == 4);
1800 vtd_set_long(s, addr, val);
1801 vtd_handle_fsts_write(s);
1802 break;
1803
1804 /* Fault Event Control Register, 32-bit */
1805 case DMAR_FECTL_REG:
1806 VTD_DPRINTF(FLOG, "DMAR_FECTL_REG write addr 0x%"PRIx64
1807 ", size %d, val 0x%"PRIx64, addr, size, val);
1808 assert(size == 4);
1809 vtd_set_long(s, addr, val);
1810 vtd_handle_fectl_write(s);
1811 break;
1812
1813 /* Fault Event Data Register, 32-bit */
1814 case DMAR_FEDATA_REG:
1815 VTD_DPRINTF(FLOG, "DMAR_FEDATA_REG write addr 0x%"PRIx64
1816 ", size %d, val 0x%"PRIx64, addr, size, val);
1817 assert(size == 4);
1818 vtd_set_long(s, addr, val);
1819 break;
1820
1821 /* Fault Event Address Register, 32-bit */
1822 case DMAR_FEADDR_REG:
1823 VTD_DPRINTF(FLOG, "DMAR_FEADDR_REG write addr 0x%"PRIx64
1824 ", size %d, val 0x%"PRIx64, addr, size, val);
1825 assert(size == 4);
1826 vtd_set_long(s, addr, val);
1827 break;
1828
1829 /* Fault Event Upper Address Register, 32-bit */
1830 case DMAR_FEUADDR_REG:
1831 VTD_DPRINTF(FLOG, "DMAR_FEUADDR_REG write addr 0x%"PRIx64
1832 ", size %d, val 0x%"PRIx64, addr, size, val);
1833 assert(size == 4);
1834 vtd_set_long(s, addr, val);
1835 break;
1836
1837 /* Protected Memory Enable Register, 32-bit */
1838 case DMAR_PMEN_REG:
1839 VTD_DPRINTF(CSR, "DMAR_PMEN_REG write addr 0x%"PRIx64
1840 ", size %d, val 0x%"PRIx64, addr, size, val);
1841 assert(size == 4);
1842 vtd_set_long(s, addr, val);
1843 break;
1844
1845 /* Root Table Address Register, 64-bit */
1846 case DMAR_RTADDR_REG:
1847 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG write addr 0x%"PRIx64
1848 ", size %d, val 0x%"PRIx64, addr, size, val);
1849 if (size == 4) {
1850 vtd_set_long(s, addr, val);
1851 } else {
1852 vtd_set_quad(s, addr, val);
1853 }
1854 break;
1855
1856 case DMAR_RTADDR_REG_HI:
1857 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG_HI write addr 0x%"PRIx64
1858 ", size %d, val 0x%"PRIx64, addr, size, val);
1859 assert(size == 4);
1860 vtd_set_long(s, addr, val);
1861 break;
1862
ed7b8fbc
LT		 1863 /* Invalidation Queue Tail Register, 64-bit */
1864 case DMAR_IQT_REG:
1865 VTD_DPRINTF(INV, "DMAR_IQT_REG write addr 0x%"PRIx64
1866 ", size %d, val 0x%"PRIx64, addr, size, val);
1867 if (size == 4) {
1868 vtd_set_long(s, addr, val);
1869 } else {
1870 vtd_set_quad(s, addr, val);
1871 }
1872 vtd_handle_iqt_write(s);
1873 break;
1874
1875 case DMAR_IQT_REG_HI:
1876 VTD_DPRINTF(INV, "DMAR_IQT_REG_HI write addr 0x%"PRIx64
1877 ", size %d, val 0x%"PRIx64, addr, size, val);
1878 assert(size == 4);
1879 vtd_set_long(s, addr, val);
1880 /* 19:63 of IQT_REG is RsvdZ, do nothing here */
1881 break;
1882
1883 /* Invalidation Queue Address Register, 64-bit */
1884 case DMAR_IQA_REG:
1885 VTD_DPRINTF(INV, "DMAR_IQA_REG write addr 0x%"PRIx64
1886 ", size %d, val 0x%"PRIx64, addr, size, val);
1887 if (size == 4) {
1888 vtd_set_long(s, addr, val);
1889 } else {
1890 vtd_set_quad(s, addr, val);
1891 }
1892 break;
1893
1894 case DMAR_IQA_REG_HI:
1895 VTD_DPRINTF(INV, "DMAR_IQA_REG_HI write addr 0x%"PRIx64
1896 ", size %d, val 0x%"PRIx64, addr, size, val);
1897 assert(size == 4);
1898 vtd_set_long(s, addr, val);
1899 break;
1900
1901 /* Invalidation Completion Status Register, 32-bit */
1902 case DMAR_ICS_REG:
1903 VTD_DPRINTF(INV, "DMAR_ICS_REG write addr 0x%"PRIx64
1904 ", size %d, val 0x%"PRIx64, addr, size, val);
1905 assert(size == 4);
1906 vtd_set_long(s, addr, val);
1907 vtd_handle_ics_write(s);
1908 break;
1909
1910 /* Invalidation Event Control Register, 32-bit */
1911 case DMAR_IECTL_REG:
1912 VTD_DPRINTF(INV, "DMAR_IECTL_REG write addr 0x%"PRIx64
1913 ", size %d, val 0x%"PRIx64, addr, size, val);
1914 assert(size == 4);
1915 vtd_set_long(s, addr, val);
1916 vtd_handle_iectl_write(s);
1917 break;
1918
1919 /* Invalidation Event Data Register, 32-bit */
1920 case DMAR_IEDATA_REG:
1921 VTD_DPRINTF(INV, "DMAR_IEDATA_REG write addr 0x%"PRIx64
1922 ", size %d, val 0x%"PRIx64, addr, size, val);
1923 assert(size == 4);
1924 vtd_set_long(s, addr, val);
1925 break;
1926
1927 /* Invalidation Event Address Register, 32-bit */
1928 case DMAR_IEADDR_REG:
1929 VTD_DPRINTF(INV, "DMAR_IEADDR_REG write addr 0x%"PRIx64
1930 ", size %d, val 0x%"PRIx64, addr, size, val);
1931 assert(size == 4);
1932 vtd_set_long(s, addr, val);
1933 break;
1934
1935 /* Invalidation Event Upper Address Register, 32-bit */
1936 case DMAR_IEUADDR_REG:
1937 VTD_DPRINTF(INV, "DMAR_IEUADDR_REG write addr 0x%"PRIx64
1938 ", size %d, val 0x%"PRIx64, addr, size, val);
1939 assert(size == 4);
1940 vtd_set_long(s, addr, val);
1941 break;
1942
1da12ec4
LT		 1943 /* Fault Recording Registers, 128-bit */
1944 case DMAR_FRCD_REG_0_0:
1945 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_0 write addr 0x%"PRIx64
1946 ", size %d, val 0x%"PRIx64, addr, size, val);
1947 if (size == 4) {
1948 vtd_set_long(s, addr, val);
1949 } else {
1950 vtd_set_quad(s, addr, val);
1951 }
1952 break;
1953
1954 case DMAR_FRCD_REG_0_1:
1955 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_1 write addr 0x%"PRIx64
1956 ", size %d, val 0x%"PRIx64, addr, size, val);
1957 assert(size == 4);
1958 vtd_set_long(s, addr, val);
1959 break;
1960
1961 case DMAR_FRCD_REG_0_2:
1962 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_2 write addr 0x%"PRIx64
1963 ", size %d, val 0x%"PRIx64, addr, size, val);
1964 if (size == 4) {
1965 vtd_set_long(s, addr, val);
1966 } else {
1967 vtd_set_quad(s, addr, val);
1968 /* May clear bit 127 (Fault), update PPF */
1969 vtd_update_fsts_ppf(s);
1970 }
1971 break;
1972
1973 case DMAR_FRCD_REG_0_3:
1974 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_3 write addr 0x%"PRIx64
1975 ", size %d, val 0x%"PRIx64, addr, size, val);
1976 assert(size == 4);
1977 vtd_set_long(s, addr, val);
1978 /* May clear bit 127 (Fault), update PPF */
1979 vtd_update_fsts_ppf(s);
1980 break;
1981
a5861439
PX		 1982 case DMAR_IRTA_REG:
1983 VTD_DPRINTF(IR, "DMAR_IRTA_REG write addr 0x%"PRIx64
1984 ", size %d, val 0x%"PRIx64, addr, size, val);
1985 if (size == 4) {
1986 vtd_set_long(s, addr, val);
1987 } else {
1988 vtd_set_quad(s, addr, val);
1989 }
1990 break;
1991
1992 case DMAR_IRTA_REG_HI:
1993 VTD_DPRINTF(IR, "DMAR_IRTA_REG_HI write addr 0x%"PRIx64
1994 ", size %d, val 0x%"PRIx64, addr, size, val);
1995 assert(size == 4);
1996 vtd_set_long(s, addr, val);
1997 break;
1998
1da12ec4
LT		 1999 default:
2000 VTD_DPRINTF(GENERAL, "error: unhandled reg write addr 0x%"PRIx64
2001 ", size %d, val 0x%"PRIx64, addr, size, val);
2002 if (size == 4) {
2003 vtd_set_long(s, addr, val);
2004 } else {
2005 vtd_set_quad(s, addr, val);
2006 }
2007 }
2008}
2009
2010static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr,
2011 bool is_write)
2012{
2013 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
2014 IntelIOMMUState *s = vtd_as->iommu_state;
1da12ec4
LT		 2015 IOMMUTLBEntry ret = {
2016 .target_as = &address_space_memory,
2017 .iova = addr,
2018 .translated_addr = 0,
2019 .addr_mask = ~(hwaddr)0,
2020 .perm = IOMMU_NONE,
2021 };
2022
2023 if (!s->dmar_enabled) {
2024 /* DMAR disabled, passthrough, use 4k-page*/
2025 ret.iova = addr & VTD_PAGE_MASK_4K;
2026 ret.translated_addr = addr & VTD_PAGE_MASK_4K;
2027 ret.addr_mask = ~VTD_PAGE_MASK_4K;
2028 ret.perm = IOMMU_RW;
2029 return ret;
2030 }
2031
7df953bd 2032 vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, addr,
d92fa2dc 2033 is_write, &ret);
1da12ec4
LT		 2034 VTD_DPRINTF(MMU,
2035 "bus %"PRIu8 " slot %"PRIu8 " func %"PRIu8 " devfn %"PRIu8
6e905564 2036 " iova 0x%"PRIx64 " hpa 0x%"PRIx64, pci_bus_num(vtd_as->bus),
d92fa2dc
LT		 2037 VTD_PCI_SLOT(vtd_as->devfn), VTD_PCI_FUNC(vtd_as->devfn),
2038 vtd_as->devfn, addr, ret.translated_addr);
1da12ec4
LT		 2039 return ret;
2040}
2041
5bf3d319
PX		 2042static void vtd_iommu_notify_flag_changed(MemoryRegion *iommu,
2043 IOMMUNotifierFlag old,
2044 IOMMUNotifierFlag new)
3cb3b154
AW		 2045{
2046 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
2047
a3276f78
PX		 2048 if (new & IOMMU_NOTIFIER_MAP) {
2049 error_report("Device at bus %s addr %02x.%d requires iommu "
2050 "notifier which is currently not supported by "
2051 "intel-iommu emulation",
2052 vtd_as->bus->qbus.name, PCI_SLOT(vtd_as->devfn),
2053 PCI_FUNC(vtd_as->devfn));
2054 exit(1);
2055 }
3cb3b154
AW		 2056}
2057
1da12ec4
LT		 2058static const VMStateDescription vtd_vmstate = {
2059 .name = "iommu-intel",
8cdcf3c1
PX		 2060 .version_id = 1,
2061 .minimum_version_id = 1,
2062 .priority = MIG_PRI_IOMMU,
2063 .fields = (VMStateField[]) {
2064 VMSTATE_UINT64(root, IntelIOMMUState),
2065 VMSTATE_UINT64(intr_root, IntelIOMMUState),
2066 VMSTATE_UINT64(iq, IntelIOMMUState),
2067 VMSTATE_UINT32(intr_size, IntelIOMMUState),
2068 VMSTATE_UINT16(iq_head, IntelIOMMUState),
2069 VMSTATE_UINT16(iq_tail, IntelIOMMUState),
2070 VMSTATE_UINT16(iq_size, IntelIOMMUState),
2071 VMSTATE_UINT16(next_frcd_reg, IntelIOMMUState),
2072 VMSTATE_UINT8_ARRAY(csr, IntelIOMMUState, DMAR_REG_SIZE),
2073 VMSTATE_UINT8(iq_last_desc_type, IntelIOMMUState),
2074 VMSTATE_BOOL(root_extended, IntelIOMMUState),
2075 VMSTATE_BOOL(dmar_enabled, IntelIOMMUState),
2076 VMSTATE_BOOL(qi_enabled, IntelIOMMUState),
2077 VMSTATE_BOOL(intr_enabled, IntelIOMMUState),
2078 VMSTATE_BOOL(intr_eime, IntelIOMMUState),
2079 VMSTATE_END_OF_LIST()
2080 }
1da12ec4
LT		 2081};
2082
2083static const MemoryRegionOps vtd_mem_ops = {
2084 .read = vtd_mem_read,
2085 .write = vtd_mem_write,
2086 .endianness = DEVICE_LITTLE_ENDIAN,
2087 .impl = {
2088 .min_access_size = 4,
2089 .max_access_size = 8,
2090 },
2091 .valid = {
2092 .min_access_size = 4,
2093 .max_access_size = 8,
2094 },
2095};
2096
2097static Property vtd_properties[] = {
2098 DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0),
e6b6af05
RK		 2099 DEFINE_PROP_ON_OFF_AUTO("eim", IntelIOMMUState, intr_eim,
2100 ON_OFF_AUTO_AUTO),
fb506e70 2101 DEFINE_PROP_BOOL("x-buggy-eim", IntelIOMMUState, buggy_eim, false),
3b40f0e5 2102 DEFINE_PROP_BOOL("caching-mode", IntelIOMMUState, caching_mode, FALSE),
1da12ec4
LT		 2103 DEFINE_PROP_END_OF_LIST(),
2104};
2105
651e4cef
PX		 2106/* Read IRTE entry with specific index */
2107static int vtd_irte_get(IntelIOMMUState *iommu, uint16_t index,
bc38ee10 2108 VTD_IR_TableEntry *entry, uint16_t sid)
651e4cef 2109{
ede9c94a
PX		 2110 static const uint16_t vtd_svt_mask[VTD_SQ_MAX] = \
2111 {0xffff, 0xfffb, 0xfff9, 0xfff8};
651e4cef 2112 dma_addr_t addr = 0x00;
ede9c94a
PX		 2113 uint16_t mask, source_id;
2114 uint8_t bus, bus_max, bus_min;
651e4cef
PX		 2115
2116 addr = iommu->intr_root + index * sizeof(*entry);
2117 if (dma_memory_read(&address_space_memory, addr, entry,
2118 sizeof(*entry))) {
2119 VTD_DPRINTF(GENERAL, "error: fail to access IR root at 0x%"PRIx64
2120 " + %"PRIu16, iommu->intr_root, index);
2121 return -VTD_FR_IR_ROOT_INVAL;
2122 }
2123
bc38ee10 2124 if (!entry->irte.present) {
651e4cef
PX		 2125 VTD_DPRINTF(GENERAL, "error: present flag not set in IRTE"
2126 " entry index %u value 0x%"PRIx64 " 0x%"PRIx64,
2127 index, le64_to_cpu(entry->data[1]),
2128 le64_to_cpu(entry->data[0]));
2129 return -VTD_FR_IR_ENTRY_P;
2130 }
2131
bc38ee10
MT		 2132 if (entry->irte.__reserved_0 || entry->irte.__reserved_1 ||
2133 entry->irte.__reserved_2) {
651e4cef
PX		 2134 VTD_DPRINTF(GENERAL, "error: IRTE entry index %"PRIu16
2135 " reserved fields non-zero: 0x%"PRIx64 " 0x%"PRIx64,
2136 index, le64_to_cpu(entry->data[1]),
2137 le64_to_cpu(entry->data[0]));
2138 return -VTD_FR_IR_IRTE_RSVD;
2139 }
2140
ede9c94a
PX		 2141 if (sid != X86_IOMMU_SID_INVALID) {
2142 /* Validate IRTE SID */
bc38ee10
MT		 2143 source_id = le32_to_cpu(entry->irte.source_id);
2144 switch (entry->irte.sid_vtype) {
ede9c94a
PX		 2145 case VTD_SVT_NONE:
2146 VTD_DPRINTF(IR, "No SID validation for IRTE index %d", index);
2147 break;
2148
2149 case VTD_SVT_ALL:
bc38ee10 2150 mask = vtd_svt_mask[entry->irte.sid_q];
ede9c94a
PX		 2151 if ((source_id & mask) != (sid & mask)) {
2152 VTD_DPRINTF(GENERAL, "SID validation for IRTE index "
2153 "%d failed (reqid 0x%04x sid 0x%04x)", index,
2154 sid, source_id);
2155 return -VTD_FR_IR_SID_ERR;
2156 }
2157 break;
2158
2159 case VTD_SVT_BUS:
2160 bus_max = source_id >> 8;
2161 bus_min = source_id & 0xff;
2162 bus = sid >> 8;
2163 if (bus > bus_max || bus < bus_min) {
2164 VTD_DPRINTF(GENERAL, "SID validation for IRTE index %d "
2165 "failed (bus %d outside %d-%d)", index, bus,
2166 bus_min, bus_max);
2167 return -VTD_FR_IR_SID_ERR;
2168 }
2169 break;
2170
2171 default:
2172 VTD_DPRINTF(GENERAL, "Invalid SVT bits (0x%x) in IRTE index "
bc38ee10 2173 "%d", entry->irte.sid_vtype, index);
ede9c94a
PX		 2174 /* Take this as verification failure. */
2175 return -VTD_FR_IR_SID_ERR;
2176 break;
2177 }
2178 }
651e4cef
PX		 2179
2180 return 0;
2181}
2182
2183/* Fetch IRQ information of specific IR index */
ede9c94a
PX		 2184static int vtd_remap_irq_get(IntelIOMMUState *iommu, uint16_t index,
2185 VTDIrq *irq, uint16_t sid)
651e4cef 2186{
bc38ee10 2187 VTD_IR_TableEntry irte = {};
651e4cef
PX		 2188 int ret = 0;
2189
ede9c94a 2190 ret = vtd_irte_get(iommu, index, &irte, sid);
651e4cef
PX		 2191 if (ret) {
2192 return ret;
2193 }
2194
bc38ee10
MT		 2195 irq->trigger_mode = irte.irte.trigger_mode;
2196 irq->vector = irte.irte.vector;
2197 irq->delivery_mode = irte.irte.delivery_mode;
2198 irq->dest = le32_to_cpu(irte.irte.dest_id);
28589311 2199 if (!iommu->intr_eime) {
651e4cef
PX		 2200#define VTD_IR_APIC_DEST_MASK (0xff00ULL)
2201#define VTD_IR_APIC_DEST_SHIFT (8)
28589311
JK		 2202 irq->dest = (irq->dest & VTD_IR_APIC_DEST_MASK) >>
2203 VTD_IR_APIC_DEST_SHIFT;
2204 }
bc38ee10
MT		 2205 irq->dest_mode = irte.irte.dest_mode;
2206 irq->redir_hint = irte.irte.redir_hint;
651e4cef
PX		 2207
2208 VTD_DPRINTF(IR, "remapping interrupt index %d: trig:%u,vec:%u,"
2209 "deliver:%u,dest:%u,dest_mode:%u", index,
2210 irq->trigger_mode, irq->vector, irq->delivery_mode,
2211 irq->dest, irq->dest_mode);
2212
2213 return 0;
2214}
2215
2216/* Generate one MSI message from VTDIrq info */
2217static void vtd_generate_msi_message(VTDIrq *irq, MSIMessage *msg_out)
2218{
2219 VTD_MSIMessage msg = {};
2220
2221 /* Generate address bits */
2222 msg.dest_mode = irq->dest_mode;
2223 msg.redir_hint = irq->redir_hint;
2224 msg.dest = irq->dest;
32946019 2225 msg.__addr_hi = irq->dest & 0xffffff00;
651e4cef
PX		 2226 msg.__addr_head = cpu_to_le32(0xfee);
2227 /* Keep this from original MSI address bits */
2228 msg.__not_used = irq->msi_addr_last_bits;
2229
2230 /* Generate data bits */
2231 msg.vector = irq->vector;
2232 msg.delivery_mode = irq->delivery_mode;
2233 msg.level = 1;
2234 msg.trigger_mode = irq->trigger_mode;
2235
2236 msg_out->address = msg.msi_addr;
2237 msg_out->data = msg.msi_data;
2238}
2239
2240/* Interrupt remapping for MSI/MSI-X entry */
2241static int vtd_interrupt_remap_msi(IntelIOMMUState *iommu,
2242 MSIMessage *origin,
ede9c94a
PX		 2243 MSIMessage *translated,
2244 uint16_t sid)
651e4cef
PX		 2245{
2246 int ret = 0;
2247 VTD_IR_MSIAddress addr;
2248 uint16_t index;
09cd058a 2249 VTDIrq irq = {};
651e4cef
PX		 2250
2251 assert(origin && translated);
2252
2253 if (!iommu || !iommu->intr_enabled) {
2254 goto do_not_translate;
2255 }
2256
2257 if (origin->address & VTD_MSI_ADDR_HI_MASK) {
2258 VTD_DPRINTF(GENERAL, "error: MSI addr high 32 bits nonzero"
2259 " during interrupt remapping: 0x%"PRIx32,
2260 (uint32_t)((origin->address & VTD_MSI_ADDR_HI_MASK) >> \
2261 VTD_MSI_ADDR_HI_SHIFT));
2262 return -VTD_FR_IR_REQ_RSVD;
2263 }
2264
2265 addr.data = origin->address & VTD_MSI_ADDR_LO_MASK;
1a43713b 2266 if (addr.addr.__head != 0xfee) {
651e4cef
PX		 2267 VTD_DPRINTF(GENERAL, "error: MSI addr low 32 bits invalid: "
2268 "0x%"PRIx32, addr.data);
2269 return -VTD_FR_IR_REQ_RSVD;
2270 }
2271
2272 /* This is compatible mode. */
bc38ee10 2273 if (addr.addr.int_mode != VTD_IR_INT_FORMAT_REMAP) {
651e4cef
PX		 2274 goto do_not_translate;
2275 }
2276
bc38ee10 2277 index = addr.addr.index_h << 15 | le16_to_cpu(addr.addr.index_l);
651e4cef
PX		 2278
2279#define VTD_IR_MSI_DATA_SUBHANDLE (0x0000ffff)
2280#define VTD_IR_MSI_DATA_RESERVED (0xffff0000)
2281
bc38ee10 2282 if (addr.addr.sub_valid) {
651e4cef
PX		 2283 /* See VT-d spec 5.1.2.2 and 5.1.3 on subhandle */
2284 index += origin->data & VTD_IR_MSI_DATA_SUBHANDLE;
2285 }
2286
ede9c94a 2287 ret = vtd_remap_irq_get(iommu, index, &irq, sid);
651e4cef
PX		 2288 if (ret) {
2289 return ret;
2290 }
2291
bc38ee10 2292 if (addr.addr.sub_valid) {
651e4cef
PX		 2293 VTD_DPRINTF(IR, "received MSI interrupt");
2294 if (origin->data & VTD_IR_MSI_DATA_RESERVED) {
2295 VTD_DPRINTF(GENERAL, "error: MSI data bits non-zero for "
2296 "interrupt remappable entry: 0x%"PRIx32,
2297 origin->data);
2298 return -VTD_FR_IR_REQ_RSVD;
2299 }
2300 } else {
2301 uint8_t vector = origin->data & 0xff;
dea651a9
FW		 2302 uint8_t trigger_mode = (origin->data >> MSI_DATA_TRIGGER_SHIFT) & 0x1;
2303
651e4cef
PX		 2304 VTD_DPRINTF(IR, "received IOAPIC interrupt");
2305 /* IOAPIC entry vector should be aligned with IRTE vector
2306 * (see vt-d spec 5.1.5.1). */
2307 if (vector != irq.vector) {
2308 VTD_DPRINTF(GENERAL, "IOAPIC vector inconsistent: "
2309 "entry: %d, IRTE: %d, index: %d",
2310 vector, irq.vector, index);
2311 }
dea651a9
FW		 2312
2313 /* The Trigger Mode field must match the Trigger Mode in the IRTE.
2314 * (see vt-d spec 5.1.5.1). */
2315 if (trigger_mode != irq.trigger_mode) {
2316 VTD_DPRINTF(GENERAL, "IOAPIC trigger mode inconsistent: "
2317 "entry: %u, IRTE: %u, index: %d",
2318 trigger_mode, irq.trigger_mode, index);
2319 }
2320
651e4cef
PX		 2321 }
2322
2323 /*
2324 * We'd better keep the last two bits, assuming that guest OS
2325 * might modify it. Keep it does not hurt after all.
2326 */
bc38ee10 2327 irq.msi_addr_last_bits = addr.addr.__not_care;
651e4cef
PX		 2328
2329 /* Translate VTDIrq to MSI message */
2330 vtd_generate_msi_message(&irq, translated);
2331
2332 VTD_DPRINTF(IR, "mapping MSI 0x%"PRIx64":0x%"PRIx32 " -> "
2333 "0x%"PRIx64":0x%"PRIx32, origin->address, origin->data,
2334 translated->address, translated->data);
2335 return 0;
2336
2337do_not_translate:
2338 memcpy(translated, origin, sizeof(*origin));
2339 return 0;
2340}
2341
8b5ed7df
PX		 2342static int vtd_int_remap(X86IOMMUState *iommu, MSIMessage *src,
2343 MSIMessage *dst, uint16_t sid)
2344{
ede9c94a
PX		 2345 return vtd_interrupt_remap_msi(INTEL_IOMMU_DEVICE(iommu),
2346 src, dst, sid);
8b5ed7df
PX		 2347}
2348
651e4cef
PX		 2349static MemTxResult vtd_mem_ir_read(void *opaque, hwaddr addr,
2350 uint64_t *data, unsigned size,
2351 MemTxAttrs attrs)
2352{
2353 return MEMTX_OK;
2354}
2355
2356static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr,
2357 uint64_t value, unsigned size,
2358 MemTxAttrs attrs)
2359{
2360 int ret = 0;
09cd058a 2361 MSIMessage from = {}, to = {};
ede9c94a 2362 uint16_t sid = X86_IOMMU_SID_INVALID;
651e4cef
PX		 2363
2364 from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST;
2365 from.data = (uint32_t) value;
2366
ede9c94a
PX		 2367 if (!attrs.unspecified) {
2368 /* We have explicit Source ID */
2369 sid = attrs.requester_id;
2370 }
2371
2372 ret = vtd_interrupt_remap_msi(opaque, &from, &to, sid);
651e4cef
PX		 2373 if (ret) {
2374 /* TODO: report error */
2375 VTD_DPRINTF(GENERAL, "int remap fail for addr 0x%"PRIx64
2376 " data 0x%"PRIx32, from.address, from.data);
2377 /* Drop this interrupt */
2378 return MEMTX_ERROR;
2379 }
2380
2381 VTD_DPRINTF(IR, "delivering MSI 0x%"PRIx64":0x%"PRIx32
2382 " for device sid 0x%04x",
2383 to.address, to.data, sid);
2384
32946019 2385 apic_get_class()->send_msi(&to);
651e4cef
PX		 2386
2387 return MEMTX_OK;
2388}
2389
2390static const MemoryRegionOps vtd_mem_ir_ops = {
2391 .read_with_attrs = vtd_mem_ir_read,
2392 .write_with_attrs = vtd_mem_ir_write,
2393 .endianness = DEVICE_LITTLE_ENDIAN,
2394 .impl = {
2395 .min_access_size = 4,
2396 .max_access_size = 4,
2397 },
2398 .valid = {
2399 .min_access_size = 4,
2400 .max_access_size = 4,
2401 },
2402};
7df953bd
KO		 2403
2404VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn)
2405{
2406 uintptr_t key = (uintptr_t)bus;
2407 VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key);
2408 VTDAddressSpace *vtd_dev_as;
e0a3c8cc 2409 char name[128];
7df953bd
KO		 2410
2411 if (!vtd_bus) {
2d3fc581
JW		 2412 uintptr_t *new_key = g_malloc(sizeof(*new_key));
2413 *new_key = (uintptr_t)bus;
7df953bd 2414 /* No corresponding free() */
04af0e18
PX		 2415 vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * \
2416 X86_IOMMU_PCI_DEVFN_MAX);
7df953bd 2417 vtd_bus->bus = bus;
2d3fc581 2418 g_hash_table_insert(s->vtd_as_by_busptr, new_key, vtd_bus);
7df953bd
KO		 2419 }
2420
2421 vtd_dev_as = vtd_bus->dev_as[devfn];
2422
2423 if (!vtd_dev_as) {
e0a3c8cc 2424 snprintf(name, sizeof(name), "intel_iommu_devfn_%d", devfn);
7df953bd
KO		 2425 vtd_bus->dev_as[devfn] = vtd_dev_as = g_malloc0(sizeof(VTDAddressSpace));
2426
2427 vtd_dev_as->bus = bus;
2428 vtd_dev_as->devfn = (uint8_t)devfn;
2429 vtd_dev_as->iommu_state = s;
2430 vtd_dev_as->context_cache_entry.context_cache_gen = 0;
2431 memory_region_init_iommu(&vtd_dev_as->iommu, OBJECT(s),
2432 &s->iommu_ops, "intel_iommu", UINT64_MAX);
651e4cef
PX		 2433 memory_region_init_io(&vtd_dev_as->iommu_ir, OBJECT(s),
2434 &vtd_mem_ir_ops, s, "intel_iommu_ir",
2435 VTD_INTERRUPT_ADDR_SIZE);
2436 memory_region_add_subregion(&vtd_dev_as->iommu, VTD_INTERRUPT_ADDR_FIRST,
2437 &vtd_dev_as->iommu_ir);
7df953bd 2438 address_space_init(&vtd_dev_as->as,
e0a3c8cc 2439 &vtd_dev_as->iommu, name);
7df953bd
KO		 2440 }
2441 return vtd_dev_as;
2442}
2443
1da12ec4
LT		 2444/* Do the initialization. It will also be called when reset, so pay
2445 * attention when adding new initialization stuff.
2446 */
2447static void vtd_init(IntelIOMMUState *s)
2448{
d54bd7f8
PX		 2449 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
2450
1da12ec4
LT		 2451 memset(s->csr, 0, DMAR_REG_SIZE);
2452 memset(s->wmask, 0, DMAR_REG_SIZE);
2453 memset(s->w1cmask, 0, DMAR_REG_SIZE);
2454 memset(s->womask, 0, DMAR_REG_SIZE);
2455
2456 s->iommu_ops.translate = vtd_iommu_translate;
5bf3d319 2457 s->iommu_ops.notify_flag_changed = vtd_iommu_notify_flag_changed;
1da12ec4
LT		 2458 s->root = 0;
2459 s->root_extended = false;
2460 s->dmar_enabled = false;
2461 s->iq_head = 0;
2462 s->iq_tail = 0;
2463 s->iq = 0;
2464 s->iq_size = 0;
2465 s->qi_enabled = false;
2466 s->iq_last_desc_type = VTD_INV_DESC_NONE;
2467 s->next_frcd_reg = 0;
2468 s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | VTD_CAP_MGAW |
d66b969b 2469 VTD_CAP_SAGAW | VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS;
ed7b8fbc 2470 s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO;
1da12ec4 2471
d54bd7f8 2472 if (x86_iommu->intr_supported) {
e6b6af05
RK		 2473 s->ecap |= VTD_ECAP_IR | VTD_ECAP_MHMV;
2474 if (s->intr_eim == ON_OFF_AUTO_ON) {
2475 s->ecap |= VTD_ECAP_EIM;
2476 }
2477 assert(s->intr_eim != ON_OFF_AUTO_AUTO);
d54bd7f8
PX		 2478 }
2479
554f5e16
JW		 2480 if (x86_iommu->dt_supported) {
2481 s->ecap |= VTD_ECAP_DT;
2482 }
2483
3b40f0e5
ABD		 2484 if (s->caching_mode) {
2485 s->cap |= VTD_CAP_CM;
2486 }
2487
d92fa2dc 2488 vtd_reset_context_cache(s);
b5a280c0 2489 vtd_reset_iotlb(s);
d92fa2dc 2490
1da12ec4
LT		 2491 /* Define registers with default values and bit semantics */
2492 vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0);
2493 vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0);
2494 vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0);
2495 vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0);
2496 vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL);
2497 vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0);
2498 vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffff000ULL, 0);
2499 vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0);
2500 vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL);
2501
2502 /* Advanced Fault Logging not supported */
2503 vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL);
2504 vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0);
2505 vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0);
2506 vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0);
2507
2508 /* Treated as RsvdZ when EIM in ECAP_REG is not supported
2509 * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0);
2510 */
2511 vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0);
2512
2513 /* Treated as RO for implementations that PLMR and PHMR fields reported
2514 * as Clear in the CAP_REG.
2515 * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0);
2516 */
2517 vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0);
2518
ed7b8fbc
LT		 2519 vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0);
2520 vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0);
2521 vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff007ULL, 0);
2522 vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL);
2523 vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0);
2524 vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0);
2525 vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0);
2526 /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */
2527 vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0);
2528
1da12ec4
LT		 2529 /* IOTLB registers */
2530 vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0);
2531 vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0);
2532 vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL);
2533
2534 /* Fault Recording Registers, 128-bit */
2535 vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0);
2536 vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL);
a5861439
PX		 2537
2538 /*
28589311 2539 * Interrupt remapping registers.
a5861439 2540 */
28589311 2541 vtd_define_quad(s, DMAR_IRTA_REG, 0, 0xfffffffffffff80fULL, 0);
1da12ec4
LT		 2542}
2543
2544/* Should not reset address_spaces when reset because devices will still use
2545 * the address space they got at first (won't ask the bus again).
2546 */
2547static void vtd_reset(DeviceState *dev)
2548{
2549 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
2550
2551 VTD_DPRINTF(GENERAL, "");
2552 vtd_init(s);
2553}
2554
621d983a
MA		 2555static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
2556{
2557 IntelIOMMUState *s = opaque;
2558 VTDAddressSpace *vtd_as;
2559
8e7a0a16 2560 assert(0 <= devfn && devfn < X86_IOMMU_PCI_DEVFN_MAX);
621d983a
MA		 2561
2562 vtd_as = vtd_find_add_as(s, bus, devfn);
2563 return &vtd_as->as;
2564}
2565
e6b6af05 2566static bool vtd_decide_config(IntelIOMMUState *s, Error **errp)
6333e93c 2567{
e6b6af05
RK		 2568 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
2569
6333e93c
RK		 2570 /* Currently Intel IOMMU IR only support "kernel-irqchip={off|split}" */
2571 if (x86_iommu->intr_supported && kvm_irqchip_in_kernel() &&
2572 !kvm_irqchip_is_split()) {
2573 error_setg(errp, "Intel Interrupt Remapping cannot work with "
2574 "kernel-irqchip=on, please use 'split|off'.");
2575 return false;
2576 }
e6b6af05
RK		 2577 if (s->intr_eim == ON_OFF_AUTO_ON && !x86_iommu->intr_supported) {
2578 error_setg(errp, "eim=on cannot be selected without intremap=on");
2579 return false;
2580 }
2581
2582 if (s->intr_eim == ON_OFF_AUTO_AUTO) {
fb506e70
RK		 2583 s->intr_eim = (kvm_irqchip_in_kernel() || s->buggy_eim)
2584 && x86_iommu->intr_supported ?
e6b6af05
RK		 2585 ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
2586 }
fb506e70
RK		 2587 if (s->intr_eim == ON_OFF_AUTO_ON && !s->buggy_eim) {
2588 if (!kvm_irqchip_in_kernel()) {
2589 error_setg(errp, "eim=on requires accel=kvm,kernel-irqchip=split");
2590 return false;
2591 }
2592 if (!kvm_enable_x2apic()) {
2593 error_setg(errp, "eim=on requires support on the KVM side"
2594 "(X2APIC_API, first shipped in v4.7)");
2595 return false;
2596 }
2597 }
e6b6af05 2598
6333e93c
RK		 2599 return true;
2600}
2601
1da12ec4
LT		 2602static void vtd_realize(DeviceState *dev, Error **errp)
2603{
cb135f59
PX		 2604 PCMachineState *pcms = PC_MACHINE(qdev_get_machine());
2605 PCIBus *bus = pcms->bus;
1da12ec4 2606 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
4684a204 2607 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(dev);
1da12ec4
LT		 2608
2609 VTD_DPRINTF(GENERAL, "");
fb9f5926 2610 x86_iommu->type = TYPE_INTEL;
6333e93c 2611
e6b6af05 2612 if (!vtd_decide_config(s, errp)) {
6333e93c
RK		 2613 return;
2614 }
2615
7df953bd 2616 memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num));
1da12ec4
LT		 2617 memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s,
2618 "intel_iommu", DMAR_REG_SIZE);
2619 sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem);
b5a280c0
LT		 2620 /* No corresponding destroy */
2621 s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
2622 g_free, g_free);
7df953bd
KO		 2623 s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
2624 g_free, g_free);
1da12ec4 2625 vtd_init(s);
621d983a
MA		 2626 sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, Q35_HOST_BRIDGE_IOMMU_ADDR);
2627 pci_setup_iommu(bus, vtd_host_dma_iommu, dev);
cb135f59
PX		 2628 /* Pseudo address space under root PCI bus. */
2629 pcms->ioapic_as = vtd_host_dma_iommu(bus, s, Q35_PSEUDO_DEVFN_IOAPIC);
1da12ec4
LT		 2630}
2631
2632static void vtd_class_init(ObjectClass *klass, void *data)
2633{
2634 DeviceClass *dc = DEVICE_CLASS(klass);
1c7955c4 2635 X86IOMMUClass *x86_class = X86_IOMMU_CLASS(klass);
1da12ec4
LT		 2636
2637 dc->reset = vtd_reset;
1da12ec4
LT		 2638 dc->vmsd = &vtd_vmstate;
2639 dc->props = vtd_properties;
621d983a 2640 dc->hotpluggable = false;
1c7955c4 2641 x86_class->realize = vtd_realize;
8b5ed7df 2642 x86_class->int_remap = vtd_int_remap;
1da12ec4
LT		 2643}
2644
2645static const TypeInfo vtd_info = {
2646 .name = TYPE_INTEL_IOMMU_DEVICE,
1c7955c4 2647 .parent = TYPE_X86_IOMMU_DEVICE,
1da12ec4
LT		 2648 .instance_size = sizeof(IntelIOMMUState),
2649 .class_init = vtd_class_init,
2650};
2651
2652static void vtd_register_types(void)
2653{
2654 VTD_DPRINTF(GENERAL, "");
2655 type_register_static(&vtd_info);
2656}
2657
2658type_init(vtd_register_types)
QEMU mirror