]> git.proxmox.com Git - mirror_qemu.git/blame - hw/i386/intel_iommu.c
projects / mirror_qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
intel_iommu: redo configuraton check in realize
[mirror_qemu.git] / hw / i386 / intel_iommu.c
CommitLineData
1da12ec4
LT		 1/*
2 * QEMU emulation of an Intel IOMMU (VT-d)
3 * (DMA Remapping device)
4 *
5 * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com>
6 * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, see <http://www.gnu.org/licenses/>.
20 */
21
b6a0aa05 22#include "qemu/osdep.h"
4684a204 23#include "qemu/error-report.h"
6333e93c 24#include "qapi/error.h"
1da12ec4
LT		 25#include "hw/sysbus.h"
26#include "exec/address-spaces.h"
27#include "intel_iommu_internal.h"
7df953bd 28#include "hw/pci/pci.h"
3cb3b154 29#include "hw/pci/pci_bus.h"
621d983a 30#include "hw/i386/pc.h"
dea651a9 31#include "hw/i386/apic-msidef.h"
04af0e18
PX		 32#include "hw/boards.h"
33#include "hw/i386/x86-iommu.h"
cb135f59 34#include "hw/pci-host/q35.h"
4684a204 35#include "sysemu/kvm.h"
32946019 36#include "hw/i386/apic_internal.h"
1da12ec4
LT		 37
38/*#define DEBUG_INTEL_IOMMU*/
39#ifdef DEBUG_INTEL_IOMMU
40enum {
41 DEBUG_GENERAL, DEBUG_CSR, DEBUG_INV, DEBUG_MMU, DEBUG_FLOG,
a5861439 42 DEBUG_CACHE, DEBUG_IR,
1da12ec4
LT		 43};
44#define VTD_DBGBIT(x) (1 << DEBUG_##x)
45static int vtd_dbgflags = VTD_DBGBIT(GENERAL) | VTD_DBGBIT(CSR);
46
47#define VTD_DPRINTF(what, fmt, ...) do { \
48 if (vtd_dbgflags & VTD_DBGBIT(what)) { \
49 fprintf(stderr, "(vtd)%s: " fmt "\n", __func__, \
50 ## __VA_ARGS__); } \
51 } while (0)
52#else
53#define VTD_DPRINTF(what, fmt, ...) do {} while (0)
54#endif
55
56static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val,
57 uint64_t wmask, uint64_t w1cmask)
58{
59 stq_le_p(&s->csr[addr], val);
60 stq_le_p(&s->wmask[addr], wmask);
61 stq_le_p(&s->w1cmask[addr], w1cmask);
62}
63
64static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask)
65{
66 stq_le_p(&s->womask[addr], mask);
67}
68
69static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val,
70 uint32_t wmask, uint32_t w1cmask)
71{
72 stl_le_p(&s->csr[addr], val);
73 stl_le_p(&s->wmask[addr], wmask);
74 stl_le_p(&s->w1cmask[addr], w1cmask);
75}
76
77static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask)
78{
79 stl_le_p(&s->womask[addr], mask);
80}
81
82/* "External" get/set operations */
83static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val)
84{
85 uint64_t oldval = ldq_le_p(&s->csr[addr]);
86 uint64_t wmask = ldq_le_p(&s->wmask[addr]);
87 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]);
88 stq_le_p(&s->csr[addr],
89 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val));
90}
91
92static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val)
93{
94 uint32_t oldval = ldl_le_p(&s->csr[addr]);
95 uint32_t wmask = ldl_le_p(&s->wmask[addr]);
96 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]);
97 stl_le_p(&s->csr[addr],
98 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val));
99}
100
101static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr)
102{
103 uint64_t val = ldq_le_p(&s->csr[addr]);
104 uint64_t womask = ldq_le_p(&s->womask[addr]);
105 return val & ~womask;
106}
107
108static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr)
109{
110 uint32_t val = ldl_le_p(&s->csr[addr]);
111 uint32_t womask = ldl_le_p(&s->womask[addr]);
112 return val & ~womask;
113}
114
115/* "Internal" get/set operations */
116static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr)
117{
118 return ldq_le_p(&s->csr[addr]);
119}
120
121static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr)
122{
123 return ldl_le_p(&s->csr[addr]);
124}
125
126static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val)
127{
128 stq_le_p(&s->csr[addr], val);
129}
130
131static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr,
132 uint32_t clear, uint32_t mask)
133{
134 uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask;
135 stl_le_p(&s->csr[addr], new_val);
136 return new_val;
137}
138
139static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr,
140 uint64_t clear, uint64_t mask)
141{
142 uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask;
143 stq_le_p(&s->csr[addr], new_val);
144 return new_val;
145}
146
b5a280c0
LT		 147/* GHashTable functions */
148static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2)
149{
150 return *((const uint64_t *)v1) == *((const uint64_t *)v2);
151}
152
153static guint vtd_uint64_hash(gconstpointer v)
154{
155 return (guint)*(const uint64_t *)v;
156}
157
158static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value,
159 gpointer user_data)
160{
161 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
162 uint16_t domain_id = *(uint16_t *)user_data;
163 return entry->domain_id == domain_id;
164}
165
d66b969b
JW		 166/* The shift of an addr for a certain level of paging structure */
167static inline uint32_t vtd_slpt_level_shift(uint32_t level)
168{
169 return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS;
170}
171
172static inline uint64_t vtd_slpt_level_page_mask(uint32_t level)
173{
174 return ~((1ULL << vtd_slpt_level_shift(level)) - 1);
175}
176
b5a280c0
LT		 177static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value,
178 gpointer user_data)
179{
180 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value;
181 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data;
d66b969b
JW		 182 uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask;
183 uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K;
b5a280c0 184 return (entry->domain_id == info->domain_id) &&
d66b969b
JW		 185 (((entry->gfn & info->mask) == gfn) ||
186 (entry->gfn == gfn_tlb));
b5a280c0
LT		 187}
188
d92fa2dc
LT		 189/* Reset all the gen of VTDAddressSpace to zero and set the gen of
190 * IntelIOMMUState to 1.
191 */
192static void vtd_reset_context_cache(IntelIOMMUState *s)
193{
d92fa2dc 194 VTDAddressSpace *vtd_as;
7df953bd
KO		 195 VTDBus *vtd_bus;
196 GHashTableIter bus_it;
d92fa2dc
LT		 197 uint32_t devfn_it;
198
7df953bd
KO		 199 g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr);
200
d92fa2dc 201 VTD_DPRINTF(CACHE, "global context_cache_gen=1");
7df953bd 202 while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) {
04af0e18 203 for (devfn_it = 0; devfn_it < X86_IOMMU_PCI_DEVFN_MAX; ++devfn_it) {
7df953bd 204 vtd_as = vtd_bus->dev_as[devfn_it];
d92fa2dc
LT		 205 if (!vtd_as) {
206 continue;
207 }
208 vtd_as->context_cache_entry.context_cache_gen = 0;
209 }
210 }
211 s->context_cache_gen = 1;
212}
213
b5a280c0
LT		 214static void vtd_reset_iotlb(IntelIOMMUState *s)
215{
216 assert(s->iotlb);
217 g_hash_table_remove_all(s->iotlb);
218}
219
d66b969b
JW		 220static uint64_t vtd_get_iotlb_key(uint64_t gfn, uint8_t source_id,
221 uint32_t level)
222{
223 return gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT) |
224 ((uint64_t)(level) << VTD_IOTLB_LVL_SHIFT);
225}
226
227static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level)
228{
229 return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K;
230}
231
b5a280c0
LT		 232static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id,
233 hwaddr addr)
234{
d66b969b 235 VTDIOTLBEntry *entry;
b5a280c0 236 uint64_t key;
d66b969b
JW		 237 int level;
238
239 for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) {
240 key = vtd_get_iotlb_key(vtd_get_iotlb_gfn(addr, level),
241 source_id, level);
242 entry = g_hash_table_lookup(s->iotlb, &key);
243 if (entry) {
244 goto out;
245 }
246 }
b5a280c0 247
d66b969b
JW		 248out:
249 return entry;
b5a280c0
LT		 250}
251
252static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id,
253 uint16_t domain_id, hwaddr addr, uint64_t slpte,
d66b969b
JW		 254 bool read_flags, bool write_flags,
255 uint32_t level)
b5a280c0
LT		 256{
257 VTDIOTLBEntry *entry = g_malloc(sizeof(*entry));
258 uint64_t *key = g_malloc(sizeof(*key));
d66b969b 259 uint64_t gfn = vtd_get_iotlb_gfn(addr, level);
b5a280c0
LT		 260
261 VTD_DPRINTF(CACHE, "update iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64
262 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr, slpte,
263 domain_id);
264 if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) {
265 VTD_DPRINTF(CACHE, "iotlb exceeds size limit, forced to reset");
266 vtd_reset_iotlb(s);
267 }
268
269 entry->gfn = gfn;
270 entry->domain_id = domain_id;
271 entry->slpte = slpte;
272 entry->read_flags = read_flags;
273 entry->write_flags = write_flags;
d66b969b
JW		 274 entry->mask = vtd_slpt_level_page_mask(level);
275 *key = vtd_get_iotlb_key(gfn, source_id, level);
b5a280c0
LT		 276 g_hash_table_replace(s->iotlb, key, entry);
277}
278
1da12ec4
LT		 279/* Given the reg addr of both the message data and address, generate an
280 * interrupt via MSI.
281 */
282static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg,
283 hwaddr mesg_data_reg)
284{
32946019 285 MSIMessage msi;
1da12ec4
LT		 286
287 assert(mesg_data_reg < DMAR_REG_SIZE);
288 assert(mesg_addr_reg < DMAR_REG_SIZE);
289
32946019
RK		 290 msi.address = vtd_get_long_raw(s, mesg_addr_reg);
291 msi.data = vtd_get_long_raw(s, mesg_data_reg);
1da12ec4 292
32946019
RK		 293 VTD_DPRINTF(FLOG, "msi: addr 0x%"PRIx64 " data 0x%"PRIx32,
294 msi.address, msi.data);
295 apic_get_class()->send_msi(&msi);
1da12ec4
LT		 296}
297
298/* Generate a fault event to software via MSI if conditions are met.
299 * Notice that the value of FSTS_REG being passed to it should be the one
300 * before any update.
301 */
302static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts)
303{
304 if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO ||
305 pre_fsts & VTD_FSTS_IQE) {
306 VTD_DPRINTF(FLOG, "there are previous interrupt conditions "
307 "to be serviced by software, fault event is not generated "
308 "(FSTS_REG 0x%"PRIx32 ")", pre_fsts);
309 return;
310 }
311 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP);
312 if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) {
313 VTD_DPRINTF(FLOG, "Interrupt Mask set, fault event is not generated");
314 } else {
315 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
316 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
317 }
318}
319
320/* Check if the Fault (F) field of the Fault Recording Register referenced by
321 * @index is Set.
322 */
323static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index)
324{
325 /* Each reg is 128-bit */
326 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
327 addr += 8; /* Access the high 64-bit half */
328
329 assert(index < DMAR_FRCD_REG_NR);
330
331 return vtd_get_quad_raw(s, addr) & VTD_FRCD_F;
332}
333
334/* Update the PPF field of Fault Status Register.
335 * Should be called whenever change the F field of any fault recording
336 * registers.
337 */
338static void vtd_update_fsts_ppf(IntelIOMMUState *s)
339{
340 uint32_t i;
341 uint32_t ppf_mask = 0;
342
343 for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
344 if (vtd_is_frcd_set(s, i)) {
345 ppf_mask = VTD_FSTS_PPF;
346 break;
347 }
348 }
349 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask);
350 VTD_DPRINTF(FLOG, "set PPF of FSTS_REG to %d", ppf_mask ? 1 : 0);
351}
352
353static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index)
354{
355 /* Each reg is 128-bit */
356 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
357 addr += 8; /* Access the high 64-bit half */
358
359 assert(index < DMAR_FRCD_REG_NR);
360
361 vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F);
362 vtd_update_fsts_ppf(s);
363}
364
365/* Must not update F field now, should be done later */
366static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index,
367 uint16_t source_id, hwaddr addr,
368 VTDFaultReason fault, bool is_write)
369{
370 uint64_t hi = 0, lo;
371 hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4);
372
373 assert(index < DMAR_FRCD_REG_NR);
374
375 lo = VTD_FRCD_FI(addr);
376 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault);
377 if (!is_write) {
378 hi |= VTD_FRCD_T;
379 }
380 vtd_set_quad_raw(s, frcd_reg_addr, lo);
381 vtd_set_quad_raw(s, frcd_reg_addr + 8, hi);
382 VTD_DPRINTF(FLOG, "record to FRCD_REG #%"PRIu16 ": hi 0x%"PRIx64
383 ", lo 0x%"PRIx64, index, hi, lo);
384}
385
386/* Try to collapse multiple pending faults from the same requester */
387static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id)
388{
389 uint32_t i;
390 uint64_t frcd_reg;
391 hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */
392
393 for (i = 0; i < DMAR_FRCD_REG_NR; i++) {
394 frcd_reg = vtd_get_quad_raw(s, addr);
395 VTD_DPRINTF(FLOG, "frcd_reg #%d 0x%"PRIx64, i, frcd_reg);
396 if ((frcd_reg & VTD_FRCD_F) &&
397 ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) {
398 return true;
399 }
400 addr += 16; /* 128-bit for each */
401 }
402 return false;
403}
404
405/* Log and report an DMAR (address translation) fault to software */
406static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id,
407 hwaddr addr, VTDFaultReason fault,
408 bool is_write)
409{
410 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
411
412 assert(fault < VTD_FR_MAX);
413
414 if (fault == VTD_FR_RESERVED_ERR) {
415 /* This is not a normal fault reason case. Drop it. */
416 return;
417 }
418 VTD_DPRINTF(FLOG, "sid 0x%"PRIx16 ", fault %d, addr 0x%"PRIx64
419 ", is_write %d", source_id, fault, addr, is_write);
420 if (fsts_reg & VTD_FSTS_PFO) {
421 VTD_DPRINTF(FLOG, "new fault is not recorded due to "
422 "Primary Fault Overflow");
423 return;
424 }
425 if (vtd_try_collapse_fault(s, source_id)) {
426 VTD_DPRINTF(FLOG, "new fault is not recorded due to "
427 "compression of faults");
428 return;
429 }
430 if (vtd_is_frcd_set(s, s->next_frcd_reg)) {
431 VTD_DPRINTF(FLOG, "Primary Fault Overflow and "
432 "new fault is not recorded, set PFO field");
433 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO);
434 return;
435 }
436
437 vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write);
438
439 if (fsts_reg & VTD_FSTS_PPF) {
440 VTD_DPRINTF(FLOG, "there are pending faults already, "
441 "fault event is not generated");
442 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg);
443 s->next_frcd_reg++;
444 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
445 s->next_frcd_reg = 0;
446 }
447 } else {
448 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK,
449 VTD_FSTS_FRI(s->next_frcd_reg));
450 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */
451 s->next_frcd_reg++;
452 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) {
453 s->next_frcd_reg = 0;
454 }
455 /* This case actually cause the PPF to be Set.
456 * So generate fault event (interrupt).
457 */
458 vtd_generate_fault_event(s, fsts_reg);
459 }
460}
461
ed7b8fbc
LT		 462/* Handle Invalidation Queue Errors of queued invalidation interface error
463 * conditions.
464 */
465static void vtd_handle_inv_queue_error(IntelIOMMUState *s)
466{
467 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
468
469 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE);
470 vtd_generate_fault_event(s, fsts_reg);
471}
472
473/* Set the IWC field and try to generate an invalidation completion interrupt */
474static void vtd_generate_completion_event(IntelIOMMUState *s)
475{
476 VTD_DPRINTF(INV, "completes an invalidation wait command with "
477 "Interrupt Flag");
478 if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) {
479 VTD_DPRINTF(INV, "there is a previous interrupt condition to be "
480 "serviced by software, "
481 "new invalidation event is not generated");
482 return;
483 }
484 vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC);
485 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP);
486 if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) {
487 VTD_DPRINTF(INV, "IM filed in IECTL_REG is set, new invalidation "
488 "event is not generated");
489 return;
490 } else {
491 /* Generate the interrupt event */
492 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
493 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
494 }
495}
496
1da12ec4
LT		 497static inline bool vtd_root_entry_present(VTDRootEntry *root)
498{
499 return root->val & VTD_ROOT_ENTRY_P;
500}
501
502static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index,
503 VTDRootEntry *re)
504{
505 dma_addr_t addr;
506
507 addr = s->root + index * sizeof(*re);
508 if (dma_memory_read(&address_space_memory, addr, re, sizeof(*re))) {
509 VTD_DPRINTF(GENERAL, "error: fail to access root-entry at 0x%"PRIx64
510 " + %"PRIu8, s->root, index);
511 re->val = 0;
512 return -VTD_FR_ROOT_TABLE_INV;
513 }
514 re->val = le64_to_cpu(re->val);
515 return 0;
516}
517
518static inline bool vtd_context_entry_present(VTDContextEntry *context)
519{
520 return context->lo & VTD_CONTEXT_ENTRY_P;
521}
522
523static int vtd_get_context_entry_from_root(VTDRootEntry *root, uint8_t index,
524 VTDContextEntry *ce)
525{
526 dma_addr_t addr;
527
528 if (!vtd_root_entry_present(root)) {
529 VTD_DPRINTF(GENERAL, "error: root-entry is not present");
530 return -VTD_FR_ROOT_ENTRY_P;
531 }
532 addr = (root->val & VTD_ROOT_ENTRY_CTP) + index * sizeof(*ce);
533 if (dma_memory_read(&address_space_memory, addr, ce, sizeof(*ce))) {
534 VTD_DPRINTF(GENERAL, "error: fail to access context-entry at 0x%"PRIx64
535 " + %"PRIu8,
536 (uint64_t)(root->val & VTD_ROOT_ENTRY_CTP), index);
537 return -VTD_FR_CONTEXT_TABLE_INV;
538 }
539 ce->lo = le64_to_cpu(ce->lo);
540 ce->hi = le64_to_cpu(ce->hi);
541 return 0;
542}
543
544static inline dma_addr_t vtd_get_slpt_base_from_context(VTDContextEntry *ce)
545{
546 return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR;
547}
548
1da12ec4
LT		 549static inline uint64_t vtd_get_slpte_addr(uint64_t slpte)
550{
551 return slpte & VTD_SL_PT_BASE_ADDR_MASK;
552}
553
554/* Whether the pte indicates the address of the page frame */
555static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level)
556{
557 return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK);
558}
559
560/* Get the content of a spte located in @base_addr[@index] */
561static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index)
562{
563 uint64_t slpte;
564
565 assert(index < VTD_SL_PT_ENTRY_NR);
566
567 if (dma_memory_read(&address_space_memory,
568 base_addr + index * sizeof(slpte), &slpte,
569 sizeof(slpte))) {
570 slpte = (uint64_t)-1;
571 return slpte;
572 }
573 slpte = le64_to_cpu(slpte);
574 return slpte;
575}
576
577/* Given a gpa and the level of paging structure, return the offset of current
578 * level.
579 */
580static inline uint32_t vtd_gpa_level_offset(uint64_t gpa, uint32_t level)
581{
582 return (gpa >> vtd_slpt_level_shift(level)) &
583 ((1ULL << VTD_SL_LEVEL_BITS) - 1);
584}
585
586/* Check Capability Register to see if the @level of page-table is supported */
587static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level)
588{
589 return VTD_CAP_SAGAW_MASK & s->cap &
590 (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT));
591}
592
593/* Get the page-table level that hardware should use for the second-level
594 * page-table walk from the Address Width field of context-entry.
595 */
596static inline uint32_t vtd_get_level_from_context_entry(VTDContextEntry *ce)
597{
598 return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW);
599}
600
601static inline uint32_t vtd_get_agaw_from_context_entry(VTDContextEntry *ce)
602{
603 return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9;
604}
605
606static const uint64_t vtd_paging_entry_rsvd_field[] = {
607 [0] = ~0ULL,
608 /* For not large page */
609 [1] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
610 [2] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
611 [3] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
612 [4] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
613 /* For large page */
614 [5] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
615 [6] = 0x1ff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
616 [7] = 0x3ffff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
617 [8] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM),
618};
619
620static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level)
621{
622 if (slpte & VTD_SL_PT_PAGE_SIZE_MASK) {
623 /* Maybe large page */
624 return slpte & vtd_paging_entry_rsvd_field[level + 4];
625 } else {
626 return slpte & vtd_paging_entry_rsvd_field[level];
627 }
628}
629
630/* Given the @gpa, get relevant @slptep. @slpte_level will be the last level
631 * of the translation, can be used for deciding the size of large page.
632 */
633static int vtd_gpa_to_slpte(VTDContextEntry *ce, uint64_t gpa, bool is_write,
634 uint64_t *slptep, uint32_t *slpte_level,
635 bool *reads, bool *writes)
636{
637 dma_addr_t addr = vtd_get_slpt_base_from_context(ce);
638 uint32_t level = vtd_get_level_from_context_entry(ce);
639 uint32_t offset;
640 uint64_t slpte;
641 uint32_t ce_agaw = vtd_get_agaw_from_context_entry(ce);
642 uint64_t access_right_check;
643
644 /* Check if @gpa is above 2^X-1, where X is the minimum of MGAW in CAP_REG
645 * and AW in context-entry.
646 */
647 if (gpa & ~((1ULL << MIN(ce_agaw, VTD_MGAW)) - 1)) {
648 VTD_DPRINTF(GENERAL, "error: gpa 0x%"PRIx64 " exceeds limits", gpa);
649 return -VTD_FR_ADDR_BEYOND_MGAW;
650 }
651
652 /* FIXME: what is the Atomics request here? */
653 access_right_check = is_write ? VTD_SL_W : VTD_SL_R;
654
655 while (true) {
656 offset = vtd_gpa_level_offset(gpa, level);
657 slpte = vtd_get_slpte(addr, offset);
658
659 if (slpte == (uint64_t)-1) {
660 VTD_DPRINTF(GENERAL, "error: fail to access second-level paging "
661 "entry at level %"PRIu32 " for gpa 0x%"PRIx64,
662 level, gpa);
663 if (level == vtd_get_level_from_context_entry(ce)) {
664 /* Invalid programming of context-entry */
665 return -VTD_FR_CONTEXT_ENTRY_INV;
666 } else {
667 return -VTD_FR_PAGING_ENTRY_INV;
668 }
669 }
670 *reads = (*reads) && (slpte & VTD_SL_R);
671 *writes = (*writes) && (slpte & VTD_SL_W);
672 if (!(slpte & access_right_check)) {
673 VTD_DPRINTF(GENERAL, "error: lack of %s permission for "
674 "gpa 0x%"PRIx64 " slpte 0x%"PRIx64,
675 (is_write ? "write" : "read"), gpa, slpte);
676 return is_write ? -VTD_FR_WRITE : -VTD_FR_READ;
677 }
678 if (vtd_slpte_nonzero_rsvd(slpte, level)) {
679 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in second "
680 "level paging entry level %"PRIu32 " slpte 0x%"PRIx64,
681 level, slpte);
682 return -VTD_FR_PAGING_ENTRY_RSVD;
683 }
684
685 if (vtd_is_last_slpte(slpte, level)) {
686 *slptep = slpte;
687 *slpte_level = level;
688 return 0;
689 }
690 addr = vtd_get_slpte_addr(slpte);
691 level--;
692 }
693}
694
695/* Map a device to its corresponding domain (context-entry) */
696static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num,
697 uint8_t devfn, VTDContextEntry *ce)
698{
699 VTDRootEntry re;
700 int ret_fr;
701
702 ret_fr = vtd_get_root_entry(s, bus_num, &re);
703 if (ret_fr) {
704 return ret_fr;
705 }
706
707 if (!vtd_root_entry_present(&re)) {
708 VTD_DPRINTF(GENERAL, "error: root-entry #%"PRIu8 " is not present",
709 bus_num);
710 return -VTD_FR_ROOT_ENTRY_P;
711 } else if (re.rsvd || (re.val & VTD_ROOT_ENTRY_RSVD)) {
712 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in root-entry "
713 "hi 0x%"PRIx64 " lo 0x%"PRIx64, re.rsvd, re.val);
714 return -VTD_FR_ROOT_ENTRY_RSVD;
715 }
716
717 ret_fr = vtd_get_context_entry_from_root(&re, devfn, ce);
718 if (ret_fr) {
719 return ret_fr;
720 }
721
722 if (!vtd_context_entry_present(ce)) {
723 VTD_DPRINTF(GENERAL,
724 "error: context-entry #%"PRIu8 "(bus #%"PRIu8 ") "
725 "is not present", devfn, bus_num);
726 return -VTD_FR_CONTEXT_ENTRY_P;
727 } else if ((ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI) ||
728 (ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO)) {
729 VTD_DPRINTF(GENERAL,
730 "error: non-zero reserved field in context-entry "
731 "hi 0x%"PRIx64 " lo 0x%"PRIx64, ce->hi, ce->lo);
732 return -VTD_FR_CONTEXT_ENTRY_RSVD;
733 }
734 /* Check if the programming of context-entry is valid */
735 if (!vtd_is_level_supported(s, vtd_get_level_from_context_entry(ce))) {
736 VTD_DPRINTF(GENERAL, "error: unsupported Address Width value in "
737 "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64,
738 ce->hi, ce->lo);
739 return -VTD_FR_CONTEXT_ENTRY_INV;
740 } else if (ce->lo & VTD_CONTEXT_ENTRY_TT) {
741 VTD_DPRINTF(GENERAL, "error: unsupported Translation Type in "
742 "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64,
743 ce->hi, ce->lo);
744 return -VTD_FR_CONTEXT_ENTRY_INV;
745 }
746 return 0;
747}
748
749static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn)
750{
751 return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL);
752}
753
754static const bool vtd_qualified_faults[] = {
755 [VTD_FR_RESERVED] = false,
756 [VTD_FR_ROOT_ENTRY_P] = false,
757 [VTD_FR_CONTEXT_ENTRY_P] = true,
758 [VTD_FR_CONTEXT_ENTRY_INV] = true,
759 [VTD_FR_ADDR_BEYOND_MGAW] = true,
760 [VTD_FR_WRITE] = true,
761 [VTD_FR_READ] = true,
762 [VTD_FR_PAGING_ENTRY_INV] = true,
763 [VTD_FR_ROOT_TABLE_INV] = false,
764 [VTD_FR_CONTEXT_TABLE_INV] = false,
765 [VTD_FR_ROOT_ENTRY_RSVD] = false,
766 [VTD_FR_PAGING_ENTRY_RSVD] = true,
767 [VTD_FR_CONTEXT_ENTRY_TT] = true,
768 [VTD_FR_RESERVED_ERR] = false,
769 [VTD_FR_MAX] = false,
770};
771
772/* To see if a fault condition is "qualified", which is reported to software
773 * only if the FPD field in the context-entry used to process the faulting
774 * request is 0.
775 */
776static inline bool vtd_is_qualified_fault(VTDFaultReason fault)
777{
778 return vtd_qualified_faults[fault];
779}
780
781static inline bool vtd_is_interrupt_addr(hwaddr addr)
782{
783 return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST;
784}
785
786/* Map dev to context-entry then do a paging-structures walk to do a iommu
787 * translation.
79e2b9ae
PB		 788 *
789 * Called from RCU critical section.
790 *
1da12ec4
LT		 791 * @bus_num: The bus number
792 * @devfn: The devfn, which is the combined of device and function number
793 * @is_write: The access is a write operation
794 * @entry: IOMMUTLBEntry that contain the addr to be translated and result
795 */
7df953bd 796static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus,
1da12ec4
LT		 797 uint8_t devfn, hwaddr addr, bool is_write,
798 IOMMUTLBEntry *entry)
799{
d92fa2dc 800 IntelIOMMUState *s = vtd_as->iommu_state;
1da12ec4 801 VTDContextEntry ce;
7df953bd 802 uint8_t bus_num = pci_bus_num(bus);
d92fa2dc 803 VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry;
d66b969b 804 uint64_t slpte, page_mask;
1da12ec4
LT		 805 uint32_t level;
806 uint16_t source_id = vtd_make_source_id(bus_num, devfn);
807 int ret_fr;
808 bool is_fpd_set = false;
809 bool reads = true;
810 bool writes = true;
b5a280c0 811 VTDIOTLBEntry *iotlb_entry;
1da12ec4
LT		 812
813 /* Check if the request is in interrupt address range */
814 if (vtd_is_interrupt_addr(addr)) {
815 if (is_write) {
816 /* FIXME: since we don't know the length of the access here, we
817 * treat Non-DWORD length write requests without PASID as
818 * interrupt requests, too. Withoud interrupt remapping support,
819 * we just use 1:1 mapping.
820 */
821 VTD_DPRINTF(MMU, "write request to interrupt address "
822 "gpa 0x%"PRIx64, addr);
823 entry->iova = addr & VTD_PAGE_MASK_4K;
824 entry->translated_addr = addr & VTD_PAGE_MASK_4K;
825 entry->addr_mask = ~VTD_PAGE_MASK_4K;
826 entry->perm = IOMMU_WO;
827 return;
828 } else {
829 VTD_DPRINTF(GENERAL, "error: read request from interrupt address "
830 "gpa 0x%"PRIx64, addr);
831 vtd_report_dmar_fault(s, source_id, addr, VTD_FR_READ, is_write);
832 return;
833 }
834 }
b5a280c0
LT		 835 /* Try to fetch slpte form IOTLB */
836 iotlb_entry = vtd_lookup_iotlb(s, source_id, addr);
837 if (iotlb_entry) {
838 VTD_DPRINTF(CACHE, "hit iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64
839 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr,
840 iotlb_entry->slpte, iotlb_entry->domain_id);
841 slpte = iotlb_entry->slpte;
842 reads = iotlb_entry->read_flags;
843 writes = iotlb_entry->write_flags;
d66b969b 844 page_mask = iotlb_entry->mask;
b5a280c0
LT		 845 goto out;
846 }
d92fa2dc
LT		 847 /* Try to fetch context-entry from cache first */
848 if (cc_entry->context_cache_gen == s->context_cache_gen) {
849 VTD_DPRINTF(CACHE, "hit context-cache bus %d devfn %d "
850 "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 ")",
851 bus_num, devfn, cc_entry->context_entry.hi,
852 cc_entry->context_entry.lo, cc_entry->context_cache_gen);
853 ce = cc_entry->context_entry;
854 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
855 } else {
856 ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce);
857 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD;
858 if (ret_fr) {
859 ret_fr = -ret_fr;
860 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {
861 VTD_DPRINTF(FLOG, "fault processing is disabled for DMA "
862 "requests through this context-entry "
863 "(with FPD Set)");
864 } else {
865 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);
866 }
867 return;
1da12ec4 868 }
d92fa2dc
LT		 869 /* Update context-cache */
870 VTD_DPRINTF(CACHE, "update context-cache bus %d devfn %d "
871 "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 "->%"PRIu32 ")",
872 bus_num, devfn, ce.hi, ce.lo,
873 cc_entry->context_cache_gen, s->context_cache_gen);
874 cc_entry->context_entry = ce;
875 cc_entry->context_cache_gen = s->context_cache_gen;
1da12ec4
LT		 876 }
877
878 ret_fr = vtd_gpa_to_slpte(&ce, addr, is_write, &slpte, &level,
879 &reads, &writes);
880 if (ret_fr) {
881 ret_fr = -ret_fr;
882 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) {
883 VTD_DPRINTF(FLOG, "fault processing is disabled for DMA requests "
884 "through this context-entry (with FPD Set)");
885 } else {
886 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write);
887 }
888 return;
889 }
890
d66b969b 891 page_mask = vtd_slpt_level_page_mask(level);
b5a280c0 892 vtd_update_iotlb(s, source_id, VTD_CONTEXT_ENTRY_DID(ce.hi), addr, slpte,
d66b969b 893 reads, writes, level);
b5a280c0 894out:
d66b969b
JW		 895 entry->iova = addr & page_mask;
896 entry->translated_addr = vtd_get_slpte_addr(slpte) & page_mask;
897 entry->addr_mask = ~page_mask;
1da12ec4
LT		 898 entry->perm = (writes ? 2 : 0) + (reads ? 1 : 0);
899}
900
901static void vtd_root_table_setup(IntelIOMMUState *s)
902{
903 s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG);
904 s->root_extended = s->root & VTD_RTADDR_RTT;
905 s->root &= VTD_RTADDR_ADDR_MASK;
906
907 VTD_DPRINTF(CSR, "root_table addr 0x%"PRIx64 " %s", s->root,
908 (s->root_extended ? "(extended)" : ""));
909}
910
02a2cbc8
PX		 911static void vtd_iec_notify_all(IntelIOMMUState *s, bool global,
912 uint32_t index, uint32_t mask)
913{
914 x86_iommu_iec_notify_all(X86_IOMMU_DEVICE(s), global, index, mask);
915}
916
a5861439
PX		 917static void vtd_interrupt_remap_table_setup(IntelIOMMUState *s)
918{
919 uint64_t value = 0;
920 value = vtd_get_quad_raw(s, DMAR_IRTA_REG);
921 s->intr_size = 1UL << ((value & VTD_IRTA_SIZE_MASK) + 1);
922 s->intr_root = value & VTD_IRTA_ADDR_MASK;
28589311 923 s->intr_eime = value & VTD_IRTA_EIME;
a5861439 924
02a2cbc8
PX		 925 /* Notify global invalidation */
926 vtd_iec_notify_all(s, true, 0, 0);
a5861439
PX		 927
928 VTD_DPRINTF(CSR, "int remap table addr 0x%"PRIx64 " size %"PRIu32,
929 s->intr_root, s->intr_size);
930}
931
d92fa2dc
LT		 932static void vtd_context_global_invalidate(IntelIOMMUState *s)
933{
934 s->context_cache_gen++;
935 if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) {
936 vtd_reset_context_cache(s);
937 }
938}
939
7df953bd
KO		 940
941/* Find the VTD address space currently associated with a given bus number,
942 */
943static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num)
944{
945 VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num];
946 if (!vtd_bus) {
947 /* Iterate over the registered buses to find the one
948 * which currently hold this bus number, and update the bus_num lookup table:
949 */
950 GHashTableIter iter;
951
952 g_hash_table_iter_init(&iter, s->vtd_as_by_busptr);
953 while (g_hash_table_iter_next (&iter, NULL, (void**)&vtd_bus)) {
954 if (pci_bus_num(vtd_bus->bus) == bus_num) {
955 s->vtd_as_by_bus_num[bus_num] = vtd_bus;
956 return vtd_bus;
957 }
958 }
959 }
960 return vtd_bus;
961}
962
d92fa2dc
LT		 963/* Do a context-cache device-selective invalidation.
964 * @func_mask: FM field after shifting
965 */
966static void vtd_context_device_invalidate(IntelIOMMUState *s,
967 uint16_t source_id,
968 uint16_t func_mask)
969{
970 uint16_t mask;
7df953bd 971 VTDBus *vtd_bus;
d92fa2dc
LT		 972 VTDAddressSpace *vtd_as;
973 uint16_t devfn;
974 uint16_t devfn_it;
975
976 switch (func_mask & 3) {
977 case 0:
978 mask = 0; /* No bits in the SID field masked */
979 break;
980 case 1:
981 mask = 4; /* Mask bit 2 in the SID field */
982 break;
983 case 2:
984 mask = 6; /* Mask bit 2:1 in the SID field */
985 break;
986 case 3:
987 mask = 7; /* Mask bit 2:0 in the SID field */
988 break;
989 }
990 VTD_DPRINTF(INV, "device-selective invalidation source 0x%"PRIx16
991 " mask %"PRIu16, source_id, mask);
7df953bd
KO		 992 vtd_bus = vtd_find_as_from_bus_num(s, VTD_SID_TO_BUS(source_id));
993 if (vtd_bus) {
d92fa2dc 994 devfn = VTD_SID_TO_DEVFN(source_id);
04af0e18 995 for (devfn_it = 0; devfn_it < X86_IOMMU_PCI_DEVFN_MAX; ++devfn_it) {
7df953bd 996 vtd_as = vtd_bus->dev_as[devfn_it];
d92fa2dc
LT		 997 if (vtd_as && ((devfn_it & mask) == (devfn & mask))) {
998 VTD_DPRINTF(INV, "invalidate context-cahce of devfn 0x%"PRIx16,
999 devfn_it);
1000 vtd_as->context_cache_entry.context_cache_gen = 0;
1001 }
1002 }
1003 }
1004}
1005
1da12ec4
LT		 1006/* Context-cache invalidation
1007 * Returns the Context Actual Invalidation Granularity.
1008 * @val: the content of the CCMD_REG
1009 */
1010static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val)
1011{
1012 uint64_t caig;
1013 uint64_t type = val & VTD_CCMD_CIRG_MASK;
1014
1015 switch (type) {
d92fa2dc
LT		 1016 case VTD_CCMD_DOMAIN_INVL:
1017 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
1018 (uint16_t)VTD_CCMD_DID(val));
1019 /* Fall through */
1da12ec4 1020 case VTD_CCMD_GLOBAL_INVL:
d92fa2dc 1021 VTD_DPRINTF(INV, "global invalidation");
1da12ec4 1022 caig = VTD_CCMD_GLOBAL_INVL_A;
d92fa2dc 1023 vtd_context_global_invalidate(s);
1da12ec4
LT		 1024 break;
1025
1026 case VTD_CCMD_DEVICE_INVL:
1da12ec4 1027 caig = VTD_CCMD_DEVICE_INVL_A;
d92fa2dc 1028 vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val));
1da12ec4
LT		 1029 break;
1030
1031 default:
d92fa2dc 1032 VTD_DPRINTF(GENERAL, "error: invalid granularity");
1da12ec4
LT		 1033 caig = 0;
1034 }
1035 return caig;
1036}
1037
b5a280c0
LT		 1038static void vtd_iotlb_global_invalidate(IntelIOMMUState *s)
1039{
1040 vtd_reset_iotlb(s);
1041}
1042
1043static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id)
1044{
1045 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain,
1046 &domain_id);
1047}
1048
1049static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id,
1050 hwaddr addr, uint8_t am)
1051{
1052 VTDIOTLBPageInvInfo info;
1053
1054 assert(am <= VTD_MAMV);
1055 info.domain_id = domain_id;
d66b969b 1056 info.addr = addr;
b5a280c0
LT		 1057 info.mask = ~((1 << am) - 1);
1058 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info);
1059}
1060
1da12ec4
LT		 1061/* Flush IOTLB
1062 * Returns the IOTLB Actual Invalidation Granularity.
1063 * @val: the content of the IOTLB_REG
1064 */
1065static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val)
1066{
1067 uint64_t iaig;
1068 uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK;
b5a280c0
LT		 1069 uint16_t domain_id;
1070 hwaddr addr;
1071 uint8_t am;
1da12ec4
LT		 1072
1073 switch (type) {
1074 case VTD_TLB_GLOBAL_FLUSH:
b5a280c0 1075 VTD_DPRINTF(INV, "global invalidation");
1da12ec4 1076 iaig = VTD_TLB_GLOBAL_FLUSH_A;
b5a280c0 1077 vtd_iotlb_global_invalidate(s);
1da12ec4
LT		 1078 break;
1079
1080 case VTD_TLB_DSI_FLUSH:
b5a280c0
LT		 1081 domain_id = VTD_TLB_DID(val);
1082 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
1083 domain_id);
1da12ec4 1084 iaig = VTD_TLB_DSI_FLUSH_A;
b5a280c0 1085 vtd_iotlb_domain_invalidate(s, domain_id);
1da12ec4
LT		 1086 break;
1087
1088 case VTD_TLB_PSI_FLUSH:
b5a280c0
LT		 1089 domain_id = VTD_TLB_DID(val);
1090 addr = vtd_get_quad_raw(s, DMAR_IVA_REG);
1091 am = VTD_IVA_AM(addr);
1092 addr = VTD_IVA_ADDR(addr);
1093 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16
1094 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am);
1095 if (am > VTD_MAMV) {
1096 VTD_DPRINTF(GENERAL, "error: supported max address mask value is "
1097 "%"PRIu8, (uint8_t)VTD_MAMV);
1098 iaig = 0;
1099 break;
1100 }
1da12ec4 1101 iaig = VTD_TLB_PSI_FLUSH_A;
b5a280c0 1102 vtd_iotlb_page_invalidate(s, domain_id, addr, am);
1da12ec4
LT		 1103 break;
1104
1105 default:
b5a280c0 1106 VTD_DPRINTF(GENERAL, "error: invalid granularity");
1da12ec4
LT		 1107 iaig = 0;
1108 }
1109 return iaig;
1110}
1111
ed7b8fbc
LT		 1112static inline bool vtd_queued_inv_enable_check(IntelIOMMUState *s)
1113{
1114 return s->iq_tail == 0;
1115}
1116
1117static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s)
1118{
1119 return s->qi_enabled && (s->iq_tail == s->iq_head) &&
1120 (s->iq_last_desc_type == VTD_INV_DESC_WAIT);
1121}
1122
1123static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en)
1124{
1125 uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG);
1126
1127 VTD_DPRINTF(INV, "Queued Invalidation Enable %s", (en ? "on" : "off"));
1128 if (en) {
1129 if (vtd_queued_inv_enable_check(s)) {
1130 s->iq = iqa_val & VTD_IQA_IQA_MASK;
1131 /* 2^(x+8) entries */
1132 s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8);
1133 s->qi_enabled = true;
1134 VTD_DPRINTF(INV, "DMAR_IQA_REG 0x%"PRIx64, iqa_val);
1135 VTD_DPRINTF(INV, "Invalidation Queue addr 0x%"PRIx64 " size %d",
1136 s->iq, s->iq_size);
1137 /* Ok - report back to driver */
1138 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES);
1139 } else {
1140 VTD_DPRINTF(GENERAL, "error: can't enable Queued Invalidation: "
1141 "tail %"PRIu16, s->iq_tail);
1142 }
1143 } else {
1144 if (vtd_queued_inv_disable_check(s)) {
1145 /* disable Queued Invalidation */
1146 vtd_set_quad_raw(s, DMAR_IQH_REG, 0);
1147 s->iq_head = 0;
1148 s->qi_enabled = false;
1149 /* Ok - report back to driver */
1150 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0);
1151 } else {
1152 VTD_DPRINTF(GENERAL, "error: can't disable Queued Invalidation: "
1153 "head %"PRIu16 ", tail %"PRIu16
1154 ", last_descriptor %"PRIu8,
1155 s->iq_head, s->iq_tail, s->iq_last_desc_type);
1156 }
1157 }
1158}
1159
1da12ec4
LT		 1160/* Set Root Table Pointer */
1161static void vtd_handle_gcmd_srtp(IntelIOMMUState *s)
1162{
1163 VTD_DPRINTF(CSR, "set Root Table Pointer");
1164
1165 vtd_root_table_setup(s);
1166 /* Ok - report back to driver */
1167 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS);
1168}
1169
a5861439
PX		 1170/* Set Interrupt Remap Table Pointer */
1171static void vtd_handle_gcmd_sirtp(IntelIOMMUState *s)
1172{
1173 VTD_DPRINTF(CSR, "set Interrupt Remap Table Pointer");
1174
1175 vtd_interrupt_remap_table_setup(s);
1176 /* Ok - report back to driver */
1177 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRTPS);
1178}
1179
1da12ec4
LT		 1180/* Handle Translation Enable/Disable */
1181static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en)
1182{
1183 VTD_DPRINTF(CSR, "Translation Enable %s", (en ? "on" : "off"));
1184
1185 if (en) {
1186 s->dmar_enabled = true;
1187 /* Ok - report back to driver */
1188 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES);
1189 } else {
1190 s->dmar_enabled = false;
1191
1192 /* Clear the index of Fault Recording Register */
1193 s->next_frcd_reg = 0;
1194 /* Ok - report back to driver */
1195 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0);
1196 }
1197}
1198
80de52ba
PX		 1199/* Handle Interrupt Remap Enable/Disable */
1200static void vtd_handle_gcmd_ire(IntelIOMMUState *s, bool en)
1201{
1202 VTD_DPRINTF(CSR, "Interrupt Remap Enable %s", (en ? "on" : "off"));
1203
1204 if (en) {
1205 s->intr_enabled = true;
1206 /* Ok - report back to driver */
1207 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_IRES);
1208 } else {
1209 s->intr_enabled = false;
1210 /* Ok - report back to driver */
1211 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_IRES, 0);
1212 }
1213}
1214
1da12ec4
LT		 1215/* Handle write to Global Command Register */
1216static void vtd_handle_gcmd_write(IntelIOMMUState *s)
1217{
1218 uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG);
1219 uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG);
1220 uint32_t changed = status ^ val;
1221
1222 VTD_DPRINTF(CSR, "value 0x%"PRIx32 " status 0x%"PRIx32, val, status);
1223 if (changed & VTD_GCMD_TE) {
1224 /* Translation enable/disable */
1225 vtd_handle_gcmd_te(s, val & VTD_GCMD_TE);
1226 }
1227 if (val & VTD_GCMD_SRTP) {
1228 /* Set/update the root-table pointer */
1229 vtd_handle_gcmd_srtp(s);
1230 }
ed7b8fbc
LT		 1231 if (changed & VTD_GCMD_QIE) {
1232 /* Queued Invalidation Enable */
1233 vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE);
1234 }
a5861439
PX		 1235 if (val & VTD_GCMD_SIRTP) {
1236 /* Set/update the interrupt remapping root-table pointer */
1237 vtd_handle_gcmd_sirtp(s);
1238 }
80de52ba
PX		 1239 if (changed & VTD_GCMD_IRE) {
1240 /* Interrupt remap enable/disable */
1241 vtd_handle_gcmd_ire(s, val & VTD_GCMD_IRE);
1242 }
1da12ec4
LT		 1243}
1244
1245/* Handle write to Context Command Register */
1246static void vtd_handle_ccmd_write(IntelIOMMUState *s)
1247{
1248 uint64_t ret;
1249 uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG);
1250
1251 /* Context-cache invalidation request */
1252 if (val & VTD_CCMD_ICC) {
ed7b8fbc
LT		 1253 if (s->qi_enabled) {
1254 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, "
1255 "should not use register-based invalidation");
1256 return;
1257 }
1da12ec4
LT		 1258 ret = vtd_context_cache_invalidate(s, val);
1259 /* Invalidation completed. Change something to show */
1260 vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL);
1261 ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK,
1262 ret);
1263 VTD_DPRINTF(INV, "CCMD_REG write-back val: 0x%"PRIx64, ret);
1264 }
1265}
1266
1267/* Handle write to IOTLB Invalidation Register */
1268static void vtd_handle_iotlb_write(IntelIOMMUState *s)
1269{
1270 uint64_t ret;
1271 uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG);
1272
1273 /* IOTLB invalidation request */
1274 if (val & VTD_TLB_IVT) {
ed7b8fbc
LT		 1275 if (s->qi_enabled) {
1276 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, "
1277 "should not use register-based invalidation");
1278 return;
1279 }
1da12ec4
LT		 1280 ret = vtd_iotlb_flush(s, val);
1281 /* Invalidation completed. Change something to show */
1282 vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL);
1283 ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG,
1284 VTD_TLB_FLUSH_GRANU_MASK_A, ret);
1285 VTD_DPRINTF(INV, "IOTLB_REG write-back val: 0x%"PRIx64, ret);
1286 }
1287}
1288
ed7b8fbc
LT		 1289/* Fetch an Invalidation Descriptor from the Invalidation Queue */
1290static bool vtd_get_inv_desc(dma_addr_t base_addr, uint32_t offset,
1291 VTDInvDesc *inv_desc)
1292{
1293 dma_addr_t addr = base_addr + offset * sizeof(*inv_desc);
1294 if (dma_memory_read(&address_space_memory, addr, inv_desc,
1295 sizeof(*inv_desc))) {
1296 VTD_DPRINTF(GENERAL, "error: fail to fetch Invalidation Descriptor "
1297 "base_addr 0x%"PRIx64 " offset %"PRIu32, base_addr, offset);
1298 inv_desc->lo = 0;
1299 inv_desc->hi = 0;
1300
1301 return false;
1302 }
1303 inv_desc->lo = le64_to_cpu(inv_desc->lo);
1304 inv_desc->hi = le64_to_cpu(inv_desc->hi);
1305 return true;
1306}
1307
1308static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
1309{
1310 if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) ||
1311 (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) {
1312 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Invalidation "
1313 "Wait Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1314 inv_desc->hi, inv_desc->lo);
1315 return false;
1316 }
1317 if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) {
1318 /* Status Write */
1319 uint32_t status_data = (uint32_t)(inv_desc->lo >>
1320 VTD_INV_DESC_WAIT_DATA_SHIFT);
1321
1322 assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF));
1323
1324 /* FIXME: need to be masked with HAW? */
1325 dma_addr_t status_addr = inv_desc->hi;
1326 VTD_DPRINTF(INV, "status data 0x%x, status addr 0x%"PRIx64,
1327 status_data, status_addr);
1328 status_data = cpu_to_le32(status_data);
1329 if (dma_memory_write(&address_space_memory, status_addr, &status_data,
1330 sizeof(status_data))) {
1331 VTD_DPRINTF(GENERAL, "error: fail to perform a coherent write");
1332 return false;
1333 }
1334 } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) {
1335 /* Interrupt flag */
1336 VTD_DPRINTF(INV, "Invalidation Wait Descriptor interrupt completion");
1337 vtd_generate_completion_event(s);
1338 } else {
1339 VTD_DPRINTF(GENERAL, "error: invalid Invalidation Wait Descriptor: "
1340 "hi 0x%"PRIx64 " lo 0x%"PRIx64, inv_desc->hi, inv_desc->lo);
1341 return false;
1342 }
1343 return true;
1344}
1345
d92fa2dc
LT		 1346static bool vtd_process_context_cache_desc(IntelIOMMUState *s,
1347 VTDInvDesc *inv_desc)
1348{
1349 if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) {
1350 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Context-cache "
1351 "Invalidate Descriptor");
1352 return false;
1353 }
1354 switch (inv_desc->lo & VTD_INV_DESC_CC_G) {
1355 case VTD_INV_DESC_CC_DOMAIN:
1356 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
1357 (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo));
1358 /* Fall through */
1359 case VTD_INV_DESC_CC_GLOBAL:
1360 VTD_DPRINTF(INV, "global invalidation");
1361 vtd_context_global_invalidate(s);
1362 break;
1363
1364 case VTD_INV_DESC_CC_DEVICE:
1365 vtd_context_device_invalidate(s, VTD_INV_DESC_CC_SID(inv_desc->lo),
1366 VTD_INV_DESC_CC_FM(inv_desc->lo));
1367 break;
1368
1369 default:
1370 VTD_DPRINTF(GENERAL, "error: invalid granularity in Context-cache "
1371 "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1372 inv_desc->hi, inv_desc->lo);
1373 return false;
1374 }
1375 return true;
1376}
1377
b5a280c0
LT		 1378static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc)
1379{
1380 uint16_t domain_id;
1381 uint8_t am;
1382 hwaddr addr;
1383
1384 if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) ||
1385 (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) {
1386 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in IOTLB "
1387 "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1388 inv_desc->hi, inv_desc->lo);
1389 return false;
1390 }
1391
1392 switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) {
1393 case VTD_INV_DESC_IOTLB_GLOBAL:
1394 VTD_DPRINTF(INV, "global invalidation");
1395 vtd_iotlb_global_invalidate(s);
1396 break;
1397
1398 case VTD_INV_DESC_IOTLB_DOMAIN:
1399 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
1400 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16,
1401 domain_id);
1402 vtd_iotlb_domain_invalidate(s, domain_id);
1403 break;
1404
1405 case VTD_INV_DESC_IOTLB_PAGE:
1406 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo);
1407 addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi);
1408 am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi);
1409 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16
1410 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am);
1411 if (am > VTD_MAMV) {
1412 VTD_DPRINTF(GENERAL, "error: supported max address mask value is "
1413 "%"PRIu8, (uint8_t)VTD_MAMV);
1414 return false;
1415 }
1416 vtd_iotlb_page_invalidate(s, domain_id, addr, am);
1417 break;
1418
1419 default:
1420 VTD_DPRINTF(GENERAL, "error: invalid granularity in IOTLB Invalidate "
1421 "Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1422 inv_desc->hi, inv_desc->lo);
1423 return false;
1424 }
1425 return true;
1426}
1427
02a2cbc8
PX		 1428static bool vtd_process_inv_iec_desc(IntelIOMMUState *s,
1429 VTDInvDesc *inv_desc)
1430{
1431 VTD_DPRINTF(INV, "inv ir glob %d index %d mask %d",
1432 inv_desc->iec.granularity,
1433 inv_desc->iec.index,
1434 inv_desc->iec.index_mask);
1435
1436 vtd_iec_notify_all(s, !inv_desc->iec.granularity,
1437 inv_desc->iec.index,
1438 inv_desc->iec.index_mask);
1439
1440 return true;
1441}
1442
ed7b8fbc
LT		 1443static bool vtd_process_inv_desc(IntelIOMMUState *s)
1444{
1445 VTDInvDesc inv_desc;
1446 uint8_t desc_type;
1447
1448 VTD_DPRINTF(INV, "iq head %"PRIu16, s->iq_head);
1449 if (!vtd_get_inv_desc(s->iq, s->iq_head, &inv_desc)) {
1450 s->iq_last_desc_type = VTD_INV_DESC_NONE;
1451 return false;
1452 }
1453 desc_type = inv_desc.lo & VTD_INV_DESC_TYPE;
1454 /* FIXME: should update at first or at last? */
1455 s->iq_last_desc_type = desc_type;
1456
1457 switch (desc_type) {
1458 case VTD_INV_DESC_CC:
1459 VTD_DPRINTF(INV, "Context-cache Invalidate Descriptor hi 0x%"PRIx64
1460 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
d92fa2dc
LT		 1461 if (!vtd_process_context_cache_desc(s, &inv_desc)) {
1462 return false;
1463 }
ed7b8fbc
LT		 1464 break;
1465
1466 case VTD_INV_DESC_IOTLB:
1467 VTD_DPRINTF(INV, "IOTLB Invalidate Descriptor hi 0x%"PRIx64
1468 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
b5a280c0
LT		 1469 if (!vtd_process_iotlb_desc(s, &inv_desc)) {
1470 return false;
1471 }
ed7b8fbc
LT		 1472 break;
1473
1474 case VTD_INV_DESC_WAIT:
1475 VTD_DPRINTF(INV, "Invalidation Wait Descriptor hi 0x%"PRIx64
1476 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo);
1477 if (!vtd_process_wait_desc(s, &inv_desc)) {
1478 return false;
1479 }
1480 break;
1481
b7910472 1482 case VTD_INV_DESC_IEC:
02a2cbc8
PX		 1483 VTD_DPRINTF(INV, "Invalidation Interrupt Entry Cache "
1484 "Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64,
1485 inv_desc.hi, inv_desc.lo);
1486 if (!vtd_process_inv_iec_desc(s, &inv_desc)) {
1487 return false;
1488 }
b7910472
PX		 1489 break;
1490
ed7b8fbc
LT		 1491 default:
1492 VTD_DPRINTF(GENERAL, "error: unkonw Invalidation Descriptor type "
1493 "hi 0x%"PRIx64 " lo 0x%"PRIx64 " type %"PRIu8,
1494 inv_desc.hi, inv_desc.lo, desc_type);
1495 return false;
1496 }
1497 s->iq_head++;
1498 if (s->iq_head == s->iq_size) {
1499 s->iq_head = 0;
1500 }
1501 return true;
1502}
1503
1504/* Try to fetch and process more Invalidation Descriptors */
1505static void vtd_fetch_inv_desc(IntelIOMMUState *s)
1506{
1507 VTD_DPRINTF(INV, "fetch Invalidation Descriptors");
1508 if (s->iq_tail >= s->iq_size) {
1509 /* Detects an invalid Tail pointer */
1510 VTD_DPRINTF(GENERAL, "error: iq_tail is %"PRIu16
1511 " while iq_size is %"PRIu16, s->iq_tail, s->iq_size);
1512 vtd_handle_inv_queue_error(s);
1513 return;
1514 }
1515 while (s->iq_head != s->iq_tail) {
1516 if (!vtd_process_inv_desc(s)) {
1517 /* Invalidation Queue Errors */
1518 vtd_handle_inv_queue_error(s);
1519 break;
1520 }
1521 /* Must update the IQH_REG in time */
1522 vtd_set_quad_raw(s, DMAR_IQH_REG,
1523 (((uint64_t)(s->iq_head)) << VTD_IQH_QH_SHIFT) &
1524 VTD_IQH_QH_MASK);
1525 }
1526}
1527
1528/* Handle write to Invalidation Queue Tail Register */
1529static void vtd_handle_iqt_write(IntelIOMMUState *s)
1530{
1531 uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG);
1532
1533 s->iq_tail = VTD_IQT_QT(val);
1534 VTD_DPRINTF(INV, "set iq tail %"PRIu16, s->iq_tail);
1535 if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) {
1536 /* Process Invalidation Queue here */
1537 vtd_fetch_inv_desc(s);
1538 }
1539}
1540
1da12ec4
LT		 1541static void vtd_handle_fsts_write(IntelIOMMUState *s)
1542{
1543 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG);
1544 uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
1545 uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE;
1546
1547 if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) {
1548 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
1549 VTD_DPRINTF(FLOG, "all pending interrupt conditions serviced, clear "
1550 "IP field of FECTL_REG");
1551 }
ed7b8fbc
LT		 1552 /* FIXME: when IQE is Clear, should we try to fetch some Invalidation
1553 * Descriptors if there are any when Queued Invalidation is enabled?
1554 */
1da12ec4
LT		 1555}
1556
1557static void vtd_handle_fectl_write(IntelIOMMUState *s)
1558{
1559 uint32_t fectl_reg;
1560 /* FIXME: when software clears the IM field, check the IP field. But do we
1561 * need to compare the old value and the new value to conclude that
1562 * software clears the IM field? Or just check if the IM field is zero?
1563 */
1564 fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG);
1565 if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) {
1566 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG);
1567 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0);
1568 VTD_DPRINTF(FLOG, "IM field is cleared, generate "
1569 "fault event interrupt");
1570 }
1571}
1572
ed7b8fbc
LT		 1573static void vtd_handle_ics_write(IntelIOMMUState *s)
1574{
1575 uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG);
1576 uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
1577
1578 if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) {
1579 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
1580 VTD_DPRINTF(INV, "pending completion interrupt condition serviced, "
1581 "clear IP field of IECTL_REG");
1582 }
1583}
1584
1585static void vtd_handle_iectl_write(IntelIOMMUState *s)
1586{
1587 uint32_t iectl_reg;
1588 /* FIXME: when software clears the IM field, check the IP field. But do we
1589 * need to compare the old value and the new value to conclude that
1590 * software clears the IM field? Or just check if the IM field is zero?
1591 */
1592 iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG);
1593 if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) {
1594 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG);
1595 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0);
1596 VTD_DPRINTF(INV, "IM field is cleared, generate "
1597 "invalidation event interrupt");
1598 }
1599}
1600
1da12ec4
LT		 1601static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size)
1602{
1603 IntelIOMMUState *s = opaque;
1604 uint64_t val;
1605
1606 if (addr + size > DMAR_REG_SIZE) {
1607 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64
1608 ", got 0x%"PRIx64 " %d",
1609 (uint64_t)DMAR_REG_SIZE, addr, size);
1610 return (uint64_t)-1;
1611 }
1612
1613 switch (addr) {
1614 /* Root Table Address Register, 64-bit */
1615 case DMAR_RTADDR_REG:
1616 if (size == 4) {
1617 val = s->root & ((1ULL << 32) - 1);
1618 } else {
1619 val = s->root;
1620 }
1621 break;
1622
1623 case DMAR_RTADDR_REG_HI:
1624 assert(size == 4);
1625 val = s->root >> 32;
1626 break;
1627
ed7b8fbc
LT		 1628 /* Invalidation Queue Address Register, 64-bit */
1629 case DMAR_IQA_REG:
1630 val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS);
1631 if (size == 4) {
1632 val = val & ((1ULL << 32) - 1);
1633 }
1634 break;
1635
1636 case DMAR_IQA_REG_HI:
1637 assert(size == 4);
1638 val = s->iq >> 32;
1639 break;
1640
1da12ec4
LT		 1641 default:
1642 if (size == 4) {
1643 val = vtd_get_long(s, addr);
1644 } else {
1645 val = vtd_get_quad(s, addr);
1646 }
1647 }
1648 VTD_DPRINTF(CSR, "addr 0x%"PRIx64 " size %d val 0x%"PRIx64,
1649 addr, size, val);
1650 return val;
1651}
1652
1653static void vtd_mem_write(void *opaque, hwaddr addr,
1654 uint64_t val, unsigned size)
1655{
1656 IntelIOMMUState *s = opaque;
1657
1658 if (addr + size > DMAR_REG_SIZE) {
1659 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64
1660 ", got 0x%"PRIx64 " %d",
1661 (uint64_t)DMAR_REG_SIZE, addr, size);
1662 return;
1663 }
1664
1665 switch (addr) {
1666 /* Global Command Register, 32-bit */
1667 case DMAR_GCMD_REG:
1668 VTD_DPRINTF(CSR, "DMAR_GCMD_REG write addr 0x%"PRIx64
1669 ", size %d, val 0x%"PRIx64, addr, size, val);
1670 vtd_set_long(s, addr, val);
1671 vtd_handle_gcmd_write(s);
1672 break;
1673
1674 /* Context Command Register, 64-bit */
1675 case DMAR_CCMD_REG:
1676 VTD_DPRINTF(CSR, "DMAR_CCMD_REG write addr 0x%"PRIx64
1677 ", size %d, val 0x%"PRIx64, addr, size, val);
1678 if (size == 4) {
1679 vtd_set_long(s, addr, val);
1680 } else {
1681 vtd_set_quad(s, addr, val);
1682 vtd_handle_ccmd_write(s);
1683 }
1684 break;
1685
1686 case DMAR_CCMD_REG_HI:
1687 VTD_DPRINTF(CSR, "DMAR_CCMD_REG_HI write addr 0x%"PRIx64
1688 ", size %d, val 0x%"PRIx64, addr, size, val);
1689 assert(size == 4);
1690 vtd_set_long(s, addr, val);
1691 vtd_handle_ccmd_write(s);
1692 break;
1693
1694 /* IOTLB Invalidation Register, 64-bit */
1695 case DMAR_IOTLB_REG:
1696 VTD_DPRINTF(INV, "DMAR_IOTLB_REG write addr 0x%"PRIx64
1697 ", size %d, val 0x%"PRIx64, addr, size, val);
1698 if (size == 4) {
1699 vtd_set_long(s, addr, val);
1700 } else {
1701 vtd_set_quad(s, addr, val);
1702 vtd_handle_iotlb_write(s);
1703 }
1704 break;
1705
1706 case DMAR_IOTLB_REG_HI:
1707 VTD_DPRINTF(INV, "DMAR_IOTLB_REG_HI write addr 0x%"PRIx64
1708 ", size %d, val 0x%"PRIx64, addr, size, val);
1709 assert(size == 4);
1710 vtd_set_long(s, addr, val);
1711 vtd_handle_iotlb_write(s);
1712 break;
1713
b5a280c0
LT		 1714 /* Invalidate Address Register, 64-bit */
1715 case DMAR_IVA_REG:
1716 VTD_DPRINTF(INV, "DMAR_IVA_REG write addr 0x%"PRIx64
1717 ", size %d, val 0x%"PRIx64, addr, size, val);
1718 if (size == 4) {
1719 vtd_set_long(s, addr, val);
1720 } else {
1721 vtd_set_quad(s, addr, val);
1722 }
1723 break;
1724
1725 case DMAR_IVA_REG_HI:
1726 VTD_DPRINTF(INV, "DMAR_IVA_REG_HI write addr 0x%"PRIx64
1727 ", size %d, val 0x%"PRIx64, addr, size, val);
1728 assert(size == 4);
1729 vtd_set_long(s, addr, val);
1730 break;
1731
1da12ec4
LT		 1732 /* Fault Status Register, 32-bit */
1733 case DMAR_FSTS_REG:
1734 VTD_DPRINTF(FLOG, "DMAR_FSTS_REG write addr 0x%"PRIx64
1735 ", size %d, val 0x%"PRIx64, addr, size, val);
1736 assert(size == 4);
1737 vtd_set_long(s, addr, val);
1738 vtd_handle_fsts_write(s);
1739 break;
1740
1741 /* Fault Event Control Register, 32-bit */
1742 case DMAR_FECTL_REG:
1743 VTD_DPRINTF(FLOG, "DMAR_FECTL_REG write addr 0x%"PRIx64
1744 ", size %d, val 0x%"PRIx64, addr, size, val);
1745 assert(size == 4);
1746 vtd_set_long(s, addr, val);
1747 vtd_handle_fectl_write(s);
1748 break;
1749
1750 /* Fault Event Data Register, 32-bit */
1751 case DMAR_FEDATA_REG:
1752 VTD_DPRINTF(FLOG, "DMAR_FEDATA_REG write addr 0x%"PRIx64
1753 ", size %d, val 0x%"PRIx64, addr, size, val);
1754 assert(size == 4);
1755 vtd_set_long(s, addr, val);
1756 break;
1757
1758 /* Fault Event Address Register, 32-bit */
1759 case DMAR_FEADDR_REG:
1760 VTD_DPRINTF(FLOG, "DMAR_FEADDR_REG write addr 0x%"PRIx64
1761 ", size %d, val 0x%"PRIx64, addr, size, val);
1762 assert(size == 4);
1763 vtd_set_long(s, addr, val);
1764 break;
1765
1766 /* Fault Event Upper Address Register, 32-bit */
1767 case DMAR_FEUADDR_REG:
1768 VTD_DPRINTF(FLOG, "DMAR_FEUADDR_REG write addr 0x%"PRIx64
1769 ", size %d, val 0x%"PRIx64, addr, size, val);
1770 assert(size == 4);
1771 vtd_set_long(s, addr, val);
1772 break;
1773
1774 /* Protected Memory Enable Register, 32-bit */
1775 case DMAR_PMEN_REG:
1776 VTD_DPRINTF(CSR, "DMAR_PMEN_REG write addr 0x%"PRIx64
1777 ", size %d, val 0x%"PRIx64, addr, size, val);
1778 assert(size == 4);
1779 vtd_set_long(s, addr, val);
1780 break;
1781
1782 /* Root Table Address Register, 64-bit */
1783 case DMAR_RTADDR_REG:
1784 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG write addr 0x%"PRIx64
1785 ", size %d, val 0x%"PRIx64, addr, size, val);
1786 if (size == 4) {
1787 vtd_set_long(s, addr, val);
1788 } else {
1789 vtd_set_quad(s, addr, val);
1790 }
1791 break;
1792
1793 case DMAR_RTADDR_REG_HI:
1794 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG_HI write addr 0x%"PRIx64
1795 ", size %d, val 0x%"PRIx64, addr, size, val);
1796 assert(size == 4);
1797 vtd_set_long(s, addr, val);
1798 break;
1799
ed7b8fbc
LT		 1800 /* Invalidation Queue Tail Register, 64-bit */
1801 case DMAR_IQT_REG:
1802 VTD_DPRINTF(INV, "DMAR_IQT_REG write addr 0x%"PRIx64
1803 ", size %d, val 0x%"PRIx64, addr, size, val);
1804 if (size == 4) {
1805 vtd_set_long(s, addr, val);
1806 } else {
1807 vtd_set_quad(s, addr, val);
1808 }
1809 vtd_handle_iqt_write(s);
1810 break;
1811
1812 case DMAR_IQT_REG_HI:
1813 VTD_DPRINTF(INV, "DMAR_IQT_REG_HI write addr 0x%"PRIx64
1814 ", size %d, val 0x%"PRIx64, addr, size, val);
1815 assert(size == 4);
1816 vtd_set_long(s, addr, val);
1817 /* 19:63 of IQT_REG is RsvdZ, do nothing here */
1818 break;
1819
1820 /* Invalidation Queue Address Register, 64-bit */
1821 case DMAR_IQA_REG:
1822 VTD_DPRINTF(INV, "DMAR_IQA_REG write addr 0x%"PRIx64
1823 ", size %d, val 0x%"PRIx64, addr, size, val);
1824 if (size == 4) {
1825 vtd_set_long(s, addr, val);
1826 } else {
1827 vtd_set_quad(s, addr, val);
1828 }
1829 break;
1830
1831 case DMAR_IQA_REG_HI:
1832 VTD_DPRINTF(INV, "DMAR_IQA_REG_HI write addr 0x%"PRIx64
1833 ", size %d, val 0x%"PRIx64, addr, size, val);
1834 assert(size == 4);
1835 vtd_set_long(s, addr, val);
1836 break;
1837
1838 /* Invalidation Completion Status Register, 32-bit */
1839 case DMAR_ICS_REG:
1840 VTD_DPRINTF(INV, "DMAR_ICS_REG write addr 0x%"PRIx64
1841 ", size %d, val 0x%"PRIx64, addr, size, val);
1842 assert(size == 4);
1843 vtd_set_long(s, addr, val);
1844 vtd_handle_ics_write(s);
1845 break;
1846
1847 /* Invalidation Event Control Register, 32-bit */
1848 case DMAR_IECTL_REG:
1849 VTD_DPRINTF(INV, "DMAR_IECTL_REG write addr 0x%"PRIx64
1850 ", size %d, val 0x%"PRIx64, addr, size, val);
1851 assert(size == 4);
1852 vtd_set_long(s, addr, val);
1853 vtd_handle_iectl_write(s);
1854 break;
1855
1856 /* Invalidation Event Data Register, 32-bit */
1857 case DMAR_IEDATA_REG:
1858 VTD_DPRINTF(INV, "DMAR_IEDATA_REG write addr 0x%"PRIx64
1859 ", size %d, val 0x%"PRIx64, addr, size, val);
1860 assert(size == 4);
1861 vtd_set_long(s, addr, val);
1862 break;
1863
1864 /* Invalidation Event Address Register, 32-bit */
1865 case DMAR_IEADDR_REG:
1866 VTD_DPRINTF(INV, "DMAR_IEADDR_REG write addr 0x%"PRIx64
1867 ", size %d, val 0x%"PRIx64, addr, size, val);
1868 assert(size == 4);
1869 vtd_set_long(s, addr, val);
1870 break;
1871
1872 /* Invalidation Event Upper Address Register, 32-bit */
1873 case DMAR_IEUADDR_REG:
1874 VTD_DPRINTF(INV, "DMAR_IEUADDR_REG write addr 0x%"PRIx64
1875 ", size %d, val 0x%"PRIx64, addr, size, val);
1876 assert(size == 4);
1877 vtd_set_long(s, addr, val);
1878 break;
1879
1da12ec4
LT		 1880 /* Fault Recording Registers, 128-bit */
1881 case DMAR_FRCD_REG_0_0:
1882 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_0 write addr 0x%"PRIx64
1883 ", size %d, val 0x%"PRIx64, addr, size, val);
1884 if (size == 4) {
1885 vtd_set_long(s, addr, val);
1886 } else {
1887 vtd_set_quad(s, addr, val);
1888 }
1889 break;
1890
1891 case DMAR_FRCD_REG_0_1:
1892 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_1 write addr 0x%"PRIx64
1893 ", size %d, val 0x%"PRIx64, addr, size, val);
1894 assert(size == 4);
1895 vtd_set_long(s, addr, val);
1896 break;
1897
1898 case DMAR_FRCD_REG_0_2:
1899 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_2 write addr 0x%"PRIx64
1900 ", size %d, val 0x%"PRIx64, addr, size, val);
1901 if (size == 4) {
1902 vtd_set_long(s, addr, val);
1903 } else {
1904 vtd_set_quad(s, addr, val);
1905 /* May clear bit 127 (Fault), update PPF */
1906 vtd_update_fsts_ppf(s);
1907 }
1908 break;
1909
1910 case DMAR_FRCD_REG_0_3:
1911 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_3 write addr 0x%"PRIx64
1912 ", size %d, val 0x%"PRIx64, addr, size, val);
1913 assert(size == 4);
1914 vtd_set_long(s, addr, val);
1915 /* May clear bit 127 (Fault), update PPF */
1916 vtd_update_fsts_ppf(s);
1917 break;
1918
a5861439
PX		 1919 case DMAR_IRTA_REG:
1920 VTD_DPRINTF(IR, "DMAR_IRTA_REG write addr 0x%"PRIx64
1921 ", size %d, val 0x%"PRIx64, addr, size, val);
1922 if (size == 4) {
1923 vtd_set_long(s, addr, val);
1924 } else {
1925 vtd_set_quad(s, addr, val);
1926 }
1927 break;
1928
1929 case DMAR_IRTA_REG_HI:
1930 VTD_DPRINTF(IR, "DMAR_IRTA_REG_HI write addr 0x%"PRIx64
1931 ", size %d, val 0x%"PRIx64, addr, size, val);
1932 assert(size == 4);
1933 vtd_set_long(s, addr, val);
1934 break;
1935
1da12ec4
LT		 1936 default:
1937 VTD_DPRINTF(GENERAL, "error: unhandled reg write addr 0x%"PRIx64
1938 ", size %d, val 0x%"PRIx64, addr, size, val);
1939 if (size == 4) {
1940 vtd_set_long(s, addr, val);
1941 } else {
1942 vtd_set_quad(s, addr, val);
1943 }
1944 }
1945}
1946
1947static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr,
1948 bool is_write)
1949{
1950 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
1951 IntelIOMMUState *s = vtd_as->iommu_state;
1da12ec4
LT		 1952 IOMMUTLBEntry ret = {
1953 .target_as = &address_space_memory,
1954 .iova = addr,
1955 .translated_addr = 0,
1956 .addr_mask = ~(hwaddr)0,
1957 .perm = IOMMU_NONE,
1958 };
1959
1960 if (!s->dmar_enabled) {
1961 /* DMAR disabled, passthrough, use 4k-page*/
1962 ret.iova = addr & VTD_PAGE_MASK_4K;
1963 ret.translated_addr = addr & VTD_PAGE_MASK_4K;
1964 ret.addr_mask = ~VTD_PAGE_MASK_4K;
1965 ret.perm = IOMMU_RW;
1966 return ret;
1967 }
1968
7df953bd 1969 vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, addr,
d92fa2dc 1970 is_write, &ret);
1da12ec4
LT		 1971 VTD_DPRINTF(MMU,
1972 "bus %"PRIu8 " slot %"PRIu8 " func %"PRIu8 " devfn %"PRIu8
7df953bd 1973 " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, pci_bus_num(vtd_as->bus),
d92fa2dc
LT		 1974 VTD_PCI_SLOT(vtd_as->devfn), VTD_PCI_FUNC(vtd_as->devfn),
1975 vtd_as->devfn, addr, ret.translated_addr);
1da12ec4
LT		 1976 return ret;
1977}
1978
5bf3d319
PX		 1979static void vtd_iommu_notify_flag_changed(MemoryRegion *iommu,
1980 IOMMUNotifierFlag old,
1981 IOMMUNotifierFlag new)
3cb3b154
AW		 1982{
1983 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu);
1984
a3276f78
PX		 1985 if (new & IOMMU_NOTIFIER_MAP) {
1986 error_report("Device at bus %s addr %02x.%d requires iommu "
1987 "notifier which is currently not supported by "
1988 "intel-iommu emulation",
1989 vtd_as->bus->qbus.name, PCI_SLOT(vtd_as->devfn),
1990 PCI_FUNC(vtd_as->devfn));
1991 exit(1);
1992 }
3cb3b154
AW		 1993}
1994
1da12ec4
LT		 1995static const VMStateDescription vtd_vmstate = {
1996 .name = "iommu-intel",
1997 .unmigratable = 1,
1998};
1999
2000static const MemoryRegionOps vtd_mem_ops = {
2001 .read = vtd_mem_read,
2002 .write = vtd_mem_write,
2003 .endianness = DEVICE_LITTLE_ENDIAN,
2004 .impl = {
2005 .min_access_size = 4,
2006 .max_access_size = 8,
2007 },
2008 .valid = {
2009 .min_access_size = 4,
2010 .max_access_size = 8,
2011 },
2012};
2013
2014static Property vtd_properties[] = {
2015 DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0),
2016 DEFINE_PROP_END_OF_LIST(),
2017};
2018
651e4cef
PX		 2019/* Read IRTE entry with specific index */
2020static int vtd_irte_get(IntelIOMMUState *iommu, uint16_t index,
bc38ee10 2021 VTD_IR_TableEntry *entry, uint16_t sid)
651e4cef 2022{
ede9c94a
PX		 2023 static const uint16_t vtd_svt_mask[VTD_SQ_MAX] = \
2024 {0xffff, 0xfffb, 0xfff9, 0xfff8};
651e4cef 2025 dma_addr_t addr = 0x00;
ede9c94a
PX		 2026 uint16_t mask, source_id;
2027 uint8_t bus, bus_max, bus_min;
651e4cef
PX		 2028
2029 addr = iommu->intr_root + index * sizeof(*entry);
2030 if (dma_memory_read(&address_space_memory, addr, entry,
2031 sizeof(*entry))) {
2032 VTD_DPRINTF(GENERAL, "error: fail to access IR root at 0x%"PRIx64
2033 " + %"PRIu16, iommu->intr_root, index);
2034 return -VTD_FR_IR_ROOT_INVAL;
2035 }
2036
bc38ee10 2037 if (!entry->irte.present) {
651e4cef
PX		 2038 VTD_DPRINTF(GENERAL, "error: present flag not set in IRTE"
2039 " entry index %u value 0x%"PRIx64 " 0x%"PRIx64,
2040 index, le64_to_cpu(entry->data[1]),
2041 le64_to_cpu(entry->data[0]));
2042 return -VTD_FR_IR_ENTRY_P;
2043 }
2044
bc38ee10
MT		 2045 if (entry->irte.__reserved_0 || entry->irte.__reserved_1 ||
2046 entry->irte.__reserved_2) {
651e4cef
PX		 2047 VTD_DPRINTF(GENERAL, "error: IRTE entry index %"PRIu16
2048 " reserved fields non-zero: 0x%"PRIx64 " 0x%"PRIx64,
2049 index, le64_to_cpu(entry->data[1]),
2050 le64_to_cpu(entry->data[0]));
2051 return -VTD_FR_IR_IRTE_RSVD;
2052 }
2053
ede9c94a
PX		 2054 if (sid != X86_IOMMU_SID_INVALID) {
2055 /* Validate IRTE SID */
bc38ee10
MT		 2056 source_id = le32_to_cpu(entry->irte.source_id);
2057 switch (entry->irte.sid_vtype) {
ede9c94a
PX		 2058 case VTD_SVT_NONE:
2059 VTD_DPRINTF(IR, "No SID validation for IRTE index %d", index);
2060 break;
2061
2062 case VTD_SVT_ALL:
bc38ee10 2063 mask = vtd_svt_mask[entry->irte.sid_q];
ede9c94a
PX		 2064 if ((source_id & mask) != (sid & mask)) {
2065 VTD_DPRINTF(GENERAL, "SID validation for IRTE index "
2066 "%d failed (reqid 0x%04x sid 0x%04x)", index,
2067 sid, source_id);
2068 return -VTD_FR_IR_SID_ERR;
2069 }
2070 break;
2071
2072 case VTD_SVT_BUS:
2073 bus_max = source_id >> 8;
2074 bus_min = source_id & 0xff;
2075 bus = sid >> 8;
2076 if (bus > bus_max || bus < bus_min) {
2077 VTD_DPRINTF(GENERAL, "SID validation for IRTE index %d "
2078 "failed (bus %d outside %d-%d)", index, bus,
2079 bus_min, bus_max);
2080 return -VTD_FR_IR_SID_ERR;
2081 }
2082 break;
2083
2084 default:
2085 VTD_DPRINTF(GENERAL, "Invalid SVT bits (0x%x) in IRTE index "
bc38ee10 2086 "%d", entry->irte.sid_vtype, index);
ede9c94a
PX		 2087 /* Take this as verification failure. */
2088 return -VTD_FR_IR_SID_ERR;
2089 break;
2090 }
2091 }
651e4cef
PX		 2092
2093 return 0;
2094}
2095
2096/* Fetch IRQ information of specific IR index */
ede9c94a
PX		 2097static int vtd_remap_irq_get(IntelIOMMUState *iommu, uint16_t index,
2098 VTDIrq *irq, uint16_t sid)
651e4cef 2099{
bc38ee10 2100 VTD_IR_TableEntry irte = {};
651e4cef
PX		 2101 int ret = 0;
2102
ede9c94a 2103 ret = vtd_irte_get(iommu, index, &irte, sid);
651e4cef
PX		 2104 if (ret) {
2105 return ret;
2106 }
2107
bc38ee10
MT		 2108 irq->trigger_mode = irte.irte.trigger_mode;
2109 irq->vector = irte.irte.vector;
2110 irq->delivery_mode = irte.irte.delivery_mode;
2111 irq->dest = le32_to_cpu(irte.irte.dest_id);
28589311 2112 if (!iommu->intr_eime) {
651e4cef
PX		 2113#define VTD_IR_APIC_DEST_MASK (0xff00ULL)
2114#define VTD_IR_APIC_DEST_SHIFT (8)
28589311
JK		 2115 irq->dest = (irq->dest & VTD_IR_APIC_DEST_MASK) >>
2116 VTD_IR_APIC_DEST_SHIFT;
2117 }
bc38ee10
MT		 2118 irq->dest_mode = irte.irte.dest_mode;
2119 irq->redir_hint = irte.irte.redir_hint;
651e4cef
PX		 2120
2121 VTD_DPRINTF(IR, "remapping interrupt index %d: trig:%u,vec:%u,"
2122 "deliver:%u,dest:%u,dest_mode:%u", index,
2123 irq->trigger_mode, irq->vector, irq->delivery_mode,
2124 irq->dest, irq->dest_mode);
2125
2126 return 0;
2127}
2128
2129/* Generate one MSI message from VTDIrq info */
2130static void vtd_generate_msi_message(VTDIrq *irq, MSIMessage *msg_out)
2131{
2132 VTD_MSIMessage msg = {};
2133
2134 /* Generate address bits */
2135 msg.dest_mode = irq->dest_mode;
2136 msg.redir_hint = irq->redir_hint;
2137 msg.dest = irq->dest;
32946019 2138 msg.__addr_hi = irq->dest & 0xffffff00;
651e4cef
PX		 2139 msg.__addr_head = cpu_to_le32(0xfee);
2140 /* Keep this from original MSI address bits */
2141 msg.__not_used = irq->msi_addr_last_bits;
2142
2143 /* Generate data bits */
2144 msg.vector = irq->vector;
2145 msg.delivery_mode = irq->delivery_mode;
2146 msg.level = 1;
2147 msg.trigger_mode = irq->trigger_mode;
2148
2149 msg_out->address = msg.msi_addr;
2150 msg_out->data = msg.msi_data;
2151}
2152
2153/* Interrupt remapping for MSI/MSI-X entry */
2154static int vtd_interrupt_remap_msi(IntelIOMMUState *iommu,
2155 MSIMessage *origin,
ede9c94a
PX		 2156 MSIMessage *translated,
2157 uint16_t sid)
651e4cef
PX		 2158{
2159 int ret = 0;
2160 VTD_IR_MSIAddress addr;
2161 uint16_t index;
09cd058a 2162 VTDIrq irq = {};
651e4cef
PX		 2163
2164 assert(origin && translated);
2165
2166 if (!iommu || !iommu->intr_enabled) {
2167 goto do_not_translate;
2168 }
2169
2170 if (origin->address & VTD_MSI_ADDR_HI_MASK) {
2171 VTD_DPRINTF(GENERAL, "error: MSI addr high 32 bits nonzero"
2172 " during interrupt remapping: 0x%"PRIx32,
2173 (uint32_t)((origin->address & VTD_MSI_ADDR_HI_MASK) >> \
2174 VTD_MSI_ADDR_HI_SHIFT));
2175 return -VTD_FR_IR_REQ_RSVD;
2176 }
2177
2178 addr.data = origin->address & VTD_MSI_ADDR_LO_MASK;
bc38ee10 2179 if (le16_to_cpu(addr.addr.__head) != 0xfee) {
651e4cef
PX		 2180 VTD_DPRINTF(GENERAL, "error: MSI addr low 32 bits invalid: "
2181 "0x%"PRIx32, addr.data);
2182 return -VTD_FR_IR_REQ_RSVD;
2183 }
2184
2185 /* This is compatible mode. */
bc38ee10 2186 if (addr.addr.int_mode != VTD_IR_INT_FORMAT_REMAP) {
651e4cef
PX		 2187 goto do_not_translate;
2188 }
2189
bc38ee10 2190 index = addr.addr.index_h << 15 | le16_to_cpu(addr.addr.index_l);
651e4cef
PX		 2191
2192#define VTD_IR_MSI_DATA_SUBHANDLE (0x0000ffff)
2193#define VTD_IR_MSI_DATA_RESERVED (0xffff0000)
2194
bc38ee10 2195 if (addr.addr.sub_valid) {
651e4cef
PX		 2196 /* See VT-d spec 5.1.2.2 and 5.1.3 on subhandle */
2197 index += origin->data & VTD_IR_MSI_DATA_SUBHANDLE;
2198 }
2199
ede9c94a 2200 ret = vtd_remap_irq_get(iommu, index, &irq, sid);
651e4cef
PX		 2201 if (ret) {
2202 return ret;
2203 }
2204
bc38ee10 2205 if (addr.addr.sub_valid) {
651e4cef
PX		 2206 VTD_DPRINTF(IR, "received MSI interrupt");
2207 if (origin->data & VTD_IR_MSI_DATA_RESERVED) {
2208 VTD_DPRINTF(GENERAL, "error: MSI data bits non-zero for "
2209 "interrupt remappable entry: 0x%"PRIx32,
2210 origin->data);
2211 return -VTD_FR_IR_REQ_RSVD;
2212 }
2213 } else {
2214 uint8_t vector = origin->data & 0xff;
dea651a9
FW		 2215 uint8_t trigger_mode = (origin->data >> MSI_DATA_TRIGGER_SHIFT) & 0x1;
2216
651e4cef
PX		 2217 VTD_DPRINTF(IR, "received IOAPIC interrupt");
2218 /* IOAPIC entry vector should be aligned with IRTE vector
2219 * (see vt-d spec 5.1.5.1). */
2220 if (vector != irq.vector) {
2221 VTD_DPRINTF(GENERAL, "IOAPIC vector inconsistent: "
2222 "entry: %d, IRTE: %d, index: %d",
2223 vector, irq.vector, index);
2224 }
dea651a9
FW		 2225
2226 /* The Trigger Mode field must match the Trigger Mode in the IRTE.
2227 * (see vt-d spec 5.1.5.1). */
2228 if (trigger_mode != irq.trigger_mode) {
2229 VTD_DPRINTF(GENERAL, "IOAPIC trigger mode inconsistent: "
2230 "entry: %u, IRTE: %u, index: %d",
2231 trigger_mode, irq.trigger_mode, index);
2232 }
2233
651e4cef
PX		 2234 }
2235
2236 /*
2237 * We'd better keep the last two bits, assuming that guest OS
2238 * might modify it. Keep it does not hurt after all.
2239 */
bc38ee10 2240 irq.msi_addr_last_bits = addr.addr.__not_care;
651e4cef
PX		 2241
2242 /* Translate VTDIrq to MSI message */
2243 vtd_generate_msi_message(&irq, translated);
2244
2245 VTD_DPRINTF(IR, "mapping MSI 0x%"PRIx64":0x%"PRIx32 " -> "
2246 "0x%"PRIx64":0x%"PRIx32, origin->address, origin->data,
2247 translated->address, translated->data);
2248 return 0;
2249
2250do_not_translate:
2251 memcpy(translated, origin, sizeof(*origin));
2252 return 0;
2253}
2254
8b5ed7df
PX		 2255static int vtd_int_remap(X86IOMMUState *iommu, MSIMessage *src,
2256 MSIMessage *dst, uint16_t sid)
2257{
ede9c94a
PX		 2258 return vtd_interrupt_remap_msi(INTEL_IOMMU_DEVICE(iommu),
2259 src, dst, sid);
8b5ed7df
PX		 2260}
2261
651e4cef
PX		 2262static MemTxResult vtd_mem_ir_read(void *opaque, hwaddr addr,
2263 uint64_t *data, unsigned size,
2264 MemTxAttrs attrs)
2265{
2266 return MEMTX_OK;
2267}
2268
2269static MemTxResult vtd_mem_ir_write(void *opaque, hwaddr addr,
2270 uint64_t value, unsigned size,
2271 MemTxAttrs attrs)
2272{
2273 int ret = 0;
09cd058a 2274 MSIMessage from = {}, to = {};
ede9c94a 2275 uint16_t sid = X86_IOMMU_SID_INVALID;
651e4cef
PX		 2276
2277 from.address = (uint64_t) addr + VTD_INTERRUPT_ADDR_FIRST;
2278 from.data = (uint32_t) value;
2279
ede9c94a
PX		 2280 if (!attrs.unspecified) {
2281 /* We have explicit Source ID */
2282 sid = attrs.requester_id;
2283 }
2284
2285 ret = vtd_interrupt_remap_msi(opaque, &from, &to, sid);
651e4cef
PX		 2286 if (ret) {
2287 /* TODO: report error */
2288 VTD_DPRINTF(GENERAL, "int remap fail for addr 0x%"PRIx64
2289 " data 0x%"PRIx32, from.address, from.data);
2290 /* Drop this interrupt */
2291 return MEMTX_ERROR;
2292 }
2293
2294 VTD_DPRINTF(IR, "delivering MSI 0x%"PRIx64":0x%"PRIx32
2295 " for device sid 0x%04x",
2296 to.address, to.data, sid);
2297
32946019 2298 apic_get_class()->send_msi(&to);
651e4cef
PX		 2299
2300 return MEMTX_OK;
2301}
2302
2303static const MemoryRegionOps vtd_mem_ir_ops = {
2304 .read_with_attrs = vtd_mem_ir_read,
2305 .write_with_attrs = vtd_mem_ir_write,
2306 .endianness = DEVICE_LITTLE_ENDIAN,
2307 .impl = {
2308 .min_access_size = 4,
2309 .max_access_size = 4,
2310 },
2311 .valid = {
2312 .min_access_size = 4,
2313 .max_access_size = 4,
2314 },
2315};
7df953bd
KO		 2316
2317VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn)
2318{
2319 uintptr_t key = (uintptr_t)bus;
2320 VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key);
2321 VTDAddressSpace *vtd_dev_as;
2322
2323 if (!vtd_bus) {
2324 /* No corresponding free() */
04af0e18
PX		 2325 vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * \
2326 X86_IOMMU_PCI_DEVFN_MAX);
7df953bd
KO		 2327 vtd_bus->bus = bus;
2328 key = (uintptr_t)bus;
2329 g_hash_table_insert(s->vtd_as_by_busptr, &key, vtd_bus);
2330 }
2331
2332 vtd_dev_as = vtd_bus->dev_as[devfn];
2333
2334 if (!vtd_dev_as) {
2335 vtd_bus->dev_as[devfn] = vtd_dev_as = g_malloc0(sizeof(VTDAddressSpace));
2336
2337 vtd_dev_as->bus = bus;
2338 vtd_dev_as->devfn = (uint8_t)devfn;
2339 vtd_dev_as->iommu_state = s;
2340 vtd_dev_as->context_cache_entry.context_cache_gen = 0;
2341 memory_region_init_iommu(&vtd_dev_as->iommu, OBJECT(s),
2342 &s->iommu_ops, "intel_iommu", UINT64_MAX);
651e4cef
PX		 2343 memory_region_init_io(&vtd_dev_as->iommu_ir, OBJECT(s),
2344 &vtd_mem_ir_ops, s, "intel_iommu_ir",
2345 VTD_INTERRUPT_ADDR_SIZE);
2346 memory_region_add_subregion(&vtd_dev_as->iommu, VTD_INTERRUPT_ADDR_FIRST,
2347 &vtd_dev_as->iommu_ir);
7df953bd
KO		 2348 address_space_init(&vtd_dev_as->as,
2349 &vtd_dev_as->iommu, "intel_iommu");
2350 }
2351 return vtd_dev_as;
2352}
2353
1da12ec4
LT		 2354/* Do the initialization. It will also be called when reset, so pay
2355 * attention when adding new initialization stuff.
2356 */
2357static void vtd_init(IntelIOMMUState *s)
2358{
d54bd7f8
PX		 2359 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(s);
2360
1da12ec4
LT		 2361 memset(s->csr, 0, DMAR_REG_SIZE);
2362 memset(s->wmask, 0, DMAR_REG_SIZE);
2363 memset(s->w1cmask, 0, DMAR_REG_SIZE);
2364 memset(s->womask, 0, DMAR_REG_SIZE);
2365
2366 s->iommu_ops.translate = vtd_iommu_translate;
5bf3d319 2367 s->iommu_ops.notify_flag_changed = vtd_iommu_notify_flag_changed;
1da12ec4
LT		 2368 s->root = 0;
2369 s->root_extended = false;
2370 s->dmar_enabled = false;
2371 s->iq_head = 0;
2372 s->iq_tail = 0;
2373 s->iq = 0;
2374 s->iq_size = 0;
2375 s->qi_enabled = false;
2376 s->iq_last_desc_type = VTD_INV_DESC_NONE;
2377 s->next_frcd_reg = 0;
2378 s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | VTD_CAP_MGAW |
d66b969b 2379 VTD_CAP_SAGAW | VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS;
ed7b8fbc 2380 s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO;
1da12ec4 2381
d54bd7f8 2382 if (x86_iommu->intr_supported) {
a3f409cb 2383 s->ecap |= VTD_ECAP_IR | VTD_ECAP_EIM | VTD_ECAP_MHMV;
d54bd7f8
PX		 2384 }
2385
d92fa2dc 2386 vtd_reset_context_cache(s);
b5a280c0 2387 vtd_reset_iotlb(s);
d92fa2dc 2388
1da12ec4
LT		 2389 /* Define registers with default values and bit semantics */
2390 vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0);
2391 vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0);
2392 vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0);
2393 vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0);
2394 vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL);
2395 vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0);
2396 vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffff000ULL, 0);
2397 vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0);
2398 vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL);
2399
2400 /* Advanced Fault Logging not supported */
2401 vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL);
2402 vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0);
2403 vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0);
2404 vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0);
2405
2406 /* Treated as RsvdZ when EIM in ECAP_REG is not supported
2407 * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0);
2408 */
2409 vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0);
2410
2411 /* Treated as RO for implementations that PLMR and PHMR fields reported
2412 * as Clear in the CAP_REG.
2413 * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0);
2414 */
2415 vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0);
2416
ed7b8fbc
LT		 2417 vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0);
2418 vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0);
2419 vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff007ULL, 0);
2420 vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL);
2421 vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0);
2422 vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0);
2423 vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0);
2424 /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */
2425 vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0);
2426
1da12ec4
LT		 2427 /* IOTLB registers */
2428 vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0);
2429 vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0);
2430 vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL);
2431
2432 /* Fault Recording Registers, 128-bit */
2433 vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0);
2434 vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL);
a5861439
PX		 2435
2436 /*
28589311 2437 * Interrupt remapping registers.
a5861439 2438 */
28589311 2439 vtd_define_quad(s, DMAR_IRTA_REG, 0, 0xfffffffffffff80fULL, 0);
1da12ec4
LT		 2440}
2441
2442/* Should not reset address_spaces when reset because devices will still use
2443 * the address space they got at first (won't ask the bus again).
2444 */
2445static void vtd_reset(DeviceState *dev)
2446{
2447 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
2448
2449 VTD_DPRINTF(GENERAL, "");
2450 vtd_init(s);
2451}
2452
621d983a
MA		 2453static AddressSpace *vtd_host_dma_iommu(PCIBus *bus, void *opaque, int devfn)
2454{
2455 IntelIOMMUState *s = opaque;
2456 VTDAddressSpace *vtd_as;
2457
04af0e18 2458 assert(0 <= devfn && devfn <= X86_IOMMU_PCI_DEVFN_MAX);
621d983a
MA		 2459
2460 vtd_as = vtd_find_add_as(s, bus, devfn);
2461 return &vtd_as->as;
2462}
2463
6333e93c
RK		 2464static bool vtd_check_config(X86IOMMUState *x86_iommu, Error **errp)
2465{
2466 /* Currently Intel IOMMU IR only support "kernel-irqchip={off|split}" */
2467 if (x86_iommu->intr_supported && kvm_irqchip_in_kernel() &&
2468 !kvm_irqchip_is_split()) {
2469 error_setg(errp, "Intel Interrupt Remapping cannot work with "
2470 "kernel-irqchip=on, please use 'split|off'.");
2471 return false;
2472 }
2473 return true;
2474}
2475
1da12ec4
LT		 2476static void vtd_realize(DeviceState *dev, Error **errp)
2477{
cb135f59
PX		 2478 PCMachineState *pcms = PC_MACHINE(qdev_get_machine());
2479 PCIBus *bus = pcms->bus;
1da12ec4 2480 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev);
4684a204 2481 X86IOMMUState *x86_iommu = X86_IOMMU_DEVICE(dev);
1da12ec4
LT		 2482
2483 VTD_DPRINTF(GENERAL, "");
fb9f5926 2484 x86_iommu->type = TYPE_INTEL;
6333e93c
RK		 2485
2486 if (!vtd_check_config(x86_iommu, errp)) {
2487 return;
2488 }
2489
7df953bd 2490 memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num));
1da12ec4
LT		 2491 memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s,
2492 "intel_iommu", DMAR_REG_SIZE);
2493 sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem);
b5a280c0
LT		 2494 /* No corresponding destroy */
2495 s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
2496 g_free, g_free);
7df953bd
KO		 2497 s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal,
2498 g_free, g_free);
1da12ec4 2499 vtd_init(s);
621d983a
MA		 2500 sysbus_mmio_map(SYS_BUS_DEVICE(s), 0, Q35_HOST_BRIDGE_IOMMU_ADDR);
2501 pci_setup_iommu(bus, vtd_host_dma_iommu, dev);
cb135f59
PX		 2502 /* Pseudo address space under root PCI bus. */
2503 pcms->ioapic_as = vtd_host_dma_iommu(bus, s, Q35_PSEUDO_DEVFN_IOAPIC);
1da12ec4
LT		 2504}
2505
2506static void vtd_class_init(ObjectClass *klass, void *data)
2507{
2508 DeviceClass *dc = DEVICE_CLASS(klass);
1c7955c4 2509 X86IOMMUClass *x86_class = X86_IOMMU_CLASS(klass);
1da12ec4
LT		 2510
2511 dc->reset = vtd_reset;
1da12ec4
LT		 2512 dc->vmsd = &vtd_vmstate;
2513 dc->props = vtd_properties;
621d983a 2514 dc->hotpluggable = false;
1c7955c4 2515 x86_class->realize = vtd_realize;
8b5ed7df 2516 x86_class->int_remap = vtd_int_remap;
1da12ec4
LT		 2517}
2518
2519static const TypeInfo vtd_info = {
2520 .name = TYPE_INTEL_IOMMU_DEVICE,
1c7955c4 2521 .parent = TYPE_X86_IOMMU_DEVICE,
1da12ec4
LT		 2522 .instance_size = sizeof(IntelIOMMUState),
2523 .class_init = vtd_class_init,
2524};
2525
2526static void vtd_register_types(void)
2527{
2528 VTD_DPRINTF(GENERAL, "");
2529 type_register_static(&vtd_info);
2530}
2531
2532type_init(vtd_register_types)
QEMU mirror