]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/iommu/intel-iommu.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iommu/vt-d: Init QI before root entry is allocated
[mirror_ubuntu-zesty-kernel.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
9f10e5bf 18 * Joerg Roedel <jroedel@suse.de>
ba395927
KA		 19 */
20
9f10e5bf
JR		 21#define pr_fmt(fmt) "DMAR: " fmt
22
ba395927
KA		 23#include <linux/init.h>
24#include <linux/bitmap.h>
5e0d2a6f 25#include <linux/debugfs.h>
54485c30 26#include <linux/export.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
75f05569 35#include <linux/memory.h>
5e0d2a6f 36#include <linux/timer.h>
38717946 37#include <linux/iova.h>
5d450806 38#include <linux/iommu.h>
38717946 39#include <linux/intel-iommu.h>
134fac3f 40#include <linux/syscore_ops.h>
69575d38 41#include <linux/tboot.h>
adb2fe02 42#include <linux/dmi.h>
5cdede24 43#include <linux/pci-ats.h>
0ee332c1 44#include <linux/memblock.h>
36746436 45#include <linux/dma-contiguous.h>
8a8f422d 46#include <asm/irq_remapping.h>
ba395927 47#include <asm/cacheflush.h>
46a7fa27 48#include <asm/iommu.h>
ba395927 49
078e1ee2
JR		 50#include "irq_remapping.h"
51
5b6985ce
FY		 52#define ROOT_SIZE VTD_PAGE_SIZE
53#define CONTEXT_SIZE VTD_PAGE_SIZE
54
ba395927 55#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
18436afd 56#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
ba395927 57#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 58#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 59
60#define IOAPIC_RANGE_START (0xfee00000)
61#define IOAPIC_RANGE_END (0xfeefffff)
62#define IOVA_START_ADDR (0x1000)
63
64#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
65
4ed0d3e6 66#define MAX_AGAW_WIDTH 64
5c645b35 67#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 68
2ebe3151
DW		 69#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
70#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
71
72/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
73 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
74#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
75 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
76#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 77
1b722500
RM		 78/* IO virtual address start page frame number */
79#define IOVA_START_PFN (1)
80
f27be03b 81#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 82#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 83#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 84
df08cdc7
AM		 85/* page table handling */
86#define LEVEL_STRIDE (9)
87#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
88
6d1c56a9
OBC		 89/*
90 * This bitmap is used to advertise the page sizes our hardware support
91 * to the IOMMU core, which will then use this information to split
92 * physically contiguous memory regions it is mapping into page sizes
93 * that we support.
94 *
95 * Traditionally the IOMMU core just handed us the mappings directly,
96 * after making sure the size is an order of a 4KiB page and that the
97 * mapping has natural alignment.
98 *
99 * To retain this behavior, we currently advertise that we support
100 * all page sizes that are an order of 4KiB.
101 *
102 * If at some point we'd like to utilize the IOMMU core's new behavior,
103 * we could change this to advertise the real page sizes we support.
104 */
105#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
106
df08cdc7
AM		 107static inline int agaw_to_level(int agaw)
108{
109 return agaw + 2;
110}
111
112static inline int agaw_to_width(int agaw)
113{
5c645b35 114 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 115}
116
117static inline int width_to_agaw(int width)
118{
5c645b35 119 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 120}
121
122static inline unsigned int level_to_offset_bits(int level)
123{
124 return (level - 1) * LEVEL_STRIDE;
125}
126
127static inline int pfn_level_offset(unsigned long pfn, int level)
128{
129 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
130}
131
132static inline unsigned long level_mask(int level)
133{
134 return -1UL << level_to_offset_bits(level);
135}
136
137static inline unsigned long level_size(int level)
138{
139 return 1UL << level_to_offset_bits(level);
140}
141
142static inline unsigned long align_to_level(unsigned long pfn, int level)
143{
144 return (pfn + level_size(level) - 1) & level_mask(level);
145}
fd18de50 146
6dd9a7c7
YS		 147static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
148{
5c645b35 149 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 150}
151
dd4e8319
DW		 152/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
153 are never going to work. */
154static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
155{
156 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
157}
158
159static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
160{
161 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
162}
163static inline unsigned long page_to_dma_pfn(struct page *pg)
164{
165 return mm_to_dma_pfn(page_to_pfn(pg));
166}
167static inline unsigned long virt_to_dma_pfn(void *p)
168{
169 return page_to_dma_pfn(virt_to_page(p));
170}
171
d9630fe9
WH		 172/* global iommu list, set NULL for ignored DMAR units */
173static struct intel_iommu **g_iommus;
174
e0fc7e0b 175static void __init check_tylersburg_isoch(void);
9af88143
DW		 176static int rwbf_quirk;
177
b779260b
JC		 178/*
179 * set to 1 to panic kernel if can't successfully enable VT-d
180 * (used when kernel is launched w/ TXT)
181 */
182static int force_on = 0;
183
46b08e1a
MM		 184/*
185 * 0: Present
186 * 1-11: Reserved
187 * 12-63: Context Ptr (12 - (haw-1))
188 * 64-127: Reserved
189 */
190struct root_entry {
03ecc32c
DW		 191 u64 lo;
192 u64 hi;
46b08e1a
MM		 193};
194#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
46b08e1a 195
46b08e1a 196
7a8fc25e
MM		 197/*
198 * low 64 bits:
199 * 0: present
200 * 1: fault processing disable
201 * 2-3: translation type
202 * 12-63: address space root
203 * high 64 bits:
204 * 0-2: address width
205 * 3-6: aval
206 * 8-23: domain id
207 */
208struct context_entry {
209 u64 lo;
210 u64 hi;
211};
c07e7d21
MM		 212
213static inline bool context_present(struct context_entry *context)
214{
215 return (context->lo & 1);
216}
217static inline void context_set_present(struct context_entry *context)
218{
219 context->lo |= 1;
220}
221
222static inline void context_set_fault_enable(struct context_entry *context)
223{
224 context->lo &= (((u64)-1) << 2) | 1;
225}
226
c07e7d21
MM		 227static inline void context_set_translation_type(struct context_entry *context,
228 unsigned long value)
229{
230 context->lo &= (((u64)-1) << 4) | 3;
231 context->lo |= (value & 3) << 2;
232}
233
234static inline void context_set_address_root(struct context_entry *context,
235 unsigned long value)
236{
1a2262f9 237 context->lo &= ~VTD_PAGE_MASK;
c07e7d21
MM		 238 context->lo |= value & VTD_PAGE_MASK;
239}
240
241static inline void context_set_address_width(struct context_entry *context,
242 unsigned long value)
243{
244 context->hi |= value & 7;
245}
246
247static inline void context_set_domain_id(struct context_entry *context,
248 unsigned long value)
249{
250 context->hi |= (value & ((1 << 16) - 1)) << 8;
251}
252
253static inline void context_clear_entry(struct context_entry *context)
254{
255 context->lo = 0;
256 context->hi = 0;
257}
7a8fc25e 258
622ba12a
MM		 259/*
260 * 0: readable
261 * 1: writable
262 * 2-6: reserved
263 * 7: super page
9cf06697
SY		 264 * 8-10: available
265 * 11: snoop behavior
622ba12a
MM		 266 * 12-63: Host physcial address
267 */
268struct dma_pte {
269 u64 val;
270};
622ba12a 271
19c239ce
MM		 272static inline void dma_clear_pte(struct dma_pte *pte)
273{
274 pte->val = 0;
275}
276
19c239ce
MM		 277static inline u64 dma_pte_addr(struct dma_pte *pte)
278{
c85994e4
DW		 279#ifdef CONFIG_64BIT
280 return pte->val & VTD_PAGE_MASK;
281#else
282 /* Must have a full atomic 64-bit read */
1a8bd481 283 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 284#endif
19c239ce
MM		 285}
286
19c239ce
MM		 287static inline bool dma_pte_present(struct dma_pte *pte)
288{
289 return (pte->val & 3) != 0;
290}
622ba12a 291
4399c8bf
AK		 292static inline bool dma_pte_superpage(struct dma_pte *pte)
293{
c3c75eb7 294 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 295}
296
75e6bf96
DW		 297static inline int first_pte_in_page(struct dma_pte *pte)
298{
299 return !((unsigned long)pte & ~VTD_PAGE_MASK);
300}
301
2c2e2c38
FY		 302/*
303 * This domain is a statically identity mapping domain.
304 * 1. This domain creats a static 1:1 mapping to all usable memory.
305 * 2. It maps to each iommu if successful.
306 * 3. Each iommu mapps to this domain if successful.
307 */
19943b0e
DW		 308static struct dmar_domain *si_domain;
309static int hw_pass_through = 1;
2c2e2c38 310
1ce28feb
WH		 311/* domain represents a virtual machine, more than one devices
312 * across iommus may be owned in one domain, e.g. kvm guest.
313 */
ab8dfe25 314#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 315
2c2e2c38 316/* si_domain contains mulitple devices */
ab8dfe25 317#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 318
99126f7c
MM		 319struct dmar_domain {
320 int id; /* domain id */
4c923d47 321 int nid; /* node id */
78d8e704 322 DECLARE_BITMAP(iommu_bmp, DMAR_UNITS_SUPPORTED);
1b198bb0 323 /* bitmap of iommus this domain uses*/
99126f7c 324
00a77deb 325 struct list_head devices; /* all devices' list */
99126f7c
MM		 326 struct iova_domain iovad; /* iova's that belong to this domain */
327
328 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 329 int gaw; /* max guest address width */
330
331 /* adjusted guest address width, 0 is level 2 30-bit */
332 int agaw;
333
3b5410e7 334 int flags; /* flags to find out type of domain */
8e604097
WH		 335
336 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 337 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 338 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 339 int iommu_superpage;/* Level of superpages supported:
340 0 == 4KiB (no superpages), 1 == 2MiB,
341 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 342 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 343 u64 max_addr; /* maximum mapped address */
00a77deb
JR		 344
345 struct iommu_domain domain; /* generic domain data structure for
346 iommu core */
99126f7c
MM		 347};
348
a647dacb
MM		 349/* PCI domain-device relationship */
350struct device_domain_info {
351 struct list_head link; /* link to domain siblings */
352 struct list_head global; /* link to global list */
276dbf99 353 u8 bus; /* PCI bus number */
a647dacb 354 u8 devfn; /* PCI devfn number */
0bcb3e28 355 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 356 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 357 struct dmar_domain *domain; /* pointer to domain */
358};
359
b94e4117
JL		 360struct dmar_rmrr_unit {
361 struct list_head list; /* list of rmrr units */
362 struct acpi_dmar_header *hdr; /* ACPI header */
363 u64 base_address; /* reserved base address*/
364 u64 end_address; /* reserved end address */
832bd858 365 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 366 int devices_cnt; /* target device count */
367};
368
369struct dmar_atsr_unit {
370 struct list_head list; /* list of ATSR units */
371 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 372 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 373 int devices_cnt; /* target device count */
374 u8 include_all:1; /* include all ports */
375};
376
377static LIST_HEAD(dmar_atsr_units);
378static LIST_HEAD(dmar_rmrr_units);
379
380#define for_each_rmrr_units(rmrr) \
381 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
382
5e0d2a6f 383static void flush_unmaps_timeout(unsigned long data);
384
b707cb02 385static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 386
80b20dd8 387#define HIGH_WATER_MARK 250
388struct deferred_flush_tables {
389 int next;
390 struct iova *iova[HIGH_WATER_MARK];
391 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 392 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 393};
394
395static struct deferred_flush_tables *deferred_flush;
396
5e0d2a6f 397/* bitmap for indexing intel_iommus */
5e0d2a6f 398static int g_num_of_iommus;
399
400static DEFINE_SPINLOCK(async_umap_flush_lock);
401static LIST_HEAD(unmaps_to_do);
402
403static int timer_on;
404static long list_size;
5e0d2a6f 405
92d03cc8 406static void domain_exit(struct dmar_domain *domain);
ba395927 407static void domain_remove_dev_info(struct dmar_domain *domain);
b94e4117 408static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 409 struct device *dev);
92d03cc8 410static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 411 struct device *dev);
2a46ddf7
JL		 412static int domain_detach_iommu(struct dmar_domain *domain,
413 struct intel_iommu *iommu);
ba395927 414
d3f13810 415#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 416int dmar_disabled = 0;
417#else
418int dmar_disabled = 1;
d3f13810 419#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 420
8bc1f85c
ED		 421int intel_iommu_enabled = 0;
422EXPORT_SYMBOL_GPL(intel_iommu_enabled);
423
2d9e667e 424static int dmar_map_gfx = 1;
7d3b03ce 425static int dmar_forcedac;
5e0d2a6f 426static int intel_iommu_strict;
6dd9a7c7 427static int intel_iommu_superpage = 1;
c83b2f20
DW		 428static int intel_iommu_ecs = 1;
429
430/* We only actually use ECS when PASID support (on the new bit 40)
431 * is also advertised. Some early implementations — the ones with
432 * PASID support on bit 28 — have issues even when we *only* use
433 * extended root/context tables. */
434#define ecs_enabled(iommu) (intel_iommu_ecs && ecap_ecs(iommu->ecap) && \
435 ecap_pasid(iommu->ecap))
ba395927 436
c0771df8
DW		 437int intel_iommu_gfx_mapped;
438EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
439
ba395927
KA		 440#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
441static DEFINE_SPINLOCK(device_domain_lock);
442static LIST_HEAD(device_domain_list);
443
b22f6434 444static const struct iommu_ops intel_iommu_ops;
a8bcbb0d 445
00a77deb
JR		 446/* Convert generic 'struct iommu_domain to private struct dmar_domain */
447static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
448{
449 return container_of(dom, struct dmar_domain, domain);
450}
451
ba395927
KA		 452static int __init intel_iommu_setup(char *str)
453{
454 if (!str)
455 return -EINVAL;
456 while (*str) {
0cd5c3c8
KM		 457 if (!strncmp(str, "on", 2)) {
458 dmar_disabled = 0;
9f10e5bf 459 pr_info("IOMMU enabled\n");
0cd5c3c8 460 } else if (!strncmp(str, "off", 3)) {
ba395927 461 dmar_disabled = 1;
9f10e5bf 462 pr_info("IOMMU disabled\n");
ba395927
KA		 463 } else if (!strncmp(str, "igfx_off", 8)) {
464 dmar_map_gfx = 0;
9f10e5bf 465 pr_info("Disable GFX device mapping\n");
7d3b03ce 466 } else if (!strncmp(str, "forcedac", 8)) {
9f10e5bf 467 pr_info("Forcing DAC for PCI devices\n");
7d3b03ce 468 dmar_forcedac = 1;
5e0d2a6f 469 } else if (!strncmp(str, "strict", 6)) {
9f10e5bf 470 pr_info("Disable batched IOTLB flush\n");
5e0d2a6f 471 intel_iommu_strict = 1;
6dd9a7c7 472 } else if (!strncmp(str, "sp_off", 6)) {
9f10e5bf 473 pr_info("Disable supported super page\n");
6dd9a7c7 474 intel_iommu_superpage = 0;
c83b2f20
DW		 475 } else if (!strncmp(str, "ecs_off", 7)) {
476 printk(KERN_INFO
477 "Intel-IOMMU: disable extended context table support\n");
478 intel_iommu_ecs = 0;
ba395927
KA		 479 }
480
481 str += strcspn(str, ",");
482 while (*str == ',')
483 str++;
484 }
485 return 0;
486}
487__setup("intel_iommu=", intel_iommu_setup);
488
489static struct kmem_cache *iommu_domain_cache;
490static struct kmem_cache *iommu_devinfo_cache;
ba395927 491
4c923d47 492static inline void *alloc_pgtable_page(int node)
eb3fa7cb 493{
4c923d47
SS		 494 struct page *page;
495 void *vaddr = NULL;
eb3fa7cb 496
4c923d47
SS		 497 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
498 if (page)
499 vaddr = page_address(page);
eb3fa7cb 500 return vaddr;
ba395927
KA		 501}
502
503static inline void free_pgtable_page(void *vaddr)
504{
505 free_page((unsigned long)vaddr);
506}
507
508static inline void *alloc_domain_mem(void)
509{
354bb65e 510 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 511}
512
38717946 513static void free_domain_mem(void *vaddr)
ba395927
KA		 514{
515 kmem_cache_free(iommu_domain_cache, vaddr);
516}
517
518static inline void * alloc_devinfo_mem(void)
519{
354bb65e 520 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 521}
522
523static inline void free_devinfo_mem(void *vaddr)
524{
525 kmem_cache_free(iommu_devinfo_cache, vaddr);
526}
527
ab8dfe25
JL		 528static inline int domain_type_is_vm(struct dmar_domain *domain)
529{
530 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
531}
532
533static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
534{
535 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
536 DOMAIN_FLAG_STATIC_IDENTITY);
537}
1b573683 538
162d1b10
JL		 539static inline int domain_pfn_supported(struct dmar_domain *domain,
540 unsigned long pfn)
541{
542 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
543
544 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
545}
546
4ed0d3e6 547static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 548{
549 unsigned long sagaw;
550 int agaw = -1;
551
552 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 553 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 554 agaw >= 0; agaw--) {
555 if (test_bit(agaw, &sagaw))
556 break;
557 }
558
559 return agaw;
560}
561
4ed0d3e6
FY		 562/*
563 * Calculate max SAGAW for each iommu.
564 */
565int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
566{
567 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
568}
569
570/*
571 * calculate agaw for each iommu.
572 * "SAGAW" may be different across iommus, use a default agaw, and
573 * get a supported less agaw for iommus that don't support the default agaw.
574 */
575int iommu_calculate_agaw(struct intel_iommu *iommu)
576{
577 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
578}
579
2c2e2c38 580/* This functionin only returns single iommu in a domain */
8c11e798
WH		 581static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
582{
583 int iommu_id;
584
2c2e2c38 585 /* si_domain and vm domain should not get here. */
ab8dfe25 586 BUG_ON(domain_type_is_vm_or_si(domain));
1b198bb0 587 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 588 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
589 return NULL;
590
591 return g_iommus[iommu_id];
592}
593
8e604097
WH		 594static void domain_update_iommu_coherency(struct dmar_domain *domain)
595{
d0501960
DW		 596 struct dmar_drhd_unit *drhd;
597 struct intel_iommu *iommu;
2f119c78
QL		 598 bool found = false;
599 int i;
2e12bc29 600
d0501960 601 domain->iommu_coherency = 1;
8e604097 602
1b198bb0 603 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
2f119c78 604 found = true;
8e604097
WH		 605 if (!ecap_coherent(g_iommus[i]->ecap)) {
606 domain->iommu_coherency = 0;
607 break;
608 }
8e604097 609 }
d0501960
DW		 610 if (found)
611 return;
612
613 /* No hardware attached; use lowest common denominator */
614 rcu_read_lock();
615 for_each_active_iommu(iommu, drhd) {
616 if (!ecap_coherent(iommu->ecap)) {
617 domain->iommu_coherency = 0;
618 break;
619 }
620 }
621 rcu_read_unlock();
8e604097
WH		 622}
623
161f6934 624static int domain_update_iommu_snooping(struct intel_iommu *skip)
58c610bd 625{
161f6934
JL		 626 struct dmar_drhd_unit *drhd;
627 struct intel_iommu *iommu;
628 int ret = 1;
58c610bd 629
161f6934
JL		 630 rcu_read_lock();
631 for_each_active_iommu(iommu, drhd) {
632 if (iommu != skip) {
633 if (!ecap_sc_support(iommu->ecap)) {
634 ret = 0;
635 break;
636 }
58c610bd 637 }
58c610bd 638 }
161f6934
JL		 639 rcu_read_unlock();
640
641 return ret;
58c610bd
SY		 642}
643
161f6934 644static int domain_update_iommu_superpage(struct intel_iommu *skip)
6dd9a7c7 645{
8140a95d 646 struct dmar_drhd_unit *drhd;
161f6934 647 struct intel_iommu *iommu;
8140a95d 648 int mask = 0xf;
6dd9a7c7
YS		 649
650 if (!intel_iommu_superpage) {
161f6934 651 return 0;
6dd9a7c7
YS		 652 }
653
8140a95d 654 /* set iommu_superpage to the smallest common denominator */
0e242612 655 rcu_read_lock();
8140a95d 656 for_each_active_iommu(iommu, drhd) {
161f6934
JL		 657 if (iommu != skip) {
658 mask &= cap_super_page_val(iommu->cap);
659 if (!mask)
660 break;
6dd9a7c7
YS		 661 }
662 }
0e242612
JL		 663 rcu_read_unlock();
664
161f6934 665 return fls(mask);
6dd9a7c7
YS		 666}
667
58c610bd
SY		 668/* Some capabilities may be different across iommus */
669static void domain_update_iommu_cap(struct dmar_domain *domain)
670{
671 domain_update_iommu_coherency(domain);
161f6934
JL		 672 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
673 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
58c610bd
SY		 674}
675
03ecc32c
DW		 676static inline struct context_entry *iommu_context_addr(struct intel_iommu *iommu,
677 u8 bus, u8 devfn, int alloc)
678{
679 struct root_entry *root = &iommu->root_entry[bus];
680 struct context_entry *context;
681 u64 *entry;
682
c83b2f20 683 if (ecs_enabled(iommu)) {
03ecc32c
DW		 684 if (devfn >= 0x80) {
685 devfn -= 0x80;
686 entry = &root->hi;
687 }
688 devfn *= 2;
689 }
690 entry = &root->lo;
691 if (*entry & 1)
692 context = phys_to_virt(*entry & VTD_PAGE_MASK);
693 else {
694 unsigned long phy_addr;
695 if (!alloc)
696 return NULL;
697
698 context = alloc_pgtable_page(iommu->node);
699 if (!context)
700 return NULL;
701
702 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
703 phy_addr = virt_to_phys((void *)context);
704 *entry = phy_addr | 1;
705 __iommu_flush_cache(iommu, entry, sizeof(*entry));
706 }
707 return &context[devfn];
708}
709
4ed6a540
DW		 710static int iommu_dummy(struct device *dev)
711{
712 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
713}
714
156baca8 715static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 716{
717 struct dmar_drhd_unit *drhd = NULL;
b683b230 718 struct intel_iommu *iommu;
156baca8
DW		 719 struct device *tmp;
720 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 721 u16 segment = 0;
c7151a8d
WH		 722 int i;
723
4ed6a540
DW		 724 if (iommu_dummy(dev))
725 return NULL;
726
156baca8
DW		 727 if (dev_is_pci(dev)) {
728 pdev = to_pci_dev(dev);
729 segment = pci_domain_nr(pdev->bus);
ca5b74d2 730 } else if (has_acpi_companion(dev))
156baca8
DW		 731 dev = &ACPI_COMPANION(dev)->dev;
732
0e242612 733 rcu_read_lock();
b683b230 734 for_each_active_iommu(iommu, drhd) {
156baca8 735 if (pdev && segment != drhd->segment)
276dbf99 736 continue;
c7151a8d 737
b683b230 738 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 739 drhd->devices_cnt, i, tmp) {
740 if (tmp == dev) {
741 *bus = drhd->devices[i].bus;
742 *devfn = drhd->devices[i].devfn;
b683b230 743 goto out;
156baca8
DW		 744 }
745
746 if (!pdev || !dev_is_pci(tmp))
747 continue;
748
749 ptmp = to_pci_dev(tmp);
750 if (ptmp->subordinate &&
751 ptmp->subordinate->number <= pdev->bus->number &&
752 ptmp->subordinate->busn_res.end >= pdev->bus->number)
753 goto got_pdev;
924b6231 754 }
c7151a8d 755
156baca8
DW		 756 if (pdev && drhd->include_all) {
757 got_pdev:
758 *bus = pdev->bus->number;
759 *devfn = pdev->devfn;
b683b230 760 goto out;
156baca8 761 }
c7151a8d 762 }
b683b230 763 iommu = NULL;
156baca8 764 out:
0e242612 765 rcu_read_unlock();
c7151a8d 766
b683b230 767 return iommu;
c7151a8d
WH		 768}
769
5331fe6f
WH		 770static void domain_flush_cache(struct dmar_domain *domain,
771 void *addr, int size)
772{
773 if (!domain->iommu_coherency)
774 clflush_cache_range(addr, size);
775}
776
ba395927
KA		 777static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
778{
ba395927 779 struct context_entry *context;
03ecc32c 780 int ret = 0;
ba395927
KA		 781 unsigned long flags;
782
783 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c
DW		 784 context = iommu_context_addr(iommu, bus, devfn, 0);
785 if (context)
786 ret = context_present(context);
ba395927
KA		 787 spin_unlock_irqrestore(&iommu->lock, flags);
788 return ret;
789}
790
791static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
792{
ba395927
KA		 793 struct context_entry *context;
794 unsigned long flags;
795
796 spin_lock_irqsave(&iommu->lock, flags);
03ecc32c 797 context = iommu_context_addr(iommu, bus, devfn, 0);
ba395927 798 if (context) {
03ecc32c
DW		 799 context_clear_entry(context);
800 __iommu_flush_cache(iommu, context, sizeof(*context));
ba395927
KA		 801 }
802 spin_unlock_irqrestore(&iommu->lock, flags);
803}
804
805static void free_context_table(struct intel_iommu *iommu)
806{
ba395927
KA		 807 int i;
808 unsigned long flags;
809 struct context_entry *context;
810
811 spin_lock_irqsave(&iommu->lock, flags);
812 if (!iommu->root_entry) {
813 goto out;
814 }
815 for (i = 0; i < ROOT_ENTRY_NR; i++) {
03ecc32c 816 context = iommu_context_addr(iommu, i, 0, 0);
ba395927
KA		 817 if (context)
818 free_pgtable_page(context);
03ecc32c 819
c83b2f20 820 if (!ecs_enabled(iommu))
03ecc32c
DW		 821 continue;
822
823 context = iommu_context_addr(iommu, i, 0x80, 0);
824 if (context)
825 free_pgtable_page(context);
826
ba395927
KA		 827 }
828 free_pgtable_page(iommu->root_entry);
829 iommu->root_entry = NULL;
830out:
831 spin_unlock_irqrestore(&iommu->lock, flags);
832}
833
b026fd28 834static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 835 unsigned long pfn, int *target_level)
ba395927 836{
ba395927
KA		 837 struct dma_pte *parent, *pte = NULL;
838 int level = agaw_to_level(domain->agaw);
4399c8bf 839 int offset;
ba395927
KA		 840
841 BUG_ON(!domain->pgd);
f9423606 842
162d1b10 843 if (!domain_pfn_supported(domain, pfn))
f9423606
JS		 844 /* Address beyond IOMMU's addressing capabilities. */
845 return NULL;
846
ba395927
KA		 847 parent = domain->pgd;
848
5cf0a76f 849 while (1) {
ba395927
KA		 850 void *tmp_page;
851
b026fd28 852 offset = pfn_level_offset(pfn, level);
ba395927 853 pte = &parent[offset];
5cf0a76f 854 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 855 break;
5cf0a76f 856 if (level == *target_level)
ba395927
KA		 857 break;
858
19c239ce 859 if (!dma_pte_present(pte)) {
c85994e4
DW		 860 uint64_t pteval;
861
4c923d47 862 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 863
206a73c1 864 if (!tmp_page)
ba395927 865 return NULL;
206a73c1 866
c85994e4 867 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 868 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 869 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 870 /* Someone else set it while we were thinking; use theirs. */
871 free_pgtable_page(tmp_page);
effad4b5 872 else
c85994e4 873 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 874 }
5cf0a76f
DW		 875 if (level == 1)
876 break;
877
19c239ce 878 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 879 level--;
880 }
881
5cf0a76f
DW		 882 if (!*target_level)
883 *target_level = level;
884
ba395927
KA		 885 return pte;
886}
887
6dd9a7c7 888
ba395927 889/* return address's pte at specific level */
90dcfb5e
DW		 890static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
891 unsigned long pfn,
6dd9a7c7 892 int level, int *large_page)
ba395927
KA		 893{
894 struct dma_pte *parent, *pte = NULL;
895 int total = agaw_to_level(domain->agaw);
896 int offset;
897
898 parent = domain->pgd;
899 while (level <= total) {
90dcfb5e 900 offset = pfn_level_offset(pfn, total);
ba395927
KA		 901 pte = &parent[offset];
902 if (level == total)
903 return pte;
904
6dd9a7c7
YS		 905 if (!dma_pte_present(pte)) {
906 *large_page = total;
ba395927 907 break;
6dd9a7c7
YS		 908 }
909
e16922af 910 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 911 *large_page = total;
912 return pte;
913 }
914
19c239ce 915 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 916 total--;
917 }
918 return NULL;
919}
920
ba395927 921/* clear last level pte, a tlb flush should be followed */
5cf0a76f 922static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 923 unsigned long start_pfn,
924 unsigned long last_pfn)
ba395927 925{
6dd9a7c7 926 unsigned int large_page = 1;
310a5ab9 927 struct dma_pte *first_pte, *pte;
66eae846 928
162d1b10
JL		 929 BUG_ON(!domain_pfn_supported(domain, start_pfn));
930 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 931 BUG_ON(start_pfn > last_pfn);
ba395927 932
04b18e65 933 /* we don't need lock here; nobody else touches the iova range */
59c36286 934 do {
6dd9a7c7
YS		 935 large_page = 1;
936 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 937 if (!pte) {
6dd9a7c7 938 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 939 continue;
940 }
6dd9a7c7 941 do {
310a5ab9 942 dma_clear_pte(pte);
6dd9a7c7 943 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 944 pte++;
75e6bf96
DW		 945 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
946
310a5ab9
DW		 947 domain_flush_cache(domain, first_pte,
948 (void *)pte - (void *)first_pte);
59c36286
DW		 949
950 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 951}
952
3269ee0b
AW		 953static void dma_pte_free_level(struct dmar_domain *domain, int level,
954 struct dma_pte *pte, unsigned long pfn,
955 unsigned long start_pfn, unsigned long last_pfn)
956{
957 pfn = max(start_pfn, pfn);
958 pte = &pte[pfn_level_offset(pfn, level)];
959
960 do {
961 unsigned long level_pfn;
962 struct dma_pte *level_pte;
963
964 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
965 goto next;
966
967 level_pfn = pfn & level_mask(level - 1);
968 level_pte = phys_to_virt(dma_pte_addr(pte));
969
970 if (level > 2)
971 dma_pte_free_level(domain, level - 1, level_pte,
972 level_pfn, start_pfn, last_pfn);
973
974 /* If range covers entire pagetable, free it */
975 if (!(start_pfn > level_pfn ||
08336fd2 976 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 977 dma_clear_pte(pte);
978 domain_flush_cache(domain, pte, sizeof(*pte));
979 free_pgtable_page(level_pte);
980 }
981next:
982 pfn += level_size(level);
983 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
984}
985
ba395927
KA		 986/* free page table pages. last level pte should already be cleared */
987static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 988 unsigned long start_pfn,
989 unsigned long last_pfn)
ba395927 990{
162d1b10
JL		 991 BUG_ON(!domain_pfn_supported(domain, start_pfn));
992 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 993 BUG_ON(start_pfn > last_pfn);
ba395927 994
d41a4adb
JL		 995 dma_pte_clear_range(domain, start_pfn, last_pfn);
996
f3a0a52f 997 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 998 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
999 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 1000
ba395927 1001 /* free pgd */
d794dc9b 1002 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 1003 free_pgtable_page(domain->pgd);
1004 domain->pgd = NULL;
1005 }
1006}
1007
ea8ea460
DW		 1008/* When a page at a given level is being unlinked from its parent, we don't
1009 need to *modify* it at all. All we need to do is make a list of all the
1010 pages which can be freed just as soon as we've flushed the IOTLB and we
1011 know the hardware page-walk will no longer touch them.
1012 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1013 be freed. */
1014static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1015 int level, struct dma_pte *pte,
1016 struct page *freelist)
1017{
1018 struct page *pg;
1019
1020 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1021 pg->freelist = freelist;
1022 freelist = pg;
1023
1024 if (level == 1)
1025 return freelist;
1026
adeb2590
JL		 1027 pte = page_address(pg);
1028 do {
ea8ea460
DW		 1029 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1030 freelist = dma_pte_list_pagetables(domain, level - 1,
1031 pte, freelist);
adeb2590
JL		 1032 pte++;
1033 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1034
1035 return freelist;
1036}
1037
1038static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1039 struct dma_pte *pte, unsigned long pfn,
1040 unsigned long start_pfn,
1041 unsigned long last_pfn,
1042 struct page *freelist)
1043{
1044 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1045
1046 pfn = max(start_pfn, pfn);
1047 pte = &pte[pfn_level_offset(pfn, level)];
1048
1049 do {
1050 unsigned long level_pfn;
1051
1052 if (!dma_pte_present(pte))
1053 goto next;
1054
1055 level_pfn = pfn & level_mask(level);
1056
1057 /* If range covers entire pagetable, free it */
1058 if (start_pfn <= level_pfn &&
1059 last_pfn >= level_pfn + level_size(level) - 1) {
1060 /* These suborbinate page tables are going away entirely. Don't
1061 bother to clear them; we're just going to *free* them. */
1062 if (level > 1 && !dma_pte_superpage(pte))
1063 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1064
1065 dma_clear_pte(pte);
1066 if (!first_pte)
1067 first_pte = pte;
1068 last_pte = pte;
1069 } else if (level > 1) {
1070 /* Recurse down into a level that isn't *entirely* obsolete */
1071 freelist = dma_pte_clear_level(domain, level - 1,
1072 phys_to_virt(dma_pte_addr(pte)),
1073 level_pfn, start_pfn, last_pfn,
1074 freelist);
1075 }
1076next:
1077 pfn += level_size(level);
1078 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1079
1080 if (first_pte)
1081 domain_flush_cache(domain, first_pte,
1082 (void *)++last_pte - (void *)first_pte);
1083
1084 return freelist;
1085}
1086
1087/* We can't just free the pages because the IOMMU may still be walking
1088 the page tables, and may have cached the intermediate levels. The
1089 pages can only be freed after the IOTLB flush has been done. */
1090struct page *domain_unmap(struct dmar_domain *domain,
1091 unsigned long start_pfn,
1092 unsigned long last_pfn)
1093{
ea8ea460
DW		 1094 struct page *freelist = NULL;
1095
162d1b10
JL		 1096 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1097 BUG_ON(!domain_pfn_supported(domain, last_pfn));
ea8ea460
DW		 1098 BUG_ON(start_pfn > last_pfn);
1099
1100 /* we don't need lock here; nobody else touches the iova range */
1101 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1102 domain->pgd, 0, start_pfn, last_pfn, NULL);
1103
1104 /* free pgd */
1105 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1106 struct page *pgd_page = virt_to_page(domain->pgd);
1107 pgd_page->freelist = freelist;
1108 freelist = pgd_page;
1109
1110 domain->pgd = NULL;
1111 }
1112
1113 return freelist;
1114}
1115
1116void dma_free_pagelist(struct page *freelist)
1117{
1118 struct page *pg;
1119
1120 while ((pg = freelist)) {
1121 freelist = pg->freelist;
1122 free_pgtable_page(page_address(pg));
1123 }
1124}
1125
ba395927
KA		 1126/* iommu handling */
1127static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1128{
1129 struct root_entry *root;
1130 unsigned long flags;
1131
4c923d47 1132 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ffebeb46 1133 if (!root) {
9f10e5bf 1134 pr_err("Allocating root entry for %s failed\n",
ffebeb46 1135 iommu->name);
ba395927 1136 return -ENOMEM;
ffebeb46 1137 }
ba395927 1138
5b6985ce 1139 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1140
1141 spin_lock_irqsave(&iommu->lock, flags);
1142 iommu->root_entry = root;
1143 spin_unlock_irqrestore(&iommu->lock, flags);
1144
1145 return 0;
1146}
1147
ba395927
KA		 1148static void iommu_set_root_entry(struct intel_iommu *iommu)
1149{
03ecc32c 1150 u64 addr;
c416daa9 1151 u32 sts;
ba395927
KA		 1152 unsigned long flag;
1153
03ecc32c 1154 addr = virt_to_phys(iommu->root_entry);
c83b2f20 1155 if (ecs_enabled(iommu))
03ecc32c 1156 addr |= DMA_RTADDR_RTT;
ba395927 1157
1f5b3c3f 1158 raw_spin_lock_irqsave(&iommu->register_lock, flag);
03ecc32c 1159 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
ba395927 1160
c416daa9 1161 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1162
1163 /* Make sure hardware complete it */
1164 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1165 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1166
1f5b3c3f 1167 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1168}
1169
1170static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1171{
1172 u32 val;
1173 unsigned long flag;
1174
9af88143 1175 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1176 return;
ba395927 1177
1f5b3c3f 1178 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1179 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1180
1181 /* Make sure hardware complete it */
1182 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1183 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1184
1f5b3c3f 1185 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1186}
1187
1188/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1189static void __iommu_flush_context(struct intel_iommu *iommu,
1190 u16 did, u16 source_id, u8 function_mask,
1191 u64 type)
ba395927
KA		 1192{
1193 u64 val = 0;
1194 unsigned long flag;
1195
ba395927
KA		 1196 switch (type) {
1197 case DMA_CCMD_GLOBAL_INVL:
1198 val = DMA_CCMD_GLOBAL_INVL;
1199 break;
1200 case DMA_CCMD_DOMAIN_INVL:
1201 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1202 break;
1203 case DMA_CCMD_DEVICE_INVL:
1204 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1205 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1206 break;
1207 default:
1208 BUG();
1209 }
1210 val |= DMA_CCMD_ICC;
1211
1f5b3c3f 1212 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1213 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1214
1215 /* Make sure hardware complete it */
1216 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1217 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1218
1f5b3c3f 1219 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1220}
1221
ba395927 1222/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1223static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1224 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1225{
1226 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1227 u64 val = 0, val_iva = 0;
1228 unsigned long flag;
1229
ba395927
KA		 1230 switch (type) {
1231 case DMA_TLB_GLOBAL_FLUSH:
1232 /* global flush doesn't need set IVA_REG */
1233 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1234 break;
1235 case DMA_TLB_DSI_FLUSH:
1236 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1237 break;
1238 case DMA_TLB_PSI_FLUSH:
1239 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1240 /* IH bit is passed in as part of address */
ba395927
KA		 1241 val_iva = size_order | addr;
1242 break;
1243 default:
1244 BUG();
1245 }
1246 /* Note: set drain read/write */
1247#if 0
1248 /*
1249 * This is probably to be super secure.. Looks like we can
1250 * ignore it without any impact.
1251 */
1252 if (cap_read_drain(iommu->cap))
1253 val |= DMA_TLB_READ_DRAIN;
1254#endif
1255 if (cap_write_drain(iommu->cap))
1256 val |= DMA_TLB_WRITE_DRAIN;
1257
1f5b3c3f 1258 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1259 /* Note: Only uses first TLB reg currently */
1260 if (val_iva)
1261 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1262 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1263
1264 /* Make sure hardware complete it */
1265 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1266 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1267
1f5b3c3f 1268 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1269
1270 /* check IOTLB invalidation granularity */
1271 if (DMA_TLB_IAIG(val) == 0)
9f10e5bf 1272 pr_err("Flush IOTLB failed\n");
ba395927 1273 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
9f10e5bf 1274 pr_debug("TLB flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1275 (unsigned long long)DMA_TLB_IIRG(type),
1276 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1277}
1278
64ae892b
DW		 1279static struct device_domain_info *
1280iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1281 u8 bus, u8 devfn)
93a23a72 1282{
2f119c78 1283 bool found = false;
93a23a72
YZ		 1284 unsigned long flags;
1285 struct device_domain_info *info;
0bcb3e28 1286 struct pci_dev *pdev;
93a23a72
YZ		 1287
1288 if (!ecap_dev_iotlb_support(iommu->ecap))
1289 return NULL;
1290
1291 if (!iommu->qi)
1292 return NULL;
1293
1294 spin_lock_irqsave(&device_domain_lock, flags);
1295 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1296 if (info->iommu == iommu && info->bus == bus &&
1297 info->devfn == devfn) {
2f119c78 1298 found = true;
93a23a72
YZ		 1299 break;
1300 }
1301 spin_unlock_irqrestore(&device_domain_lock, flags);
1302
0bcb3e28 1303 if (!found || !info->dev || !dev_is_pci(info->dev))
93a23a72
YZ		 1304 return NULL;
1305
0bcb3e28
DW		 1306 pdev = to_pci_dev(info->dev);
1307
1308 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
93a23a72
YZ		 1309 return NULL;
1310
0bcb3e28 1311 if (!dmar_find_matched_atsr_unit(pdev))
93a23a72
YZ		 1312 return NULL;
1313
93a23a72
YZ		 1314 return info;
1315}
1316
1317static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1318{
0bcb3e28 1319 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1320 return;
1321
0bcb3e28 1322 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
93a23a72
YZ		 1323}
1324
1325static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1326{
0bcb3e28
DW		 1327 if (!info->dev || !dev_is_pci(info->dev) ||
1328 !pci_ats_enabled(to_pci_dev(info->dev)))
93a23a72
YZ		 1329 return;
1330
0bcb3e28 1331 pci_disable_ats(to_pci_dev(info->dev));
93a23a72
YZ		 1332}
1333
1334static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1335 u64 addr, unsigned mask)
1336{
1337 u16 sid, qdep;
1338 unsigned long flags;
1339 struct device_domain_info *info;
1340
1341 spin_lock_irqsave(&device_domain_lock, flags);
1342 list_for_each_entry(info, &domain->devices, link) {
0bcb3e28
DW		 1343 struct pci_dev *pdev;
1344 if (!info->dev || !dev_is_pci(info->dev))
1345 continue;
1346
1347 pdev = to_pci_dev(info->dev);
1348 if (!pci_ats_enabled(pdev))
93a23a72
YZ		 1349 continue;
1350
1351 sid = info->bus << 8 | info->devfn;
0bcb3e28 1352 qdep = pci_ats_queue_depth(pdev);
93a23a72
YZ		 1353 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1354 }
1355 spin_unlock_irqrestore(&device_domain_lock, flags);
1356}
1357
1f0ef2aa 1358static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
ea8ea460 1359 unsigned long pfn, unsigned int pages, int ih, int map)
ba395927 1360{
9dd2fe89 1361 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1362 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1363
ba395927
KA		 1364 BUG_ON(pages == 0);
1365
ea8ea460
DW		 1366 if (ih)
1367 ih = 1 << 6;
ba395927 1368 /*
9dd2fe89
YZ		 1369 * Fallback to domain selective flush if no PSI support or the size is
1370 * too big.
ba395927
KA		 1371 * PSI requires page size to be 2 ^ x, and the base address is naturally
1372 * aligned to the size
1373 */
9dd2fe89
YZ		 1374 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1375 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1376 DMA_TLB_DSI_FLUSH);
9dd2fe89 1377 else
ea8ea460 1378 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1379 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1380
1381 /*
82653633
NA		 1382 * In caching mode, changes of pages from non-present to present require
1383 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1384 */
82653633 1385 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1386 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1387}
1388
f8bab735 1389static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1390{
1391 u32 pmen;
1392 unsigned long flags;
1393
1f5b3c3f 1394 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1395 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1396 pmen &= ~DMA_PMEN_EPM;
1397 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1398
1399 /* wait for the protected region status bit to clear */
1400 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1401 readl, !(pmen & DMA_PMEN_PRS), pmen);
1402
1f5b3c3f 1403 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1404}
1405
2a41ccee 1406static void iommu_enable_translation(struct intel_iommu *iommu)
ba395927
KA		 1407{
1408 u32 sts;
1409 unsigned long flags;
1410
1f5b3c3f 1411 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1412 iommu->gcmd |= DMA_GCMD_TE;
1413 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1414
1415 /* Make sure hardware complete it */
1416 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1417 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1418
1f5b3c3f 1419 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1420}
1421
2a41ccee 1422static void iommu_disable_translation(struct intel_iommu *iommu)
ba395927
KA		 1423{
1424 u32 sts;
1425 unsigned long flag;
1426
1f5b3c3f 1427 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1428 iommu->gcmd &= ~DMA_GCMD_TE;
1429 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1430
1431 /* Make sure hardware complete it */
1432 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1433 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1434
1f5b3c3f 1435 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1436}
1437
3460a6d9 1438
ba395927
KA		 1439static int iommu_init_domains(struct intel_iommu *iommu)
1440{
1441 unsigned long ndomains;
1442 unsigned long nlongs;
1443
1444 ndomains = cap_ndoms(iommu->cap);
9f10e5bf
JR		 1445 pr_debug("%s: Number of Domains supported <%ld>\n",
1446 iommu->name, ndomains);
ba395927
KA		 1447 nlongs = BITS_TO_LONGS(ndomains);
1448
94a91b50
DD		 1449 spin_lock_init(&iommu->lock);
1450
ba395927
KA		 1451 /* TBD: there might be 64K domains,
1452 * consider other allocation for future chip
1453 */
1454 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1455 if (!iommu->domain_ids) {
9f10e5bf
JR		 1456 pr_err("%s: Allocating domain id array failed\n",
1457 iommu->name);
ba395927
KA		 1458 return -ENOMEM;
1459 }
1460 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1461 GFP_KERNEL);
1462 if (!iommu->domains) {
9f10e5bf
JR		 1463 pr_err("%s: Allocating domain array failed\n",
1464 iommu->name);
852bdb04
JL		 1465 kfree(iommu->domain_ids);
1466 iommu->domain_ids = NULL;
ba395927
KA		 1467 return -ENOMEM;
1468 }
1469
1470 /*
1471 * if Caching mode is set, then invalid translations are tagged
1472 * with domainid 0. Hence we need to pre-allocate it.
1473 */
1474 if (cap_caching_mode(iommu->cap))
1475 set_bit(0, iommu->domain_ids);
1476 return 0;
1477}
ba395927 1478
ffebeb46 1479static void disable_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1480{
1481 struct dmar_domain *domain;
2a46ddf7 1482 int i;
ba395927 1483
94a91b50 1484 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1485 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
a4eaa86c
JL		 1486 /*
1487 * Domain id 0 is reserved for invalid translation
1488 * if hardware supports caching mode.
1489 */
1490 if (cap_caching_mode(iommu->cap) && i == 0)
1491 continue;
1492
94a91b50
DD		 1493 domain = iommu->domains[i];
1494 clear_bit(i, iommu->domain_ids);
129ad281
JL		 1495 if (domain_detach_iommu(domain, iommu) == 0 &&
1496 !domain_type_is_vm(domain))
92d03cc8 1497 domain_exit(domain);
5e98c4b1 1498 }
ba395927
KA		 1499 }
1500
1501 if (iommu->gcmd & DMA_GCMD_TE)
1502 iommu_disable_translation(iommu);
ffebeb46 1503}
ba395927 1504
ffebeb46
JL		 1505static void free_dmar_iommu(struct intel_iommu *iommu)
1506{
1507 if ((iommu->domains) && (iommu->domain_ids)) {
1508 kfree(iommu->domains);
1509 kfree(iommu->domain_ids);
1510 iommu->domains = NULL;
1511 iommu->domain_ids = NULL;
1512 }
ba395927 1513
d9630fe9
WH		 1514 g_iommus[iommu->seq_id] = NULL;
1515
ba395927
KA		 1516 /* free context mapping */
1517 free_context_table(iommu);
ba395927
KA		 1518}
1519
ab8dfe25 1520static struct dmar_domain *alloc_domain(int flags)
ba395927 1521{
92d03cc8
JL		 1522 /* domain id for virtual machine, it won't be set in context */
1523 static atomic_t vm_domid = ATOMIC_INIT(0);
ba395927 1524 struct dmar_domain *domain;
ba395927
KA		 1525
1526 domain = alloc_domain_mem();
1527 if (!domain)
1528 return NULL;
1529
ab8dfe25 1530 memset(domain, 0, sizeof(*domain));
4c923d47 1531 domain->nid = -1;
ab8dfe25 1532 domain->flags = flags;
92d03cc8
JL		 1533 spin_lock_init(&domain->iommu_lock);
1534 INIT_LIST_HEAD(&domain->devices);
ab8dfe25 1535 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
92d03cc8 1536 domain->id = atomic_inc_return(&vm_domid);
2c2e2c38
FY		 1537
1538 return domain;
1539}
1540
fb170fb4
JL		 1541static int __iommu_attach_domain(struct dmar_domain *domain,
1542 struct intel_iommu *iommu)
2c2e2c38
FY		 1543{
1544 int num;
1545 unsigned long ndomains;
2c2e2c38 1546
ba395927 1547 ndomains = cap_ndoms(iommu->cap);
ba395927 1548 num = find_first_zero_bit(iommu->domain_ids, ndomains);
fb170fb4
JL		 1549 if (num < ndomains) {
1550 set_bit(num, iommu->domain_ids);
1551 iommu->domains[num] = domain;
1552 } else {
1553 num = -ENOSPC;
ba395927
KA		 1554 }
1555
fb170fb4
JL		 1556 return num;
1557}
1558
1559static int iommu_attach_domain(struct dmar_domain *domain,
1560 struct intel_iommu *iommu)
1561{
1562 int num;
1563 unsigned long flags;
1564
1565 spin_lock_irqsave(&iommu->lock, flags);
1566 num = __iommu_attach_domain(domain, iommu);
44bde614 1567 spin_unlock_irqrestore(&iommu->lock, flags);
fb170fb4 1568 if (num < 0)
9f10e5bf 1569 pr_err("%s: No free domain ids\n", iommu->name);
ba395927 1570
fb170fb4 1571 return num;
ba395927
KA		 1572}
1573
44bde614
JL		 1574static int iommu_attach_vm_domain(struct dmar_domain *domain,
1575 struct intel_iommu *iommu)
1576{
1577 int num;
1578 unsigned long ndomains;
1579
1580 ndomains = cap_ndoms(iommu->cap);
1581 for_each_set_bit(num, iommu->domain_ids, ndomains)
1582 if (iommu->domains[num] == domain)
1583 return num;
1584
1585 return __iommu_attach_domain(domain, iommu);
1586}
1587
2c2e2c38
FY		 1588static void iommu_detach_domain(struct dmar_domain *domain,
1589 struct intel_iommu *iommu)
ba395927
KA		 1590{
1591 unsigned long flags;
2c2e2c38 1592 int num, ndomains;
ba395927 1593
8c11e798 1594 spin_lock_irqsave(&iommu->lock, flags);
fb170fb4
JL		 1595 if (domain_type_is_vm_or_si(domain)) {
1596 ndomains = cap_ndoms(iommu->cap);
1597 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1598 if (iommu->domains[num] == domain) {
1599 clear_bit(num, iommu->domain_ids);
1600 iommu->domains[num] = NULL;
1601 break;
1602 }
2c2e2c38 1603 }
fb170fb4
JL		 1604 } else {
1605 clear_bit(domain->id, iommu->domain_ids);
1606 iommu->domains[domain->id] = NULL;
2c2e2c38 1607 }
8c11e798 1608 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1609}
1610
fb170fb4
JL		 1611static void domain_attach_iommu(struct dmar_domain *domain,
1612 struct intel_iommu *iommu)
1613{
1614 unsigned long flags;
1615
1616 spin_lock_irqsave(&domain->iommu_lock, flags);
1617 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1618 domain->iommu_count++;
1619 if (domain->iommu_count == 1)
1620 domain->nid = iommu->node;
1621 domain_update_iommu_cap(domain);
1622 }
1623 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1624}
1625
1626static int domain_detach_iommu(struct dmar_domain *domain,
1627 struct intel_iommu *iommu)
1628{
1629 unsigned long flags;
1630 int count = INT_MAX;
1631
1632 spin_lock_irqsave(&domain->iommu_lock, flags);
1633 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1634 count = --domain->iommu_count;
1635 domain_update_iommu_cap(domain);
1636 }
1637 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1638
1639 return count;
1640}
1641
ba395927 1642static struct iova_domain reserved_iova_list;
8a443df4 1643static struct lock_class_key reserved_rbtree_key;
ba395927 1644
51a63e67 1645static int dmar_init_reserved_ranges(void)
ba395927
KA		 1646{
1647 struct pci_dev *pdev = NULL;
1648 struct iova *iova;
1649 int i;
ba395927 1650
0fb5fe87
RM		 1651 init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN,
1652 DMA_32BIT_PFN);
ba395927 1653
8a443df4
MG		 1654 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1655 &reserved_rbtree_key);
1656
ba395927
KA		 1657 /* IOAPIC ranges shouldn't be accessed by DMA */
1658 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1659 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1660 if (!iova) {
9f10e5bf 1661 pr_err("Reserve IOAPIC range failed\n");
51a63e67
JC		 1662 return -ENODEV;
1663 }
ba395927
KA		 1664
1665 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1666 for_each_pci_dev(pdev) {
1667 struct resource *r;
1668
1669 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1670 r = &pdev->resource[i];
1671 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1672 continue;
1a4a4551
DW		 1673 iova = reserve_iova(&reserved_iova_list,
1674 IOVA_PFN(r->start),
1675 IOVA_PFN(r->end));
51a63e67 1676 if (!iova) {
9f10e5bf 1677 pr_err("Reserve iova failed\n");
51a63e67
JC		 1678 return -ENODEV;
1679 }
ba395927
KA		 1680 }
1681 }
51a63e67 1682 return 0;
ba395927
KA		 1683}
1684
1685static void domain_reserve_special_ranges(struct dmar_domain *domain)
1686{
1687 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1688}
1689
1690static inline int guestwidth_to_adjustwidth(int gaw)
1691{
1692 int agaw;
1693 int r = (gaw - 12) % 9;
1694
1695 if (r == 0)
1696 agaw = gaw;
1697 else
1698 agaw = gaw + 9 - r;
1699 if (agaw > 64)
1700 agaw = 64;
1701 return agaw;
1702}
1703
1704static int domain_init(struct dmar_domain *domain, int guest_width)
1705{
1706 struct intel_iommu *iommu;
1707 int adjust_width, agaw;
1708 unsigned long sagaw;
1709
0fb5fe87
RM		 1710 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
1711 DMA_32BIT_PFN);
ba395927
KA		 1712 domain_reserve_special_ranges(domain);
1713
1714 /* calculate AGAW */
8c11e798 1715 iommu = domain_get_iommu(domain);
ba395927
KA		 1716 if (guest_width > cap_mgaw(iommu->cap))
1717 guest_width = cap_mgaw(iommu->cap);
1718 domain->gaw = guest_width;
1719 adjust_width = guestwidth_to_adjustwidth(guest_width);
1720 agaw = width_to_agaw(adjust_width);
1721 sagaw = cap_sagaw(iommu->cap);
1722 if (!test_bit(agaw, &sagaw)) {
1723 /* hardware doesn't support it, choose a bigger one */
9f10e5bf 1724 pr_debug("Hardware doesn't support agaw %d\n", agaw);
ba395927
KA		 1725 agaw = find_next_bit(&sagaw, 5, agaw);
1726 if (agaw >= 5)
1727 return -ENODEV;
1728 }
1729 domain->agaw = agaw;
ba395927 1730
8e604097
WH		 1731 if (ecap_coherent(iommu->ecap))
1732 domain->iommu_coherency = 1;
1733 else
1734 domain->iommu_coherency = 0;
1735
58c610bd
SY		 1736 if (ecap_sc_support(iommu->ecap))
1737 domain->iommu_snooping = 1;
1738 else
1739 domain->iommu_snooping = 0;
1740
214e39aa
DW		 1741 if (intel_iommu_superpage)
1742 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1743 else
1744 domain->iommu_superpage = 0;
1745
4c923d47 1746 domain->nid = iommu->node;
c7151a8d 1747
ba395927 1748 /* always allocate the top pgd */
4c923d47 1749 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1750 if (!domain->pgd)
1751 return -ENOMEM;
5b6985ce 1752 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1753 return 0;
1754}
1755
1756static void domain_exit(struct dmar_domain *domain)
1757{
ea8ea460 1758 struct page *freelist = NULL;
71684406 1759 int i;
ba395927
KA		 1760
1761 /* Domain 0 is reserved, so dont process it */
1762 if (!domain)
1763 return;
1764
7b668357
AW		 1765 /* Flush any lazy unmaps that may reference this domain */
1766 if (!intel_iommu_strict)
1767 flush_unmaps_timeout(0);
1768
92d03cc8 1769 /* remove associated devices */
ba395927 1770 domain_remove_dev_info(domain);
92d03cc8 1771
ba395927
KA		 1772 /* destroy iovas */
1773 put_iova_domain(&domain->iovad);
ba395927 1774
ea8ea460 1775 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1776
92d03cc8 1777 /* clear attached or cached domains */
0e242612 1778 rcu_read_lock();
71684406
AW		 1779 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus)
1780 iommu_detach_domain(domain, g_iommus[i]);
0e242612 1781 rcu_read_unlock();
2c2e2c38 1782
ea8ea460
DW		 1783 dma_free_pagelist(freelist);
1784
ba395927
KA		 1785 free_domain_mem(domain);
1786}
1787
64ae892b
DW		 1788static int domain_context_mapping_one(struct dmar_domain *domain,
1789 struct intel_iommu *iommu,
1790 u8 bus, u8 devfn, int translation)
ba395927
KA		 1791{
1792 struct context_entry *context;
ba395927 1793 unsigned long flags;
ea6606b0 1794 struct dma_pte *pgd;
ea6606b0
WH		 1795 int id;
1796 int agaw;
93a23a72 1797 struct device_domain_info *info = NULL;
ba395927
KA		 1798
1799 pr_debug("Set context mapping for %02x:%02x.%d\n",
1800 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1801
ba395927 1802 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1803 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1804 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1805
03ecc32c
DW		 1806 spin_lock_irqsave(&iommu->lock, flags);
1807 context = iommu_context_addr(iommu, bus, devfn, 1);
1808 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1809 if (!context)
1810 return -ENOMEM;
1811 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1812 if (context_present(context)) {
ba395927
KA		 1813 spin_unlock_irqrestore(&iommu->lock, flags);
1814 return 0;
1815 }
1816
ea6606b0
WH		 1817 id = domain->id;
1818 pgd = domain->pgd;
1819
ab8dfe25 1820 if (domain_type_is_vm_or_si(domain)) {
44bde614
JL		 1821 if (domain_type_is_vm(domain)) {
1822 id = iommu_attach_vm_domain(domain, iommu);
fb170fb4 1823 if (id < 0) {
ea6606b0 1824 spin_unlock_irqrestore(&iommu->lock, flags);
9f10e5bf 1825 pr_err("%s: No free domain ids\n", iommu->name);
ea6606b0
WH		 1826 return -EFAULT;
1827 }
ea6606b0
WH		 1828 }
1829
1830 /* Skip top levels of page tables for
1831 * iommu which has less agaw than default.
1672af11 1832 * Unnecessary for PT mode.
ea6606b0 1833 */
1672af11
CW		 1834 if (translation != CONTEXT_TT_PASS_THROUGH) {
1835 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1836 pgd = phys_to_virt(dma_pte_addr(pgd));
1837 if (!dma_pte_present(pgd)) {
1838 spin_unlock_irqrestore(&iommu->lock, flags);
1839 return -ENOMEM;
1840 }
ea6606b0
WH		 1841 }
1842 }
1843 }
1844
1845 context_set_domain_id(context, id);
4ed0d3e6 1846
93a23a72 1847 if (translation != CONTEXT_TT_PASS_THROUGH) {
64ae892b 1848 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
93a23a72
YZ		 1849 translation = info ? CONTEXT_TT_DEV_IOTLB :
1850 CONTEXT_TT_MULTI_LEVEL;
1851 }
4ed0d3e6
FY		 1852 /*
1853 * In pass through mode, AW must be programmed to indicate the largest
1854 * AGAW value supported by hardware. And ASR is ignored by hardware.
1855 */
93a23a72 1856 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1857 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1858 else {
1859 context_set_address_root(context, virt_to_phys(pgd));
1860 context_set_address_width(context, iommu->agaw);
1861 }
4ed0d3e6
FY		 1862
1863 context_set_translation_type(context, translation);
c07e7d21
MM		 1864 context_set_fault_enable(context);
1865 context_set_present(context);
5331fe6f 1866 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1867
4c25a2c1
DW		 1868 /*
1869 * It's a non-present to present mapping. If hardware doesn't cache
1870 * non-present entry we only need to flush the write-buffer. If the
1871 * _does_ cache non-present entries, then it does so in the special
1872 * domain #0, which we have to flush:
1873 */
1874 if (cap_caching_mode(iommu->cap)) {
1875 iommu->flush.flush_context(iommu, 0,
1876 (((u16)bus) << 8) | devfn,
1877 DMA_CCMD_MASK_NOBIT,
1878 DMA_CCMD_DEVICE_INVL);
18fd779a 1879 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1880 } else {
ba395927 1881 iommu_flush_write_buffer(iommu);
4c25a2c1 1882 }
93a23a72 1883 iommu_enable_dev_iotlb(info);
ba395927 1884 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d 1885
fb170fb4
JL		 1886 domain_attach_iommu(domain, iommu);
1887
ba395927
KA		 1888 return 0;
1889}
1890
579305f7
AW		 1891struct domain_context_mapping_data {
1892 struct dmar_domain *domain;
1893 struct intel_iommu *iommu;
1894 int translation;
1895};
1896
1897static int domain_context_mapping_cb(struct pci_dev *pdev,
1898 u16 alias, void *opaque)
1899{
1900 struct domain_context_mapping_data *data = opaque;
1901
1902 return domain_context_mapping_one(data->domain, data->iommu,
1903 PCI_BUS_NUM(alias), alias & 0xff,
1904 data->translation);
1905}
1906
ba395927 1907static int
e1f167f3
DW		 1908domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1909 int translation)
ba395927 1910{
64ae892b 1911 struct intel_iommu *iommu;
156baca8 1912 u8 bus, devfn;
579305f7 1913 struct domain_context_mapping_data data;
64ae892b 1914
e1f167f3 1915 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 1916 if (!iommu)
1917 return -ENODEV;
ba395927 1918
579305f7
AW		 1919 if (!dev_is_pci(dev))
1920 return domain_context_mapping_one(domain, iommu, bus, devfn,
4ed0d3e6 1921 translation);
579305f7
AW		 1922
1923 data.domain = domain;
1924 data.iommu = iommu;
1925 data.translation = translation;
1926
1927 return pci_for_each_dma_alias(to_pci_dev(dev),
1928 &domain_context_mapping_cb, &data);
1929}
1930
1931static int domain_context_mapped_cb(struct pci_dev *pdev,
1932 u16 alias, void *opaque)
1933{
1934 struct intel_iommu *iommu = opaque;
1935
1936 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 1937}
1938
e1f167f3 1939static int domain_context_mapped(struct device *dev)
ba395927 1940{
5331fe6f 1941 struct intel_iommu *iommu;
156baca8 1942 u8 bus, devfn;
5331fe6f 1943
e1f167f3 1944 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 1945 if (!iommu)
1946 return -ENODEV;
ba395927 1947
579305f7
AW		 1948 if (!dev_is_pci(dev))
1949 return device_context_mapped(iommu, bus, devfn);
e1f167f3 1950
579305f7
AW		 1951 return !pci_for_each_dma_alias(to_pci_dev(dev),
1952 domain_context_mapped_cb, iommu);
ba395927
KA		 1953}
1954
f532959b
FY		 1955/* Returns a number of VTD pages, but aligned to MM page size */
1956static inline unsigned long aligned_nrpages(unsigned long host_addr,
1957 size_t size)
1958{
1959 host_addr &= ~PAGE_MASK;
1960 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1961}
1962
6dd9a7c7
YS		 1963/* Return largest possible superpage level for a given mapping */
1964static inline int hardware_largepage_caps(struct dmar_domain *domain,
1965 unsigned long iov_pfn,
1966 unsigned long phy_pfn,
1967 unsigned long pages)
1968{
1969 int support, level = 1;
1970 unsigned long pfnmerge;
1971
1972 support = domain->iommu_superpage;
1973
1974 /* To use a large page, the virtual *and* physical addresses
1975 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1976 of them will mean we have to use smaller pages. So just
1977 merge them and check both at once. */
1978 pfnmerge = iov_pfn | phy_pfn;
1979
1980 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1981 pages >>= VTD_STRIDE_SHIFT;
1982 if (!pages)
1983 break;
1984 pfnmerge >>= VTD_STRIDE_SHIFT;
1985 level++;
1986 support--;
1987 }
1988 return level;
1989}
1990
9051aa02
DW		 1991static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1992 struct scatterlist *sg, unsigned long phys_pfn,
1993 unsigned long nr_pages, int prot)
e1605495
DW		 1994{
1995 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1996 phys_addr_t uninitialized_var(pteval);
cc4f14aa 1997 unsigned long sg_res = 0;
6dd9a7c7
YS		 1998 unsigned int largepage_lvl = 0;
1999 unsigned long lvl_pages = 0;
e1605495 2000
162d1b10 2001 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
e1605495
DW		 2002
2003 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2004 return -EINVAL;
2005
2006 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
2007
cc4f14aa
JL		 2008 if (!sg) {
2009 sg_res = nr_pages;
9051aa02
DW		 2010 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2011 }
2012
6dd9a7c7 2013 while (nr_pages > 0) {
c85994e4
DW		 2014 uint64_t tmp;
2015
e1605495 2016 if (!sg_res) {
f532959b 2017 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 2018 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2019 sg->dma_length = sg->length;
2020 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 2021 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2022 }
6dd9a7c7 2023
e1605495 2024 if (!pte) {
6dd9a7c7
YS		 2025 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2026
5cf0a76f 2027 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2028 if (!pte)
2029 return -ENOMEM;
6dd9a7c7 2030 /* It is large page*/
6491d4d0 2031 if (largepage_lvl > 1) {
6dd9a7c7 2032 pteval |= DMA_PTE_LARGE_PAGE;
d41a4adb
JL		 2033 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2034 /*
2035 * Ensure that old small page tables are
2036 * removed to make room for superpage,
2037 * if they exist.
2038 */
6491d4d0 2039 dma_pte_free_pagetable(domain, iov_pfn,
d41a4adb 2040 iov_pfn + lvl_pages - 1);
6491d4d0 2041 } else {
6dd9a7c7 2042 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2043 }
6dd9a7c7 2044
e1605495
DW		 2045 }
2046 /* We don't need lock here, nobody else
2047 * touches the iova range
2048 */
7766a3fb 2049 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2050 if (tmp) {
1bf20f0d 2051 static int dumps = 5;
9f10e5bf
JR		 2052 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2053 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2054 if (dumps) {
2055 dumps--;
2056 debug_dma_dump_mappings(NULL);
2057 }
2058 WARN_ON(1);
2059 }
6dd9a7c7
YS		 2060
2061 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2062
2063 BUG_ON(nr_pages < lvl_pages);
2064 BUG_ON(sg_res < lvl_pages);
2065
2066 nr_pages -= lvl_pages;
2067 iov_pfn += lvl_pages;
2068 phys_pfn += lvl_pages;
2069 pteval += lvl_pages * VTD_PAGE_SIZE;
2070 sg_res -= lvl_pages;
2071
2072 /* If the next PTE would be the first in a new page, then we
2073 need to flush the cache on the entries we've just written.
2074 And then we'll need to recalculate 'pte', so clear it and
2075 let it get set again in the if (!pte) block above.
2076
2077 If we're done (!nr_pages) we need to flush the cache too.
2078
2079 Also if we've been setting superpages, we may need to
2080 recalculate 'pte' and switch back to smaller pages for the
2081 end of the mapping, if the trailing size is not enough to
2082 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2083 pte++;
6dd9a7c7
YS		 2084 if (!nr_pages || first_pte_in_page(pte) ||
2085 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2086 domain_flush_cache(domain, first_pte,
2087 (void *)pte - (void *)first_pte);
2088 pte = NULL;
2089 }
6dd9a7c7
YS		 2090
2091 if (!sg_res && nr_pages)
e1605495
DW		 2092 sg = sg_next(sg);
2093 }
2094 return 0;
2095}
2096
9051aa02
DW		 2097static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2098 struct scatterlist *sg, unsigned long nr_pages,
2099 int prot)
ba395927 2100{
9051aa02
DW		 2101 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2102}
6f6a00e4 2103
9051aa02
DW		 2104static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2105 unsigned long phys_pfn, unsigned long nr_pages,
2106 int prot)
2107{
2108 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2109}
2110
c7151a8d 2111static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2112{
c7151a8d
WH		 2113 if (!iommu)
2114 return;
8c11e798
WH		 2115
2116 clear_context_table(iommu, bus, devfn);
2117 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2118 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2119 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2120}
2121
109b9b04
DW		 2122static inline void unlink_domain_info(struct device_domain_info *info)
2123{
2124 assert_spin_locked(&device_domain_lock);
2125 list_del(&info->link);
2126 list_del(&info->global);
2127 if (info->dev)
0bcb3e28 2128 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2129}
2130
ba395927
KA		 2131static void domain_remove_dev_info(struct dmar_domain *domain)
2132{
3a74ca01 2133 struct device_domain_info *info, *tmp;
fb170fb4 2134 unsigned long flags;
ba395927
KA		 2135
2136 spin_lock_irqsave(&device_domain_lock, flags);
3a74ca01 2137 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
109b9b04 2138 unlink_domain_info(info);
ba395927
KA		 2139 spin_unlock_irqrestore(&device_domain_lock, flags);
2140
93a23a72 2141 iommu_disable_dev_iotlb(info);
7c7faa11 2142 iommu_detach_dev(info->iommu, info->bus, info->devfn);
ba395927 2143
ab8dfe25 2144 if (domain_type_is_vm(domain)) {
7c7faa11 2145 iommu_detach_dependent_devices(info->iommu, info->dev);
fb170fb4 2146 domain_detach_iommu(domain, info->iommu);
92d03cc8
JL		 2147 }
2148
2149 free_devinfo_mem(info);
ba395927
KA		 2150 spin_lock_irqsave(&device_domain_lock, flags);
2151 }
2152 spin_unlock_irqrestore(&device_domain_lock, flags);
2153}
2154
2155/*
2156 * find_domain
1525a29a 2157 * Note: we use struct device->archdata.iommu stores the info
ba395927 2158 */
1525a29a 2159static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2160{
2161 struct device_domain_info *info;
2162
2163 /* No lock here, assumes no domain exit in normal case */
1525a29a 2164 info = dev->archdata.iommu;
ba395927
KA		 2165 if (info)
2166 return info->domain;
2167 return NULL;
2168}
2169
5a8f40e8 2170static inline struct device_domain_info *
745f2586
JL		 2171dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2172{
2173 struct device_domain_info *info;
2174
2175 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2176 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2177 info->devfn == devfn)
5a8f40e8 2178 return info;
745f2586
JL		 2179
2180 return NULL;
2181}
2182
5a8f40e8 2183static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
41e80dca 2184 int bus, int devfn,
b718cd3d
DW		 2185 struct device *dev,
2186 struct dmar_domain *domain)
745f2586 2187{
5a8f40e8 2188 struct dmar_domain *found = NULL;
745f2586
JL		 2189 struct device_domain_info *info;
2190 unsigned long flags;
2191
2192 info = alloc_devinfo_mem();
2193 if (!info)
b718cd3d 2194 return NULL;
745f2586 2195
745f2586
JL		 2196 info->bus = bus;
2197 info->devfn = devfn;
2198 info->dev = dev;
2199 info->domain = domain;
5a8f40e8 2200 info->iommu = iommu;
745f2586
JL		 2201
2202 spin_lock_irqsave(&device_domain_lock, flags);
2203 if (dev)
0bcb3e28 2204 found = find_domain(dev);
5a8f40e8
DW		 2205 else {
2206 struct device_domain_info *info2;
41e80dca 2207 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
5a8f40e8
DW		 2208 if (info2)
2209 found = info2->domain;
2210 }
745f2586
JL		 2211 if (found) {
2212 spin_unlock_irqrestore(&device_domain_lock, flags);
2213 free_devinfo_mem(info);
b718cd3d
DW		 2214 /* Caller must free the original domain */
2215 return found;
745f2586
JL		 2216 }
2217
b718cd3d
DW		 2218 list_add(&info->link, &domain->devices);
2219 list_add(&info->global, &device_domain_list);
2220 if (dev)
2221 dev->archdata.iommu = info;
2222 spin_unlock_irqrestore(&device_domain_lock, flags);
2223
2224 return domain;
745f2586
JL		 2225}
2226
579305f7
AW		 2227static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2228{
2229 *(u16 *)opaque = alias;
2230 return 0;
2231}
2232
ba395927 2233/* domain is initialized */
146922ec 2234static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2235{
579305f7
AW		 2236 struct dmar_domain *domain, *tmp;
2237 struct intel_iommu *iommu;
5a8f40e8 2238 struct device_domain_info *info;
579305f7 2239 u16 dma_alias;
ba395927 2240 unsigned long flags;
aa4d066a 2241 u8 bus, devfn;
ba395927 2242
146922ec 2243 domain = find_domain(dev);
ba395927
KA		 2244 if (domain)
2245 return domain;
2246
579305f7
AW		 2247 iommu = device_to_iommu(dev, &bus, &devfn);
2248 if (!iommu)
2249 return NULL;
2250
146922ec
DW		 2251 if (dev_is_pci(dev)) {
2252 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2253
579305f7
AW		 2254 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2255
2256 spin_lock_irqsave(&device_domain_lock, flags);
2257 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2258 PCI_BUS_NUM(dma_alias),
2259 dma_alias & 0xff);
2260 if (info) {
2261 iommu = info->iommu;
2262 domain = info->domain;
5a8f40e8 2263 }
579305f7 2264 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2265
579305f7
AW		 2266 /* DMA alias already has a domain, uses it */
2267 if (info)
2268 goto found_domain;
2269 }
ba395927 2270
146922ec 2271 /* Allocate and initialize new domain for the device */
ab8dfe25 2272 domain = alloc_domain(0);
745f2586 2273 if (!domain)
579305f7 2274 return NULL;
44bde614
JL		 2275 domain->id = iommu_attach_domain(domain, iommu);
2276 if (domain->id < 0) {
2fe9723d 2277 free_domain_mem(domain);
579305f7 2278 return NULL;
2c2e2c38 2279 }
fb170fb4 2280 domain_attach_iommu(domain, iommu);
579305f7
AW		 2281 if (domain_init(domain, gaw)) {
2282 domain_exit(domain);
2283 return NULL;
2c2e2c38 2284 }
ba395927 2285
579305f7
AW		 2286 /* register PCI DMA alias device */
2287 if (dev_is_pci(dev)) {
2288 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2289 dma_alias & 0xff, NULL, domain);
2290
2291 if (!tmp || tmp != domain) {
2292 domain_exit(domain);
2293 domain = tmp;
2294 }
2295
b718cd3d 2296 if (!domain)
579305f7 2297 return NULL;
ba395927
KA		 2298 }
2299
2300found_domain:
579305f7
AW		 2301 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2302
2303 if (!tmp || tmp != domain) {
2304 domain_exit(domain);
2305 domain = tmp;
2306 }
b718cd3d
DW		 2307
2308 return domain;
ba395927
KA		 2309}
2310
2c2e2c38 2311static int iommu_identity_mapping;
e0fc7e0b
DW		 2312#define IDENTMAP_ALL 1
2313#define IDENTMAP_GFX 2
2314#define IDENTMAP_AZALIA 4
2c2e2c38 2315
b213203e
DW		 2316static int iommu_domain_identity_map(struct dmar_domain *domain,
2317 unsigned long long start,
2318 unsigned long long end)
ba395927 2319{
c5395d5c
DW		 2320 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2321 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2322
2323 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2324 dma_to_mm_pfn(last_vpfn))) {
9f10e5bf 2325 pr_err("Reserving iova failed\n");
b213203e 2326 return -ENOMEM;
ba395927
KA		 2327 }
2328
c5395d5c
DW		 2329 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2330 start, end, domain->id);
ba395927
KA		 2331 /*
2332 * RMRR range might have overlap with physical memory range,
2333 * clear it first
2334 */
c5395d5c 2335 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2336
c5395d5c
DW		 2337 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2338 last_vpfn - first_vpfn + 1,
61df7443 2339 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2340}
2341
0b9d9753 2342static int iommu_prepare_identity_map(struct device *dev,
b213203e
DW		 2343 unsigned long long start,
2344 unsigned long long end)
2345{
2346 struct dmar_domain *domain;
2347 int ret;
2348
0b9d9753 2349 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2350 if (!domain)
2351 return -ENOMEM;
2352
19943b0e
DW		 2353 /* For _hardware_ passthrough, don't bother. But for software
2354 passthrough, we do it anyway -- it may indicate a memory
2355 range which is reserved in E820, so which didn't get set
2356 up to start with in si_domain */
2357 if (domain == si_domain && hw_pass_through) {
9f10e5bf
JR		 2358 pr_warn("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2359 dev_name(dev), start, end);
19943b0e
DW		 2360 return 0;
2361 }
2362
9f10e5bf
JR		 2363 pr_info("Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2364 dev_name(dev), start, end);
2365
5595b528
DW		 2366 if (end < start) {
2367 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2368 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2369 dmi_get_system_info(DMI_BIOS_VENDOR),
2370 dmi_get_system_info(DMI_BIOS_VERSION),
2371 dmi_get_system_info(DMI_PRODUCT_VERSION));
2372 ret = -EIO;
2373 goto error;
2374 }
2375
2ff729f5
DW		 2376 if (end >> agaw_to_width(domain->agaw)) {
2377 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2378 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2379 agaw_to_width(domain->agaw),
2380 dmi_get_system_info(DMI_BIOS_VENDOR),
2381 dmi_get_system_info(DMI_BIOS_VERSION),
2382 dmi_get_system_info(DMI_PRODUCT_VERSION));
2383 ret = -EIO;
2384 goto error;
2385 }
19943b0e 2386
b213203e 2387 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2388 if (ret)
2389 goto error;
2390
2391 /* context entry init */
0b9d9753 2392 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2393 if (ret)
2394 goto error;
2395
2396 return 0;
2397
2398 error:
ba395927
KA		 2399 domain_exit(domain);
2400 return ret;
ba395927
KA		 2401}
2402
2403static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2404 struct device *dev)
ba395927 2405{
0b9d9753 2406 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2407 return 0;
0b9d9753
DW		 2408 return iommu_prepare_identity_map(dev, rmrr->base_address,
2409 rmrr->end_address);
ba395927
KA		 2410}
2411
d3f13810 2412#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2413static inline void iommu_prepare_isa(void)
2414{
2415 struct pci_dev *pdev;
2416 int ret;
2417
2418 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2419 if (!pdev)
2420 return;
2421
9f10e5bf 2422 pr_info("Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2423 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2424
2425 if (ret)
9f10e5bf 2426 pr_err("Failed to create 0-16MiB identity map - floppy might not work\n");
49a0429e 2427
9b27e82d 2428 pci_dev_put(pdev);
49a0429e
KA		 2429}
2430#else
2431static inline void iommu_prepare_isa(void)
2432{
2433 return;
2434}
d3f13810 2435#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2436
2c2e2c38 2437static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2438
071e1374 2439static int __init si_domain_init(int hw)
2c2e2c38
FY		 2440{
2441 struct dmar_drhd_unit *drhd;
2442 struct intel_iommu *iommu;
c7ab48d2 2443 int nid, ret = 0;
44bde614 2444 bool first = true;
2c2e2c38 2445
ab8dfe25 2446 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2447 if (!si_domain)
2448 return -EFAULT;
2449
2c2e2c38
FY		 2450 for_each_active_iommu(iommu, drhd) {
2451 ret = iommu_attach_domain(si_domain, iommu);
fb170fb4 2452 if (ret < 0) {
2c2e2c38
FY		 2453 domain_exit(si_domain);
2454 return -EFAULT;
44bde614
JL		 2455 } else if (first) {
2456 si_domain->id = ret;
2457 first = false;
2458 } else if (si_domain->id != ret) {
2459 domain_exit(si_domain);
2460 return -EFAULT;
2c2e2c38 2461 }
fb170fb4 2462 domain_attach_iommu(si_domain, iommu);
2c2e2c38
FY		 2463 }
2464
2465 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2466 domain_exit(si_domain);
2467 return -EFAULT;
2468 }
2469
9f10e5bf 2470 pr_debug("Identity mapping domain is domain %d\n",
9544c003 2471 si_domain->id);
2c2e2c38 2472
19943b0e
DW		 2473 if (hw)
2474 return 0;
2475
c7ab48d2 2476 for_each_online_node(nid) {
5dfe8660
TH		 2477 unsigned long start_pfn, end_pfn;
2478 int i;
2479
2480 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2481 ret = iommu_domain_identity_map(si_domain,
2482 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2483 if (ret)
2484 return ret;
2485 }
c7ab48d2
DW		 2486 }
2487
2c2e2c38
FY		 2488 return 0;
2489}
2490
9b226624 2491static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2492{
2493 struct device_domain_info *info;
2494
2495 if (likely(!iommu_identity_mapping))
2496 return 0;
2497
9b226624 2498 info = dev->archdata.iommu;
cb452a40
MT		 2499 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2500 return (info->domain == si_domain);
2c2e2c38 2501
2c2e2c38
FY		 2502 return 0;
2503}
2504
2505static int domain_add_dev_info(struct dmar_domain *domain,
5913c9bf 2506 struct device *dev, int translation)
2c2e2c38 2507{
0ac72664 2508 struct dmar_domain *ndomain;
5a8f40e8 2509 struct intel_iommu *iommu;
156baca8 2510 u8 bus, devfn;
5fe60f4e 2511 int ret;
2c2e2c38 2512
5913c9bf 2513 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2514 if (!iommu)
2515 return -ENODEV;
2516
5913c9bf 2517 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2518 if (ndomain != domain)
2519 return -EBUSY;
2c2e2c38 2520
5913c9bf 2521 ret = domain_context_mapping(domain, dev, translation);
e2ad23d0 2522 if (ret) {
5913c9bf 2523 domain_remove_one_dev_info(domain, dev);
e2ad23d0
DW		 2524 return ret;
2525 }
2526
2c2e2c38
FY		 2527 return 0;
2528}
2529
0b9d9753 2530static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2531{
2532 struct dmar_rmrr_unit *rmrr;
832bd858 2533 struct device *tmp;
ea2447f7
TM		 2534 int i;
2535
0e242612 2536 rcu_read_lock();
ea2447f7 2537 for_each_rmrr_units(rmrr) {
b683b230
JL		 2538 /*
2539 * Return TRUE if this RMRR contains the device that
2540 * is passed in.
2541 */
2542 for_each_active_dev_scope(rmrr->devices,
2543 rmrr->devices_cnt, i, tmp)
0b9d9753 2544 if (tmp == dev) {
0e242612 2545 rcu_read_unlock();
ea2447f7 2546 return true;
b683b230 2547 }
ea2447f7 2548 }
0e242612 2549 rcu_read_unlock();
ea2447f7
TM		 2550 return false;
2551}
2552
c875d2c1
AW		 2553/*
2554 * There are a couple cases where we need to restrict the functionality of
2555 * devices associated with RMRRs. The first is when evaluating a device for
2556 * identity mapping because problems exist when devices are moved in and out
2557 * of domains and their respective RMRR information is lost. This means that
2558 * a device with associated RMRRs will never be in a "passthrough" domain.
2559 * The second is use of the device through the IOMMU API. This interface
2560 * expects to have full control of the IOVA space for the device. We cannot
2561 * satisfy both the requirement that RMRR access is maintained and have an
2562 * unencumbered IOVA space. We also have no ability to quiesce the device's
2563 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2564 * We therefore prevent devices associated with an RMRR from participating in
2565 * the IOMMU API, which eliminates them from device assignment.
2566 *
2567 * In both cases we assume that PCI USB devices with RMRRs have them largely
2568 * for historical reasons and that the RMRR space is not actively used post
2569 * boot. This exclusion may change if vendors begin to abuse it.
18436afd
DW		 2570 *
2571 * The same exception is made for graphics devices, with the requirement that
2572 * any use of the RMRR regions will be torn down before assigning the device
2573 * to a guest.
c875d2c1
AW		 2574 */
2575static bool device_is_rmrr_locked(struct device *dev)
2576{
2577 if (!device_has_rmrr(dev))
2578 return false;
2579
2580 if (dev_is_pci(dev)) {
2581 struct pci_dev *pdev = to_pci_dev(dev);
2582
18436afd 2583 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
c875d2c1
AW		 2584 return false;
2585 }
2586
2587 return true;
2588}
2589
3bdb2591 2590static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2591{
ea2447f7 2592
3bdb2591
DW		 2593 if (dev_is_pci(dev)) {
2594 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2595
c875d2c1 2596 if (device_is_rmrr_locked(dev))
3bdb2591 2597 return 0;
e0fc7e0b 2598
3bdb2591
DW		 2599 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2600 return 1;
e0fc7e0b 2601
3bdb2591
DW		 2602 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2603 return 1;
6941af28 2604
3bdb2591 2605 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2606 return 0;
3bdb2591
DW		 2607
2608 /*
2609 * We want to start off with all devices in the 1:1 domain, and
2610 * take them out later if we find they can't access all of memory.
2611 *
2612 * However, we can't do this for PCI devices behind bridges,
2613 * because all PCI devices behind the same bridge will end up
2614 * with the same source-id on their transactions.
2615 *
2616 * Practically speaking, we can't change things around for these
2617 * devices at run-time, because we can't be sure there'll be no
2618 * DMA transactions in flight for any of their siblings.
2619 *
2620 * So PCI devices (unless they're on the root bus) as well as
2621 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2622 * the 1:1 domain, just in _case_ one of their siblings turns out
2623 * not to be able to map all of memory.
2624 */
2625 if (!pci_is_pcie(pdev)) {
2626 if (!pci_is_root_bus(pdev->bus))
2627 return 0;
2628 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2629 return 0;
2630 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2631 return 0;
3bdb2591
DW		 2632 } else {
2633 if (device_has_rmrr(dev))
2634 return 0;
2635 }
3dfc813d 2636
3bdb2591 2637 /*
3dfc813d 2638 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2639 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2640 * take them out of the 1:1 domain later.
2641 */
8fcc5372
CW		 2642 if (!startup) {
2643 /*
2644 * If the device's dma_mask is less than the system's memory
2645 * size then this is not a candidate for identity mapping.
2646 */
3bdb2591 2647 u64 dma_mask = *dev->dma_mask;
8fcc5372 2648
3bdb2591
DW		 2649 if (dev->coherent_dma_mask &&
2650 dev->coherent_dma_mask < dma_mask)
2651 dma_mask = dev->coherent_dma_mask;
8fcc5372 2652
3bdb2591 2653 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2654 }
6941af28
DW		 2655
2656 return 1;
2657}
2658
cf04eee8
DW		 2659static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2660{
2661 int ret;
2662
2663 if (!iommu_should_identity_map(dev, 1))
2664 return 0;
2665
2666 ret = domain_add_dev_info(si_domain, dev,
2667 hw ? CONTEXT_TT_PASS_THROUGH :
2668 CONTEXT_TT_MULTI_LEVEL);
2669 if (!ret)
9f10e5bf
JR		 2670 pr_info("%s identity mapping for device %s\n",
2671 hw ? "Hardware" : "Software", dev_name(dev));
cf04eee8
DW		 2672 else if (ret == -ENODEV)
2673 /* device not associated with an iommu */
2674 ret = 0;
2675
2676 return ret;
2677}
2678
2679
071e1374 2680static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2681{
2c2e2c38 2682 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2683 struct dmar_drhd_unit *drhd;
2684 struct intel_iommu *iommu;
2685 struct device *dev;
2686 int i;
2687 int ret = 0;
2c2e2c38 2688
19943b0e 2689 ret = si_domain_init(hw);
2c2e2c38
FY		 2690 if (ret)
2691 return -EFAULT;
2692
2c2e2c38 2693 for_each_pci_dev(pdev) {
cf04eee8
DW		 2694 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2695 if (ret)
2696 return ret;
2697 }
2698
2699 for_each_active_iommu(iommu, drhd)
2700 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2701 struct acpi_device_physical_node *pn;
2702 struct acpi_device *adev;
2703
2704 if (dev->bus != &acpi_bus_type)
2705 continue;
2706
2707 adev= to_acpi_device(dev);
2708 mutex_lock(&adev->physical_node_lock);
2709 list_for_each_entry(pn, &adev->physical_node_list, node) {
2710 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2711 if (ret)
2712 break;
eae460b6 2713 }
cf04eee8
DW		 2714 mutex_unlock(&adev->physical_node_lock);
2715 if (ret)
2716 return ret;
62edf5dc 2717 }
2c2e2c38
FY		 2718
2719 return 0;
2720}
2721
ffebeb46
JL		 2722static void intel_iommu_init_qi(struct intel_iommu *iommu)
2723{
2724 /*
2725 * Start from the sane iommu hardware state.
2726 * If the queued invalidation is already initialized by us
2727 * (for example, while enabling interrupt-remapping) then
2728 * we got the things already rolling from a sane state.
2729 */
2730 if (!iommu->qi) {
2731 /*
2732 * Clear any previous faults.
2733 */
2734 dmar_fault(-1, iommu);
2735 /*
2736 * Disable queued invalidation if supported and already enabled
2737 * before OS handover.
2738 */
2739 dmar_disable_qi(iommu);
2740 }
2741
2742 if (dmar_enable_qi(iommu)) {
2743 /*
2744 * Queued Invalidate not enabled, use Register Based Invalidate
2745 */
2746 iommu->flush.flush_context = __iommu_flush_context;
2747 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
9f10e5bf 2748 pr_info("%s: Using Register based invalidation\n",
ffebeb46
JL		 2749 iommu->name);
2750 } else {
2751 iommu->flush.flush_context = qi_flush_context;
2752 iommu->flush.flush_iotlb = qi_flush_iotlb;
9f10e5bf 2753 pr_info("%s: Using Queued invalidation\n", iommu->name);
ffebeb46
JL		 2754 }
2755}
2756
b779260b 2757static int __init init_dmars(void)
ba395927
KA		 2758{
2759 struct dmar_drhd_unit *drhd;
2760 struct dmar_rmrr_unit *rmrr;
832bd858 2761 struct device *dev;
ba395927 2762 struct intel_iommu *iommu;
9d783ba0 2763 int i, ret;
2c2e2c38 2764
ba395927
KA		 2765 /*
2766 * for each drhd
2767 * allocate root
2768 * initialize and program root entry to not present
2769 * endfor
2770 */
2771 for_each_drhd_unit(drhd) {
5e0d2a6f 2772 /*
2773 * lock not needed as this is only incremented in the single
2774 * threaded kernel __init code path all other access are read
2775 * only
2776 */
78d8e704 2777 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
1b198bb0
MT		 2778 g_num_of_iommus++;
2779 continue;
2780 }
9f10e5bf 2781 pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
5e0d2a6f 2782 }
2783
ffebeb46
JL		 2784 /* Preallocate enough resources for IOMMU hot-addition */
2785 if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
2786 g_num_of_iommus = DMAR_UNITS_SUPPORTED;
2787
d9630fe9
WH		 2788 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2789 GFP_KERNEL);
2790 if (!g_iommus) {
9f10e5bf 2791 pr_err("Allocating global iommu array failed\n");
d9630fe9
WH		 2792 ret = -ENOMEM;
2793 goto error;
2794 }
2795
80b20dd8 2796 deferred_flush = kzalloc(g_num_of_iommus *
2797 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2798 if (!deferred_flush) {
5e0d2a6f 2799 ret = -ENOMEM;
989d51fc 2800 goto free_g_iommus;
5e0d2a6f 2801 }
2802
7c919779 2803 for_each_active_iommu(iommu, drhd) {
d9630fe9 2804 g_iommus[iommu->seq_id] = iommu;
ba395927 2805
b63d80d1
JR		 2806 intel_iommu_init_qi(iommu);
2807
e61d98d8
SS		 2808 ret = iommu_init_domains(iommu);
2809 if (ret)
989d51fc 2810 goto free_iommu;
e61d98d8 2811
ba395927
KA		 2812 /*
2813 * TBD:
2814 * we could share the same root & context tables
25985edc 2815 * among all IOMMU's. Need to Split it later.
ba395927
KA		 2816 */
2817 ret = iommu_alloc_root_entry(iommu);
ffebeb46 2818 if (ret)
989d51fc 2819 goto free_iommu;
4ed0d3e6 2820 if (!ecap_pass_through(iommu->ecap))
19943b0e 2821 hw_pass_through = 0;
ba395927
KA		 2822 }
2823
19943b0e 2824 if (iommu_pass_through)
e0fc7e0b
DW		 2825 iommu_identity_mapping |= IDENTMAP_ALL;
2826
d3f13810 2827#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 2828 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2829#endif
e0fc7e0b
DW		 2830
2831 check_tylersburg_isoch();
2832
ba395927 2833 /*
19943b0e
DW		 2834 * If pass through is not set or not enabled, setup context entries for
2835 * identity mappings for rmrr, gfx, and isa and may fall back to static
2836 * identity mapping if iommu_identity_mapping is set.
ba395927 2837 */
19943b0e
DW		 2838 if (iommu_identity_mapping) {
2839 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2840 if (ret) {
9f10e5bf 2841 pr_crit("Failed to setup IOMMU pass-through\n");
989d51fc 2842 goto free_iommu;
ba395927
KA		 2843 }
2844 }
ba395927 2845 /*
19943b0e
DW		 2846 * For each rmrr
2847 * for each dev attached to rmrr
2848 * do
2849 * locate drhd for dev, alloc domain for dev
2850 * allocate free domain
2851 * allocate page table entries for rmrr
2852 * if context not allocated for bus
2853 * allocate and init context
2854 * set present in root table for this bus
2855 * init context with domain, translation etc
2856 * endfor
2857 * endfor
ba395927 2858 */
9f10e5bf 2859 pr_info("Setting RMRR:\n");
19943b0e 2860 for_each_rmrr_units(rmrr) {
b683b230
JL		 2861 /* some BIOS lists non-exist devices in DMAR table. */
2862 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 2863 i, dev) {
0b9d9753 2864 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e 2865 if (ret)
9f10e5bf 2866 pr_err("Mapping reserved region failed\n");
ba395927 2867 }
4ed0d3e6 2868 }
49a0429e 2869
19943b0e
DW		 2870 iommu_prepare_isa();
2871
ba395927
KA		 2872 /*
2873 * for each drhd
2874 * enable fault log
2875 * global invalidate context cache
2876 * global invalidate iotlb
2877 * enable translation
2878 */
7c919779 2879 for_each_iommu(iommu, drhd) {
51a63e67
JC		 2880 if (drhd->ignored) {
2881 /*
2882 * we always have to disable PMRs or DMA may fail on
2883 * this device
2884 */
2885 if (force_on)
7c919779 2886 iommu_disable_protect_mem_regions(iommu);
ba395927 2887 continue;
51a63e67 2888 }
ba395927
KA		 2889
2890 iommu_flush_write_buffer(iommu);
2891
3460a6d9
KA		 2892 ret = dmar_set_interrupt(iommu);
2893 if (ret)
989d51fc 2894 goto free_iommu;
3460a6d9 2895
ba395927
KA		 2896 iommu_set_root_entry(iommu);
2897
4c25a2c1 2898 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2899 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2a41ccee 2900 iommu_enable_translation(iommu);
b94996c9 2901 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2902 }
2903
2904 return 0;
989d51fc
JL		 2905
2906free_iommu:
ffebeb46
JL		 2907 for_each_active_iommu(iommu, drhd) {
2908 disable_dmar_iommu(iommu);
a868e6b7 2909 free_dmar_iommu(iommu);
ffebeb46 2910 }
9bdc531e 2911 kfree(deferred_flush);
989d51fc 2912free_g_iommus:
d9630fe9 2913 kfree(g_iommus);
989d51fc 2914error:
ba395927
KA		 2915 return ret;
2916}
2917
5a5e02a6 2918/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2919static struct iova *intel_alloc_iova(struct device *dev,
2920 struct dmar_domain *domain,
2921 unsigned long nrpages, uint64_t dma_mask)
ba395927 2922{
ba395927 2923 struct iova *iova = NULL;
ba395927 2924
875764de
DW		 2925 /* Restrict dma_mask to the width that the iommu can handle */
2926 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2927
2928 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2929 /*
2930 * First try to allocate an io virtual address in
284901a9 2931 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2932 * from higher range
ba395927 2933 */
875764de
DW		 2934 iova = alloc_iova(&domain->iovad, nrpages,
2935 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2936 if (iova)
2937 return iova;
2938 }
2939 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2940 if (unlikely(!iova)) {
9f10e5bf 2941 pr_err("Allocating %ld-page iova for %s failed",
207e3592 2942 nrpages, dev_name(dev));
f76aec76
KA		 2943 return NULL;
2944 }
2945
2946 return iova;
2947}
2948
d4b709f4 2949static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76
KA		 2950{
2951 struct dmar_domain *domain;
2952 int ret;
2953
d4b709f4 2954 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 2955 if (!domain) {
9f10e5bf 2956 pr_err("Allocating domain for %s failed\n",
d4b709f4 2957 dev_name(dev));
4fe05bbc 2958 return NULL;
ba395927
KA		 2959 }
2960
2961 /* make sure context mapping is ok */
d4b709f4
DW		 2962 if (unlikely(!domain_context_mapped(dev))) {
2963 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
f76aec76 2964 if (ret) {
9f10e5bf 2965 pr_err("Domain context map for %s failed\n",
d4b709f4 2966 dev_name(dev));
4fe05bbc 2967 return NULL;
f76aec76 2968 }
ba395927
KA		 2969 }
2970
f76aec76
KA		 2971 return domain;
2972}
2973
d4b709f4 2974static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 2975{
2976 struct device_domain_info *info;
2977
2978 /* No lock here, assumes no domain exit in normal case */
d4b709f4 2979 info = dev->archdata.iommu;
147202aa
DW		 2980 if (likely(info))
2981 return info->domain;
2982
2983 return __get_valid_domain_for_dev(dev);
2984}
2985
ecb509ec 2986/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 2987static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 2988{
2989 int found;
2990
3d89194a 2991 if (iommu_dummy(dev))
1e4c64c4
DW		 2992 return 1;
2993
2c2e2c38 2994 if (!iommu_identity_mapping)
1e4c64c4 2995 return 0;
2c2e2c38 2996
9b226624 2997 found = identity_mapping(dev);
2c2e2c38 2998 if (found) {
ecb509ec 2999 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 3000 return 1;
3001 else {
3002 /*
3003 * 32 bit DMA is removed from si_domain and fall back
3004 * to non-identity mapping.
3005 */
bf9c9eda 3006 domain_remove_one_dev_info(si_domain, dev);
9f10e5bf
JR		 3007 pr_info("32bit %s uses non-identity mapping\n",
3008 dev_name(dev));
2c2e2c38
FY		 3009 return 0;
3010 }
3011 } else {
3012 /*
3013 * In case of a detached 64 bit DMA device from vm, the device
3014 * is put into si_domain for identity mapping.
3015 */
ecb509ec 3016 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 3017 int ret;
5913c9bf 3018 ret = domain_add_dev_info(si_domain, dev,
5fe60f4e
DW		 3019 hw_pass_through ?
3020 CONTEXT_TT_PASS_THROUGH :
3021 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38 3022 if (!ret) {
9f10e5bf
JR		 3023 pr_info("64bit %s uses identity mapping\n",
3024 dev_name(dev));
2c2e2c38
FY		 3025 return 1;
3026 }
3027 }
3028 }
3029
1e4c64c4 3030 return 0;
2c2e2c38
FY		 3031}
3032
5040a918 3033static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3034 size_t size, int dir, u64 dma_mask)
f76aec76 3035{
f76aec76 3036 struct dmar_domain *domain;
5b6985ce 3037 phys_addr_t start_paddr;
f76aec76
KA		 3038 struct iova *iova;
3039 int prot = 0;
6865f0d1 3040 int ret;
8c11e798 3041 struct intel_iommu *iommu;
33041ec0 3042 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3043
3044 BUG_ON(dir == DMA_NONE);
2c2e2c38 3045
5040a918 3046 if (iommu_no_mapping(dev))
6865f0d1 3047 return paddr;
f76aec76 3048
5040a918 3049 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3050 if (!domain)
3051 return 0;
3052
8c11e798 3053 iommu = domain_get_iommu(domain);
88cb6a74 3054 size = aligned_nrpages(paddr, size);
f76aec76 3055
5040a918 3056 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 3057 if (!iova)
3058 goto error;
3059
ba395927
KA		 3060 /*
3061 * Check if DMAR supports zero-length reads on write only
3062 * mappings..
3063 */
3064 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3065 !cap_zlr(iommu->cap))
ba395927
KA		 3066 prot |= DMA_PTE_READ;
3067 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3068 prot |= DMA_PTE_WRITE;
3069 /*
6865f0d1 3070 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3071 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3072 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3073 * is not a big problem
3074 */
0ab36de2 3075 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 3076 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3077 if (ret)
3078 goto error;
3079
1f0ef2aa
DW		 3080 /* it's a non-present to present mapping. Only flush if caching mode */
3081 if (cap_caching_mode(iommu->cap))
ea8ea460 3082 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
1f0ef2aa 3083 else
8c11e798 3084 iommu_flush_write_buffer(iommu);
f76aec76 3085
03d6a246
DW		 3086 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3087 start_paddr += paddr & ~PAGE_MASK;
3088 return start_paddr;
ba395927 3089
ba395927 3090error:
f76aec76
KA		 3091 if (iova)
3092 __free_iova(&domain->iovad, iova);
9f10e5bf 3093 pr_err("Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3094 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3095 return 0;
3096}
3097
ffbbef5c
FT		 3098static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3099 unsigned long offset, size_t size,
3100 enum dma_data_direction dir,
3101 struct dma_attrs *attrs)
bb9e6d65 3102{
ffbbef5c 3103 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3104 dir, *dev->dma_mask);
bb9e6d65
FT		 3105}
3106
5e0d2a6f 3107static void flush_unmaps(void)
3108{
80b20dd8 3109 int i, j;
5e0d2a6f 3110
5e0d2a6f 3111 timer_on = 0;
3112
3113 /* just flush them all */
3114 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3115 struct intel_iommu *iommu = g_iommus[i];
3116 if (!iommu)
3117 continue;
c42d9f32 3118
9dd2fe89
YZ		 3119 if (!deferred_flush[i].next)
3120 continue;
3121
78d5f0f5
NA		 3122 /* In caching mode, global flushes turn emulation expensive */
3123 if (!cap_caching_mode(iommu->cap))
3124 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3125 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3126 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3127 unsigned long mask;
3128 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3129 struct dmar_domain *domain = deferred_flush[i].domain[j];
3130
3131 /* On real hardware multiple invalidations are expensive */
3132 if (cap_caching_mode(iommu->cap))
3133 iommu_flush_iotlb_psi(iommu, domain->id,
a156ef99 3134 iova->pfn_lo, iova_size(iova),
ea8ea460 3135 !deferred_flush[i].freelist[j], 0);
78d5f0f5 3136 else {
a156ef99 3137 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
78d5f0f5
NA		 3138 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3139 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3140 }
93a23a72 3141 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3142 if (deferred_flush[i].freelist[j])
3143 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3144 }
9dd2fe89 3145 deferred_flush[i].next = 0;
5e0d2a6f 3146 }
3147
5e0d2a6f 3148 list_size = 0;
5e0d2a6f 3149}
3150
3151static void flush_unmaps_timeout(unsigned long data)
3152{
80b20dd8 3153 unsigned long flags;
3154
3155 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3156 flush_unmaps();
80b20dd8 3157 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3158}
3159
ea8ea460 3160static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3161{
3162 unsigned long flags;
80b20dd8 3163 int next, iommu_id;
8c11e798 3164 struct intel_iommu *iommu;
5e0d2a6f 3165
3166 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3167 if (list_size == HIGH_WATER_MARK)
3168 flush_unmaps();
3169
8c11e798
WH		 3170 iommu = domain_get_iommu(dom);
3171 iommu_id = iommu->seq_id;
c42d9f32 3172
80b20dd8 3173 next = deferred_flush[iommu_id].next;
3174 deferred_flush[iommu_id].domain[next] = dom;
3175 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3176 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3177 deferred_flush[iommu_id].next++;
5e0d2a6f 3178
3179 if (!timer_on) {
3180 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3181 timer_on = 1;
3182 }
3183 list_size++;
3184 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3185}
3186
d41a4adb 3187static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
ba395927 3188{
f76aec76 3189 struct dmar_domain *domain;
d794dc9b 3190 unsigned long start_pfn, last_pfn;
ba395927 3191 struct iova *iova;
8c11e798 3192 struct intel_iommu *iommu;
ea8ea460 3193 struct page *freelist;
ba395927 3194
73676832 3195 if (iommu_no_mapping(dev))
f76aec76 3196 return;
2c2e2c38 3197
1525a29a 3198 domain = find_domain(dev);
ba395927
KA		 3199 BUG_ON(!domain);
3200
8c11e798
WH		 3201 iommu = domain_get_iommu(domain);
3202
ba395927 3203 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3204 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3205 (unsigned long long)dev_addr))
ba395927 3206 return;
ba395927 3207
d794dc9b
DW		 3208 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3209 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3210
d794dc9b 3211 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3212 dev_name(dev), start_pfn, last_pfn);
ba395927 3213
ea8ea460 3214 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3215
5e0d2a6f 3216 if (intel_iommu_strict) {
03d6a246 3217 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
ea8ea460 3218 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3219 /* free iova */
3220 __free_iova(&domain->iovad, iova);
ea8ea460 3221 dma_free_pagelist(freelist);
5e0d2a6f 3222 } else {
ea8ea460 3223 add_unmap(domain, iova, freelist);
5e0d2a6f 3224 /*
3225 * queue up the release of the unmap to save the 1/6th of the
3226 * cpu used up by the iotlb flush operation...
3227 */
5e0d2a6f 3228 }
ba395927
KA		 3229}
3230
d41a4adb
JL		 3231static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3232 size_t size, enum dma_data_direction dir,
3233 struct dma_attrs *attrs)
3234{
3235 intel_unmap(dev, dev_addr);
3236}
3237
5040a918 3238static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3239 dma_addr_t *dma_handle, gfp_t flags,
3240 struct dma_attrs *attrs)
ba395927 3241{
36746436 3242 struct page *page = NULL;
ba395927
KA		 3243 int order;
3244
5b6985ce 3245 size = PAGE_ALIGN(size);
ba395927 3246 order = get_order(size);
e8bb910d 3247
5040a918 3248 if (!iommu_no_mapping(dev))
e8bb910d 3249 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3250 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3251 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3252 flags |= GFP_DMA;
3253 else
3254 flags |= GFP_DMA32;
3255 }
ba395927 3256
36746436
AM		 3257 if (flags & __GFP_WAIT) {
3258 unsigned int count = size >> PAGE_SHIFT;
3259
3260 page = dma_alloc_from_contiguous(dev, count, order);
3261 if (page && iommu_no_mapping(dev) &&
3262 page_to_phys(page) + size > dev->coherent_dma_mask) {
3263 dma_release_from_contiguous(dev, page, count);
3264 page = NULL;
3265 }
3266 }
3267
3268 if (!page)
3269 page = alloc_pages(flags, order);
3270 if (!page)
ba395927 3271 return NULL;
36746436 3272 memset(page_address(page), 0, size);
ba395927 3273
36746436 3274 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3275 DMA_BIDIRECTIONAL,
5040a918 3276 dev->coherent_dma_mask);
ba395927 3277 if (*dma_handle)
36746436
AM		 3278 return page_address(page);
3279 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3280 __free_pages(page, order);
3281
ba395927
KA		 3282 return NULL;
3283}
3284
5040a918 3285static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3286 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3287{
3288 int order;
36746436 3289 struct page *page = virt_to_page(vaddr);
ba395927 3290
5b6985ce 3291 size = PAGE_ALIGN(size);
ba395927
KA		 3292 order = get_order(size);
3293
d41a4adb 3294 intel_unmap(dev, dma_handle);
36746436
AM		 3295 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3296 __free_pages(page, order);
ba395927
KA		 3297}
3298
5040a918 3299static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3300 int nelems, enum dma_data_direction dir,
3301 struct dma_attrs *attrs)
ba395927 3302{
d41a4adb 3303 intel_unmap(dev, sglist[0].dma_address);
ba395927
KA		 3304}
3305
ba395927 3306static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3307 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3308{
3309 int i;
c03ab37c 3310 struct scatterlist *sg;
ba395927 3311
c03ab37c 3312 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3313 BUG_ON(!sg_page(sg));
4cf2e75d 3314 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3315 sg->dma_length = sg->length;
ba395927
KA		 3316 }
3317 return nelems;
3318}
3319
5040a918 3320static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3321 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3322{
ba395927 3323 int i;
ba395927 3324 struct dmar_domain *domain;
f76aec76
KA		 3325 size_t size = 0;
3326 int prot = 0;
f76aec76
KA		 3327 struct iova *iova = NULL;
3328 int ret;
c03ab37c 3329 struct scatterlist *sg;
b536d24d 3330 unsigned long start_vpfn;
8c11e798 3331 struct intel_iommu *iommu;
ba395927
KA		 3332
3333 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3334 if (iommu_no_mapping(dev))
3335 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3336
5040a918 3337 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3338 if (!domain)
3339 return 0;
3340
8c11e798
WH		 3341 iommu = domain_get_iommu(domain);
3342
b536d24d 3343 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3344 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3345
5040a918
DW		 3346 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3347 *dev->dma_mask);
f76aec76 3348 if (!iova) {
c03ab37c 3349 sglist->dma_length = 0;
f76aec76
KA		 3350 return 0;
3351 }
3352
3353 /*
3354 * Check if DMAR supports zero-length reads on write only
3355 * mappings..
3356 */
3357 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3358 !cap_zlr(iommu->cap))
f76aec76
KA		 3359 prot |= DMA_PTE_READ;
3360 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3361 prot |= DMA_PTE_WRITE;
3362
b536d24d 3363 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3364
f532959b 3365 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495 3366 if (unlikely(ret)) {
e1605495
DW		 3367 dma_pte_free_pagetable(domain, start_vpfn,
3368 start_vpfn + size - 1);
e1605495
DW		 3369 __free_iova(&domain->iovad, iova);
3370 return 0;
ba395927
KA		 3371 }
3372
1f0ef2aa
DW		 3373 /* it's a non-present to present mapping. Only flush if caching mode */
3374 if (cap_caching_mode(iommu->cap))
ea8ea460 3375 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
1f0ef2aa 3376 else
8c11e798 3377 iommu_flush_write_buffer(iommu);
1f0ef2aa 3378
ba395927
KA		 3379 return nelems;
3380}
3381
dfb805e8
FT		 3382static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3383{
3384 return !dma_addr;
3385}
3386
160c1d8e 3387struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3388 .alloc = intel_alloc_coherent,
3389 .free = intel_free_coherent,
ba395927
KA		 3390 .map_sg = intel_map_sg,
3391 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3392 .map_page = intel_map_page,
3393 .unmap_page = intel_unmap_page,
dfb805e8 3394 .mapping_error = intel_mapping_error,
ba395927
KA		 3395};
3396
3397static inline int iommu_domain_cache_init(void)
3398{
3399 int ret = 0;
3400
3401 iommu_domain_cache = kmem_cache_create("iommu_domain",
3402 sizeof(struct dmar_domain),
3403 0,
3404 SLAB_HWCACHE_ALIGN,
3405
3406 NULL);
3407 if (!iommu_domain_cache) {
9f10e5bf 3408 pr_err("Couldn't create iommu_domain cache\n");
ba395927
KA		 3409 ret = -ENOMEM;
3410 }
3411
3412 return ret;
3413}
3414
3415static inline int iommu_devinfo_cache_init(void)
3416{
3417 int ret = 0;
3418
3419 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3420 sizeof(struct device_domain_info),
3421 0,
3422 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3423 NULL);
3424 if (!iommu_devinfo_cache) {
9f10e5bf 3425 pr_err("Couldn't create devinfo cache\n");
ba395927
KA		 3426 ret = -ENOMEM;
3427 }
3428
3429 return ret;
3430}
3431
ba395927
KA		 3432static int __init iommu_init_mempool(void)
3433{
3434 int ret;
3435 ret = iommu_iova_cache_init();
3436 if (ret)
3437 return ret;
3438
3439 ret = iommu_domain_cache_init();
3440 if (ret)
3441 goto domain_error;
3442
3443 ret = iommu_devinfo_cache_init();
3444 if (!ret)
3445 return ret;
3446
3447 kmem_cache_destroy(iommu_domain_cache);
3448domain_error:
85b45456 3449 iommu_iova_cache_destroy();
ba395927
KA		 3450
3451 return -ENOMEM;
3452}
3453
3454static void __init iommu_exit_mempool(void)
3455{
3456 kmem_cache_destroy(iommu_devinfo_cache);
3457 kmem_cache_destroy(iommu_domain_cache);
85b45456 3458 iommu_iova_cache_destroy();
ba395927
KA		 3459}
3460
556ab45f
DW		 3461static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3462{
3463 struct dmar_drhd_unit *drhd;
3464 u32 vtbar;
3465 int rc;
3466
3467 /* We know that this device on this chipset has its own IOMMU.
3468 * If we find it under a different IOMMU, then the BIOS is lying
3469 * to us. Hope that the IOMMU for this device is actually
3470 * disabled, and it needs no translation...
3471 */
3472 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3473 if (rc) {
3474 /* "can't" happen */
3475 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3476 return;
3477 }
3478 vtbar &= 0xffff0000;
3479
3480 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3481 drhd = dmar_find_matched_drhd_unit(pdev);
3482 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3483 TAINT_FIRMWARE_WORKAROUND,
3484 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3485 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3486}
3487DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3488
ba395927
KA		 3489static void __init init_no_remapping_devices(void)
3490{
3491 struct dmar_drhd_unit *drhd;
832bd858 3492 struct device *dev;
b683b230 3493 int i;
ba395927
KA		 3494
3495 for_each_drhd_unit(drhd) {
3496 if (!drhd->include_all) {
b683b230
JL		 3497 for_each_active_dev_scope(drhd->devices,
3498 drhd->devices_cnt, i, dev)
3499 break;
832bd858 3500 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3501 if (i == drhd->devices_cnt)
3502 drhd->ignored = 1;
3503 }
3504 }
3505
7c919779 3506 for_each_active_drhd_unit(drhd) {
7c919779 3507 if (drhd->include_all)
ba395927
KA		 3508 continue;
3509
b683b230
JL		 3510 for_each_active_dev_scope(drhd->devices,
3511 drhd->devices_cnt, i, dev)
832bd858 3512 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 3513 break;
ba395927
KA		 3514 if (i < drhd->devices_cnt)
3515 continue;
3516
c0771df8
DW		 3517 /* This IOMMU has *only* gfx devices. Either bypass it or
3518 set the gfx_mapped flag, as appropriate */
3519 if (dmar_map_gfx) {
3520 intel_iommu_gfx_mapped = 1;
3521 } else {
3522 drhd->ignored = 1;
b683b230
JL		 3523 for_each_active_dev_scope(drhd->devices,
3524 drhd->devices_cnt, i, dev)
832bd858 3525 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3526 }
3527 }
3528}
3529
f59c7b69
FY		 3530#ifdef CONFIG_SUSPEND
3531static int init_iommu_hw(void)
3532{
3533 struct dmar_drhd_unit *drhd;
3534 struct intel_iommu *iommu = NULL;
3535
3536 for_each_active_iommu(iommu, drhd)
3537 if (iommu->qi)
3538 dmar_reenable_qi(iommu);
3539
b779260b
JC		 3540 for_each_iommu(iommu, drhd) {
3541 if (drhd->ignored) {
3542 /*
3543 * we always have to disable PMRs or DMA may fail on
3544 * this device
3545 */
3546 if (force_on)
3547 iommu_disable_protect_mem_regions(iommu);
3548 continue;
3549 }
3550
f59c7b69
FY		 3551 iommu_flush_write_buffer(iommu);
3552
3553 iommu_set_root_entry(iommu);
3554
3555 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3556 DMA_CCMD_GLOBAL_INVL);
2a41ccee
JL		 3557 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3558 iommu_enable_translation(iommu);
b94996c9 3559 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3560 }
3561
3562 return 0;
3563}
3564
3565static void iommu_flush_all(void)
3566{
3567 struct dmar_drhd_unit *drhd;
3568 struct intel_iommu *iommu;
3569
3570 for_each_active_iommu(iommu, drhd) {
3571 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3572 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3573 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3574 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3575 }
3576}
3577
134fac3f 3578static int iommu_suspend(void)
f59c7b69
FY		 3579{
3580 struct dmar_drhd_unit *drhd;
3581 struct intel_iommu *iommu = NULL;
3582 unsigned long flag;
3583
3584 for_each_active_iommu(iommu, drhd) {
3585 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3586 GFP_ATOMIC);
3587 if (!iommu->iommu_state)
3588 goto nomem;
3589 }
3590
3591 iommu_flush_all();
3592
3593 for_each_active_iommu(iommu, drhd) {
3594 iommu_disable_translation(iommu);
3595
1f5b3c3f 3596 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3597
3598 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3599 readl(iommu->reg + DMAR_FECTL_REG);
3600 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3601 readl(iommu->reg + DMAR_FEDATA_REG);
3602 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3603 readl(iommu->reg + DMAR_FEADDR_REG);
3604 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3605 readl(iommu->reg + DMAR_FEUADDR_REG);
3606
1f5b3c3f 3607 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3608 }
3609 return 0;
3610
3611nomem:
3612 for_each_active_iommu(iommu, drhd)
3613 kfree(iommu->iommu_state);
3614
3615 return -ENOMEM;
3616}
3617
134fac3f 3618static void iommu_resume(void)
f59c7b69
FY		 3619{
3620 struct dmar_drhd_unit *drhd;
3621 struct intel_iommu *iommu = NULL;
3622 unsigned long flag;
3623
3624 if (init_iommu_hw()) {
b779260b
JC		 3625 if (force_on)
3626 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3627 else
3628 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3629 return;
f59c7b69
FY		 3630 }
3631
3632 for_each_active_iommu(iommu, drhd) {
3633
1f5b3c3f 3634 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3635
3636 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3637 iommu->reg + DMAR_FECTL_REG);
3638 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3639 iommu->reg + DMAR_FEDATA_REG);
3640 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3641 iommu->reg + DMAR_FEADDR_REG);
3642 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3643 iommu->reg + DMAR_FEUADDR_REG);
3644
1f5b3c3f 3645 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3646 }
3647
3648 for_each_active_iommu(iommu, drhd)
3649 kfree(iommu->iommu_state);
f59c7b69
FY		 3650}
3651
134fac3f 3652static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3653 .resume = iommu_resume,
3654 .suspend = iommu_suspend,
3655};
3656
134fac3f 3657static void __init init_iommu_pm_ops(void)
f59c7b69 3658{
134fac3f 3659 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3660}
3661
3662#else
99592ba4 3663static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3664#endif /* CONFIG_PM */
3665
318fe7df 3666
c2a0b538 3667int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
318fe7df
SS		 3668{
3669 struct acpi_dmar_reserved_memory *rmrr;
3670 struct dmar_rmrr_unit *rmrru;
3671
3672 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3673 if (!rmrru)
3674 return -ENOMEM;
3675
3676 rmrru->hdr = header;
3677 rmrr = (struct acpi_dmar_reserved_memory *)header;
3678 rmrru->base_address = rmrr->base_address;
3679 rmrru->end_address = rmrr->end_address;
2e455289
JL		 3680 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3681 ((void *)rmrr) + rmrr->header.length,
3682 &rmrru->devices_cnt);
3683 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3684 kfree(rmrru);
3685 return -ENOMEM;
3686 }
318fe7df 3687
2e455289 3688 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 3689
2e455289 3690 return 0;
318fe7df
SS		 3691}
3692
6b197249
JL		 3693static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
3694{
3695 struct dmar_atsr_unit *atsru;
3696 struct acpi_dmar_atsr *tmp;
3697
3698 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3699 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
3700 if (atsr->segment != tmp->segment)
3701 continue;
3702 if (atsr->header.length != tmp->header.length)
3703 continue;
3704 if (memcmp(atsr, tmp, atsr->header.length) == 0)
3705 return atsru;
3706 }
3707
3708 return NULL;
3709}
3710
3711int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
318fe7df
SS		 3712{
3713 struct acpi_dmar_atsr *atsr;
3714 struct dmar_atsr_unit *atsru;
3715
6b197249
JL		 3716 if (system_state != SYSTEM_BOOTING && !intel_iommu_enabled)
3717 return 0;
3718
318fe7df 3719 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
6b197249
JL		 3720 atsru = dmar_find_atsr(atsr);
3721 if (atsru)
3722 return 0;
3723
3724 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
318fe7df
SS		 3725 if (!atsru)
3726 return -ENOMEM;
3727
6b197249
JL		 3728 /*
3729 * If memory is allocated from slab by ACPI _DSM method, we need to
3730 * copy the memory content because the memory buffer will be freed
3731 * on return.
3732 */
3733 atsru->hdr = (void *)(atsru + 1);
3734 memcpy(atsru->hdr, hdr, hdr->length);
318fe7df 3735 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 3736 if (!atsru->include_all) {
3737 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3738 (void *)atsr + atsr->header.length,
3739 &atsru->devices_cnt);
3740 if (atsru->devices_cnt && atsru->devices == NULL) {
3741 kfree(atsru);
3742 return -ENOMEM;
3743 }
3744 }
318fe7df 3745
0e242612 3746 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 3747
3748 return 0;
3749}
3750
9bdc531e
JL		 3751static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3752{
3753 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3754 kfree(atsru);
3755}
3756
6b197249
JL		 3757int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3758{
3759 struct acpi_dmar_atsr *atsr;
3760 struct dmar_atsr_unit *atsru;
3761
3762 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3763 atsru = dmar_find_atsr(atsr);
3764 if (atsru) {
3765 list_del_rcu(&atsru->list);
3766 synchronize_rcu();
3767 intel_iommu_free_atsr(atsru);
3768 }
3769
3770 return 0;
3771}
3772
3773int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3774{
3775 int i;
3776 struct device *dev;
3777 struct acpi_dmar_atsr *atsr;
3778 struct dmar_atsr_unit *atsru;
3779
3780 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3781 atsru = dmar_find_atsr(atsr);
3782 if (!atsru)
3783 return 0;
3784
3785 if (!atsru->include_all && atsru->devices && atsru->devices_cnt)
3786 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
3787 i, dev)
3788 return -EBUSY;
3789
3790 return 0;
3791}
3792
ffebeb46
JL		 3793static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
3794{
3795 int sp, ret = 0;
3796 struct intel_iommu *iommu = dmaru->iommu;
3797
3798 if (g_iommus[iommu->seq_id])
3799 return 0;
3800
3801 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
9f10e5bf 3802 pr_warn("%s: Doesn't support hardware pass through.\n",
ffebeb46
JL		 3803 iommu->name);
3804 return -ENXIO;
3805 }
3806 if (!ecap_sc_support(iommu->ecap) &&
3807 domain_update_iommu_snooping(iommu)) {
9f10e5bf 3808 pr_warn("%s: Doesn't support snooping.\n",
ffebeb46
JL		 3809 iommu->name);
3810 return -ENXIO;
3811 }
3812 sp = domain_update_iommu_superpage(iommu) - 1;
3813 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
9f10e5bf 3814 pr_warn("%s: Doesn't support large page.\n",
ffebeb46
JL		 3815 iommu->name);
3816 return -ENXIO;
3817 }
3818
3819 /*
3820 * Disable translation if already enabled prior to OS handover.
3821 */
3822 if (iommu->gcmd & DMA_GCMD_TE)
3823 iommu_disable_translation(iommu);
3824
3825 g_iommus[iommu->seq_id] = iommu;
3826 ret = iommu_init_domains(iommu);
3827 if (ret == 0)
3828 ret = iommu_alloc_root_entry(iommu);
3829 if (ret)
3830 goto out;
3831
3832 if (dmaru->ignored) {
3833 /*
3834 * we always have to disable PMRs or DMA may fail on this device
3835 */
3836 if (force_on)
3837 iommu_disable_protect_mem_regions(iommu);
3838 return 0;
3839 }
3840
3841 intel_iommu_init_qi(iommu);
3842 iommu_flush_write_buffer(iommu);
3843 ret = dmar_set_interrupt(iommu);
3844 if (ret)
3845 goto disable_iommu;
3846
3847 iommu_set_root_entry(iommu);
3848 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
3849 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3850 iommu_enable_translation(iommu);
3851
3852 if (si_domain) {
3853 ret = iommu_attach_domain(si_domain, iommu);
3854 if (ret < 0 || si_domain->id != ret)
3855 goto disable_iommu;
3856 domain_attach_iommu(si_domain, iommu);
3857 }
3858
3859 iommu_disable_protect_mem_regions(iommu);
3860 return 0;
3861
3862disable_iommu:
3863 disable_dmar_iommu(iommu);
3864out:
3865 free_dmar_iommu(iommu);
3866 return ret;
3867}
3868
6b197249
JL		 3869int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
3870{
ffebeb46
JL		 3871 int ret = 0;
3872 struct intel_iommu *iommu = dmaru->iommu;
3873
3874 if (!intel_iommu_enabled)
3875 return 0;
3876 if (iommu == NULL)
3877 return -EINVAL;
3878
3879 if (insert) {
3880 ret = intel_iommu_add(dmaru);
3881 } else {
3882 disable_dmar_iommu(iommu);
3883 free_dmar_iommu(iommu);
3884 }
3885
3886 return ret;
6b197249
JL		 3887}
3888
9bdc531e
JL		 3889static void intel_iommu_free_dmars(void)
3890{
3891 struct dmar_rmrr_unit *rmrru, *rmrr_n;
3892 struct dmar_atsr_unit *atsru, *atsr_n;
3893
3894 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3895 list_del(&rmrru->list);
3896 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3897 kfree(rmrru);
318fe7df
SS		 3898 }
3899
9bdc531e
JL		 3900 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3901 list_del(&atsru->list);
3902 intel_iommu_free_atsr(atsru);
3903 }
318fe7df
SS		 3904}
3905
3906int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3907{
b683b230 3908 int i, ret = 1;
318fe7df 3909 struct pci_bus *bus;
832bd858
DW		 3910 struct pci_dev *bridge = NULL;
3911 struct device *tmp;
318fe7df
SS		 3912 struct acpi_dmar_atsr *atsr;
3913 struct dmar_atsr_unit *atsru;
3914
3915 dev = pci_physfn(dev);
318fe7df 3916 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 3917 bridge = bus->self;
318fe7df 3918 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 3919 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 3920 return 0;
b5f82ddf 3921 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 3922 break;
318fe7df 3923 }
b5f82ddf
JL		 3924 if (!bridge)
3925 return 0;
318fe7df 3926
0e242612 3927 rcu_read_lock();
b5f82ddf
JL		 3928 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3929 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3930 if (atsr->segment != pci_domain_nr(dev->bus))
3931 continue;
3932
b683b230 3933 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 3934 if (tmp == &bridge->dev)
b683b230 3935 goto out;
b5f82ddf
JL		 3936
3937 if (atsru->include_all)
b683b230 3938 goto out;
b5f82ddf 3939 }
b683b230
JL		 3940 ret = 0;
3941out:
0e242612 3942 rcu_read_unlock();
318fe7df 3943
b683b230 3944 return ret;
318fe7df
SS		 3945}
3946
59ce0515
JL		 3947int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3948{
3949 int ret = 0;
3950 struct dmar_rmrr_unit *rmrru;
3951 struct dmar_atsr_unit *atsru;
3952 struct acpi_dmar_atsr *atsr;
3953 struct acpi_dmar_reserved_memory *rmrr;
3954
3955 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
3956 return 0;
3957
3958 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3959 rmrr = container_of(rmrru->hdr,
3960 struct acpi_dmar_reserved_memory, header);
3961 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3962 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3963 ((void *)rmrr) + rmrr->header.length,
3964 rmrr->segment, rmrru->devices,
3965 rmrru->devices_cnt);
27e24950 3966 if(ret < 0)
59ce0515
JL		 3967 return ret;
3968 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
27e24950
JL		 3969 dmar_remove_dev_scope(info, rmrr->segment,
3970 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 3971 }
3972 }
3973
3974 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3975 if (atsru->include_all)
3976 continue;
3977
3978 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3979 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3980 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3981 (void *)atsr + atsr->header.length,
3982 atsr->segment, atsru->devices,
3983 atsru->devices_cnt);
3984 if (ret > 0)
3985 break;
3986 else if(ret < 0)
3987 return ret;
3988 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3989 if (dmar_remove_dev_scope(info, atsr->segment,
3990 atsru->devices, atsru->devices_cnt))
3991 break;
3992 }
3993 }
3994
3995 return 0;
3996}
3997
99dcaded
FY		 3998/*
3999 * Here we only respond to action of unbound device from driver.
4000 *
4001 * Added device is not attached to its DMAR domain here yet. That will happen
4002 * when mapping the device to iova.
4003 */
4004static int device_notifier(struct notifier_block *nb,
4005 unsigned long action, void *data)
4006{
4007 struct device *dev = data;
99dcaded
FY		 4008 struct dmar_domain *domain;
4009
3d89194a 4010 if (iommu_dummy(dev))
44cd613c
DW		 4011 return 0;
4012
1196c2fb 4013 if (action != BUS_NOTIFY_REMOVED_DEVICE)
7e7dfab7
JL		 4014 return 0;
4015
1525a29a 4016 domain = find_domain(dev);
99dcaded
FY		 4017 if (!domain)
4018 return 0;
4019
3a5670e8 4020 down_read(&dmar_global_lock);
bf9c9eda 4021 domain_remove_one_dev_info(domain, dev);
ab8dfe25 4022 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 4023 domain_exit(domain);
3a5670e8 4024 up_read(&dmar_global_lock);
a97590e5 4025
99dcaded
FY		 4026 return 0;
4027}
4028
4029static struct notifier_block device_nb = {
4030 .notifier_call = device_notifier,
4031};
4032
75f05569
JL		 4033static int intel_iommu_memory_notifier(struct notifier_block *nb,
4034 unsigned long val, void *v)
4035{
4036 struct memory_notify *mhp = v;
4037 unsigned long long start, end;
4038 unsigned long start_vpfn, last_vpfn;
4039
4040 switch (val) {
4041 case MEM_GOING_ONLINE:
4042 start = mhp->start_pfn << PAGE_SHIFT;
4043 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
4044 if (iommu_domain_identity_map(si_domain, start, end)) {
9f10e5bf 4045 pr_warn("Failed to build identity map for [%llx-%llx]\n",
75f05569
JL		 4046 start, end);
4047 return NOTIFY_BAD;
4048 }
4049 break;
4050
4051 case MEM_OFFLINE:
4052 case MEM_CANCEL_ONLINE:
4053 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
4054 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
4055 while (start_vpfn <= last_vpfn) {
4056 struct iova *iova;
4057 struct dmar_drhd_unit *drhd;
4058 struct intel_iommu *iommu;
ea8ea460 4059 struct page *freelist;
75f05569
JL		 4060
4061 iova = find_iova(&si_domain->iovad, start_vpfn);
4062 if (iova == NULL) {
9f10e5bf 4063 pr_debug("Failed get IOVA for PFN %lx\n",
75f05569
JL		 4064 start_vpfn);
4065 break;
4066 }
4067
4068 iova = split_and_remove_iova(&si_domain->iovad, iova,
4069 start_vpfn, last_vpfn);
4070 if (iova == NULL) {
9f10e5bf 4071 pr_warn("Failed to split IOVA PFN [%lx-%lx]\n",
75f05569
JL		 4072 start_vpfn, last_vpfn);
4073 return NOTIFY_BAD;
4074 }
4075
ea8ea460
DW		 4076 freelist = domain_unmap(si_domain, iova->pfn_lo,
4077 iova->pfn_hi);
4078
75f05569
JL		 4079 rcu_read_lock();
4080 for_each_active_iommu(iommu, drhd)
4081 iommu_flush_iotlb_psi(iommu, si_domain->id,
a156ef99 4082 iova->pfn_lo, iova_size(iova),
ea8ea460 4083 !freelist, 0);
75f05569 4084 rcu_read_unlock();
ea8ea460 4085 dma_free_pagelist(freelist);
75f05569
JL		 4086
4087 start_vpfn = iova->pfn_hi + 1;
4088 free_iova_mem(iova);
4089 }
4090 break;
4091 }
4092
4093 return NOTIFY_OK;
4094}
4095
4096static struct notifier_block intel_iommu_memory_nb = {
4097 .notifier_call = intel_iommu_memory_notifier,
4098 .priority = 0
4099};
4100
a5459cfe
AW		 4101
4102static ssize_t intel_iommu_show_version(struct device *dev,
4103 struct device_attribute *attr,
4104 char *buf)
4105{
4106 struct intel_iommu *iommu = dev_get_drvdata(dev);
4107 u32 ver = readl(iommu->reg + DMAR_VER_REG);
4108 return sprintf(buf, "%d:%d\n",
4109 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
4110}
4111static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
4112
4113static ssize_t intel_iommu_show_address(struct device *dev,
4114 struct device_attribute *attr,
4115 char *buf)
4116{
4117 struct intel_iommu *iommu = dev_get_drvdata(dev);
4118 return sprintf(buf, "%llx\n", iommu->reg_phys);
4119}
4120static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
4121
4122static ssize_t intel_iommu_show_cap(struct device *dev,
4123 struct device_attribute *attr,
4124 char *buf)
4125{
4126 struct intel_iommu *iommu = dev_get_drvdata(dev);
4127 return sprintf(buf, "%llx\n", iommu->cap);
4128}
4129static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
4130
4131static ssize_t intel_iommu_show_ecap(struct device *dev,
4132 struct device_attribute *attr,
4133 char *buf)
4134{
4135 struct intel_iommu *iommu = dev_get_drvdata(dev);
4136 return sprintf(buf, "%llx\n", iommu->ecap);
4137}
4138static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4139
4140static struct attribute *intel_iommu_attrs[] = {
4141 &dev_attr_version.attr,
4142 &dev_attr_address.attr,
4143 &dev_attr_cap.attr,
4144 &dev_attr_ecap.attr,
4145 NULL,
4146};
4147
4148static struct attribute_group intel_iommu_group = {
4149 .name = "intel-iommu",
4150 .attrs = intel_iommu_attrs,
4151};
4152
4153const struct attribute_group *intel_iommu_groups[] = {
4154 &intel_iommu_group,
4155 NULL,
4156};
4157
ba395927
KA		 4158int __init intel_iommu_init(void)
4159{
9bdc531e 4160 int ret = -ENODEV;
3a93c841 4161 struct dmar_drhd_unit *drhd;
7c919779 4162 struct intel_iommu *iommu;
ba395927 4163
a59b50e9
JC		 4164 /* VT-d is required for a TXT/tboot launch, so enforce that */
4165 force_on = tboot_force_iommu();
4166
3a5670e8
JL		 4167 if (iommu_init_mempool()) {
4168 if (force_on)
4169 panic("tboot: Failed to initialize iommu memory\n");
4170 return -ENOMEM;
4171 }
4172
4173 down_write(&dmar_global_lock);
a59b50e9
JC		 4174 if (dmar_table_init()) {
4175 if (force_on)
4176 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4177 goto out_free_dmar;
a59b50e9 4178 }
ba395927 4179
3a93c841
TI		 4180 /*
4181 * Disable translation if already enabled prior to OS handover.
4182 */
7c919779 4183 for_each_active_iommu(iommu, drhd)
3a93c841
TI		 4184 if (iommu->gcmd & DMA_GCMD_TE)
4185 iommu_disable_translation(iommu);
3a93c841 4186
c2c7286a 4187 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4188 if (force_on)
4189 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4190 goto out_free_dmar;
a59b50e9 4191 }
1886e8a9 4192
75f1cdf1 4193 if (no_iommu || dmar_disabled)
9bdc531e 4194 goto out_free_dmar;
2ae21010 4195
318fe7df 4196 if (list_empty(&dmar_rmrr_units))
9f10e5bf 4197 pr_info("No RMRR found\n");
318fe7df
SS		 4198
4199 if (list_empty(&dmar_atsr_units))
9f10e5bf 4200 pr_info("No ATSR found\n");
318fe7df 4201
51a63e67
JC		 4202 if (dmar_init_reserved_ranges()) {
4203 if (force_on)
4204 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4205 goto out_free_reserved_range;
51a63e67 4206 }
ba395927
KA		 4207
4208 init_no_remapping_devices();
4209
b779260b 4210 ret = init_dmars();
ba395927 4211 if (ret) {
a59b50e9
JC		 4212 if (force_on)
4213 panic("tboot: Failed to initialize DMARs\n");
9f10e5bf 4214 pr_err("Initialization failed\n");
9bdc531e 4215 goto out_free_reserved_range;
ba395927 4216 }
3a5670e8 4217 up_write(&dmar_global_lock);
9f10e5bf 4218 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
ba395927 4219
5e0d2a6f 4220 init_timer(&unmap_timer);
75f1cdf1
FT		 4221#ifdef CONFIG_SWIOTLB
4222 swiotlb = 0;
4223#endif
19943b0e 4224 dma_ops = &intel_dma_ops;
4ed0d3e6 4225
134fac3f 4226 init_iommu_pm_ops();
a8bcbb0d 4227
a5459cfe
AW		 4228 for_each_active_iommu(iommu, drhd)
4229 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4230 intel_iommu_groups,
4231 iommu->name);
4232
4236d97d 4233 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4234 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4235 if (si_domain && !hw_pass_through)
4236 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 4237
8bc1f85c
ED		 4238 intel_iommu_enabled = 1;
4239
ba395927 4240 return 0;
9bdc531e
JL		 4241
4242out_free_reserved_range:
4243 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4244out_free_dmar:
4245 intel_iommu_free_dmars();
3a5670e8
JL		 4246 up_write(&dmar_global_lock);
4247 iommu_exit_mempool();
9bdc531e 4248 return ret;
ba395927 4249}
e820482c 4250
579305f7
AW		 4251static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4252{
4253 struct intel_iommu *iommu = opaque;
4254
4255 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4256 return 0;
4257}
4258
4259/*
4260 * NB - intel-iommu lacks any sort of reference counting for the users of
4261 * dependent devices. If multiple endpoints have intersecting dependent
4262 * devices, unbinding the driver from any one of them will possibly leave
4263 * the others unable to operate.
4264 */
3199aa6b 4265static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 4266 struct device *dev)
3199aa6b 4267{
0bcb3e28 4268 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4269 return;
4270
579305f7 4271 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
3199aa6b
HW		 4272}
4273
2c2e2c38 4274static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 4275 struct device *dev)
c7151a8d 4276{
bca2b916 4277 struct device_domain_info *info, *tmp;
c7151a8d
WH		 4278 struct intel_iommu *iommu;
4279 unsigned long flags;
2f119c78 4280 bool found = false;
156baca8 4281 u8 bus, devfn;
c7151a8d 4282
bf9c9eda 4283 iommu = device_to_iommu(dev, &bus, &devfn);
c7151a8d
WH		 4284 if (!iommu)
4285 return;
4286
4287 spin_lock_irqsave(&device_domain_lock, flags);
bca2b916 4288 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
bf9c9eda
DW		 4289 if (info->iommu == iommu && info->bus == bus &&
4290 info->devfn == devfn) {
109b9b04 4291 unlink_domain_info(info);
c7151a8d
WH		 4292 spin_unlock_irqrestore(&device_domain_lock, flags);
4293
93a23a72 4294 iommu_disable_dev_iotlb(info);
c7151a8d 4295 iommu_detach_dev(iommu, info->bus, info->devfn);
bf9c9eda 4296 iommu_detach_dependent_devices(iommu, dev);
c7151a8d
WH		 4297 free_devinfo_mem(info);
4298
4299 spin_lock_irqsave(&device_domain_lock, flags);
4300
4301 if (found)
4302 break;
4303 else
4304 continue;
4305 }
4306
4307 /* if there is no other devices under the same iommu
4308 * owned by this domain, clear this iommu in iommu_bmp
4309 * update iommu count and coherency
4310 */
8bbc4410 4311 if (info->iommu == iommu)
2f119c78 4312 found = true;
c7151a8d
WH		 4313 }
4314
3e7abe25
RD		 4315 spin_unlock_irqrestore(&device_domain_lock, flags);
4316
c7151a8d 4317 if (found == 0) {
fb170fb4
JL		 4318 domain_detach_iommu(domain, iommu);
4319 if (!domain_type_is_vm_or_si(domain))
4320 iommu_detach_domain(domain, iommu);
c7151a8d 4321 }
c7151a8d
WH		 4322}
4323
2c2e2c38 4324static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4325{
4326 int adjust_width;
4327
0fb5fe87
RM		 4328 init_iova_domain(&domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN,
4329 DMA_32BIT_PFN);
5e98c4b1
WH		 4330 domain_reserve_special_ranges(domain);
4331
4332 /* calculate AGAW */
4333 domain->gaw = guest_width;
4334 adjust_width = guestwidth_to_adjustwidth(guest_width);
4335 domain->agaw = width_to_agaw(adjust_width);
4336
5e98c4b1 4337 domain->iommu_coherency = 0;
c5b15255 4338 domain->iommu_snooping = 0;
6dd9a7c7 4339 domain->iommu_superpage = 0;
fe40f1e0 4340 domain->max_addr = 0;
5e98c4b1
WH		 4341
4342 /* always allocate the top pgd */
4c923d47 4343 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4344 if (!domain->pgd)
4345 return -ENOMEM;
4346 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4347 return 0;
4348}
4349
00a77deb 4350static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
38717946 4351{
5d450806 4352 struct dmar_domain *dmar_domain;
00a77deb
JR		 4353 struct iommu_domain *domain;
4354
4355 if (type != IOMMU_DOMAIN_UNMANAGED)
4356 return NULL;
38717946 4357
ab8dfe25 4358 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4359 if (!dmar_domain) {
9f10e5bf 4360 pr_err("Can't allocate dmar_domain\n");
00a77deb 4361 return NULL;
38717946 4362 }
2c2e2c38 4363 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
9f10e5bf 4364 pr_err("Domain initialization failed\n");
92d03cc8 4365 domain_exit(dmar_domain);
00a77deb 4366 return NULL;
38717946 4367 }
8140a95d 4368 domain_update_iommu_cap(dmar_domain);
faa3d6f5 4369
00a77deb 4370 domain = &dmar_domain->domain;
8a0e715b
JR		 4371 domain->geometry.aperture_start = 0;
4372 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4373 domain->geometry.force_aperture = true;
4374
00a77deb 4375 return domain;
38717946 4376}
38717946 4377
00a77deb 4378static void intel_iommu_domain_free(struct iommu_domain *domain)
38717946 4379{
00a77deb 4380 domain_exit(to_dmar_domain(domain));
38717946 4381}
38717946 4382
4c5478c9
JR		 4383static int intel_iommu_attach_device(struct iommu_domain *domain,
4384 struct device *dev)
38717946 4385{
00a77deb 4386 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0
WH		 4387 struct intel_iommu *iommu;
4388 int addr_width;
156baca8 4389 u8 bus, devfn;
faa3d6f5 4390
c875d2c1
AW		 4391 if (device_is_rmrr_locked(dev)) {
4392 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4393 return -EPERM;
4394 }
4395
7207d8f9
DW		 4396 /* normally dev is not mapped */
4397 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4398 struct dmar_domain *old_domain;
4399
1525a29a 4400 old_domain = find_domain(dev);
faa3d6f5 4401 if (old_domain) {
ab8dfe25 4402 if (domain_type_is_vm_or_si(dmar_domain))
bf9c9eda 4403 domain_remove_one_dev_info(old_domain, dev);
faa3d6f5
WH		 4404 else
4405 domain_remove_dev_info(old_domain);
62c22167
JR		 4406
4407 if (!domain_type_is_vm_or_si(old_domain) &&
4408 list_empty(&old_domain->devices))
4409 domain_exit(old_domain);
faa3d6f5
WH		 4410 }
4411 }
4412
156baca8 4413 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4414 if (!iommu)
4415 return -ENODEV;
4416
4417 /* check if this iommu agaw is sufficient for max mapped address */
4418 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4419 if (addr_width > cap_mgaw(iommu->cap))
4420 addr_width = cap_mgaw(iommu->cap);
4421
4422 if (dmar_domain->max_addr > (1LL << addr_width)) {
9f10e5bf 4423 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4424 "sufficient for the mapped address (%llx)\n",
a99c47a2 4425 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4426 return -EFAULT;
4427 }
a99c47a2
TL		 4428 dmar_domain->gaw = addr_width;
4429
4430 /*
4431 * Knock out extra levels of page tables if necessary
4432 */
4433 while (iommu->agaw < dmar_domain->agaw) {
4434 struct dma_pte *pte;
4435
4436 pte = dmar_domain->pgd;
4437 if (dma_pte_present(pte)) {
25cbff16
SY		 4438 dmar_domain->pgd = (struct dma_pte *)
4439 phys_to_virt(dma_pte_addr(pte));
7a661013 4440 free_pgtable_page(pte);
a99c47a2
TL		 4441 }
4442 dmar_domain->agaw--;
4443 }
fe40f1e0 4444
5913c9bf 4445 return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
38717946 4446}
38717946 4447
4c5478c9
JR		 4448static void intel_iommu_detach_device(struct iommu_domain *domain,
4449 struct device *dev)
38717946 4450{
00a77deb 4451 domain_remove_one_dev_info(to_dmar_domain(domain), dev);
faa3d6f5 4452}
c7151a8d 4453
b146a1c9
JR		 4454static int intel_iommu_map(struct iommu_domain *domain,
4455 unsigned long iova, phys_addr_t hpa,
5009065d 4456 size_t size, int iommu_prot)
faa3d6f5 4457{
00a77deb 4458 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
fe40f1e0 4459 u64 max_addr;
dde57a21 4460 int prot = 0;
faa3d6f5 4461 int ret;
fe40f1e0 4462
dde57a21
JR		 4463 if (iommu_prot & IOMMU_READ)
4464 prot |= DMA_PTE_READ;
4465 if (iommu_prot & IOMMU_WRITE)
4466 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4467 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4468 prot |= DMA_PTE_SNP;
dde57a21 4469
163cc52c 4470 max_addr = iova + size;
dde57a21 4471 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4472 u64 end;
4473
4474 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4475 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4476 if (end < max_addr) {
9f10e5bf 4477 pr_err("%s: iommu width (%d) is not "
fe40f1e0 4478 "sufficient for the mapped address (%llx)\n",
8954da1f 4479 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4480 return -EFAULT;
4481 }
dde57a21 4482 dmar_domain->max_addr = max_addr;
fe40f1e0 4483 }
ad051221
DW		 4484 /* Round up size to next multiple of PAGE_SIZE, if it and
4485 the low bits of hpa would take us onto the next page */
88cb6a74 4486 size = aligned_nrpages(hpa, size);
ad051221
DW		 4487 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4488 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4489 return ret;
38717946 4490}
38717946 4491
5009065d 4492static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4493 unsigned long iova, size_t size)
38717946 4494{
00a77deb 4495 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
ea8ea460
DW		 4496 struct page *freelist = NULL;
4497 struct intel_iommu *iommu;
4498 unsigned long start_pfn, last_pfn;
4499 unsigned int npages;
4500 int iommu_id, num, ndomains, level = 0;
5cf0a76f
DW		 4501
4502 /* Cope with horrid API which requires us to unmap more than the
4503 size argument if it happens to be a large-page mapping. */
4504 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4505 BUG();
4506
4507 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4508 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4509
ea8ea460
DW		 4510 start_pfn = iova >> VTD_PAGE_SHIFT;
4511 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4512
4513 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4514
4515 npages = last_pfn - start_pfn + 1;
4516
4517 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4518 iommu = g_iommus[iommu_id];
4519
4520 /*
4521 * find bit position of dmar_domain
4522 */
4523 ndomains = cap_ndoms(iommu->cap);
4524 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4525 if (iommu->domains[num] == dmar_domain)
4526 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4527 npages, !freelist, 0);
4528 }
4529
4530 }
4531
4532 dma_free_pagelist(freelist);
fe40f1e0 4533
163cc52c
DW		 4534 if (dmar_domain->max_addr == iova + size)
4535 dmar_domain->max_addr = iova;
b146a1c9 4536
5cf0a76f 4537 return size;
38717946 4538}
38717946 4539
d14d6577 4540static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4541 dma_addr_t iova)
38717946 4542{
00a77deb 4543 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
38717946 4544 struct dma_pte *pte;
5cf0a76f 4545 int level = 0;
faa3d6f5 4546 u64 phys = 0;
38717946 4547
5cf0a76f 4548 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4549 if (pte)
faa3d6f5 4550 phys = dma_pte_addr(pte);
38717946 4551
faa3d6f5 4552 return phys;
38717946 4553}
a8bcbb0d 4554
5d587b8d 4555static bool intel_iommu_capable(enum iommu_cap cap)
dbb9fd86 4556{
dbb9fd86 4557 if (cap == IOMMU_CAP_CACHE_COHERENCY)
5d587b8d 4558 return domain_update_iommu_snooping(NULL) == 1;
323f99cb 4559 if (cap == IOMMU_CAP_INTR_REMAP)
5d587b8d 4560 return irq_remapping_enabled == 1;
dbb9fd86 4561
5d587b8d 4562 return false;
dbb9fd86
SY		 4563}
4564
abdfdde2
AW		 4565static int intel_iommu_add_device(struct device *dev)
4566{
a5459cfe 4567 struct intel_iommu *iommu;
abdfdde2 4568 struct iommu_group *group;
156baca8 4569 u8 bus, devfn;
70ae6f0d 4570
a5459cfe
AW		 4571 iommu = device_to_iommu(dev, &bus, &devfn);
4572 if (!iommu)
70ae6f0d
AW		 4573 return -ENODEV;
4574
a5459cfe 4575 iommu_device_link(iommu->iommu_dev, dev);
a4ff1fc2 4576
e17f9ff4 4577 group = iommu_group_get_for_dev(dev);
783f157b 4578
e17f9ff4
AW		 4579 if (IS_ERR(group))
4580 return PTR_ERR(group);
bcb71abe 4581
abdfdde2 4582 iommu_group_put(group);
e17f9ff4 4583 return 0;
abdfdde2 4584}
70ae6f0d 4585
abdfdde2
AW		 4586static void intel_iommu_remove_device(struct device *dev)
4587{
a5459cfe
AW		 4588 struct intel_iommu *iommu;
4589 u8 bus, devfn;
4590
4591 iommu = device_to_iommu(dev, &bus, &devfn);
4592 if (!iommu)
4593 return;
4594
abdfdde2 4595 iommu_group_remove_device(dev);
a5459cfe
AW		 4596
4597 iommu_device_unlink(iommu->iommu_dev, dev);
70ae6f0d
AW		 4598}
4599
b22f6434 4600static const struct iommu_ops intel_iommu_ops = {
5d587b8d 4601 .capable = intel_iommu_capable,
00a77deb
JR		 4602 .domain_alloc = intel_iommu_domain_alloc,
4603 .domain_free = intel_iommu_domain_free,
a8bcbb0d
JR		 4604 .attach_dev = intel_iommu_attach_device,
4605 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4606 .map = intel_iommu_map,
4607 .unmap = intel_iommu_unmap,
315786eb 4608 .map_sg = default_iommu_map_sg,
a8bcbb0d 4609 .iova_to_phys = intel_iommu_iova_to_phys,
abdfdde2
AW		 4610 .add_device = intel_iommu_add_device,
4611 .remove_device = intel_iommu_remove_device,
6d1c56a9 4612 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4613};
9af88143 4614
9452618e
DV		 4615static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4616{
4617 /* G4x/GM45 integrated gfx dmar support is totally busted. */
9f10e5bf 4618 pr_info("Disabling IOMMU for graphics on this chipset\n");
9452618e
DV		 4619 dmar_map_gfx = 0;
4620}
4621
4622DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4623DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4624DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4625DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4626DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4627DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4628DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4629
d34d6517 4630static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4631{
4632 /*
4633 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4634 * but needs it. Same seems to hold for the desktop versions.
9af88143 4635 */
9f10e5bf 4636 pr_info("Forcing write-buffer flush capability\n");
9af88143
DW		 4637 rwbf_quirk = 1;
4638}
4639
4640DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4641DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4642DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4643DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4644DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4645DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4646DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4647
eecfd57f
AJ		 4648#define GGC 0x52
4649#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4650#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4651#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4652#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4653#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4654#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4655#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4656#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4657
d34d6517 4658static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4659{
4660 unsigned short ggc;
4661
eecfd57f 4662 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4663 return;
4664
eecfd57f 4665 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9f10e5bf 4666 pr_info("BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
9eecabcb 4667 dmar_map_gfx = 0;
6fbcfb3e
DW		 4668 } else if (dmar_map_gfx) {
4669 /* we have to ensure the gfx device is idle before we flush */
9f10e5bf 4670 pr_info("Disabling batched IOTLB flush on Ironlake\n");
6fbcfb3e
DW		 4671 intel_iommu_strict = 1;
4672 }
9eecabcb
DW		 4673}
4674DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4675DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4676DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4677DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4678
e0fc7e0b
DW		 4679/* On Tylersburg chipsets, some BIOSes have been known to enable the
4680 ISOCH DMAR unit for the Azalia sound device, but not give it any
4681 TLB entries, which causes it to deadlock. Check for that. We do
4682 this in a function called from init_dmars(), instead of in a PCI
4683 quirk, because we don't want to print the obnoxious "BIOS broken"
4684 message if VT-d is actually disabled.
4685*/
4686static void __init check_tylersburg_isoch(void)
4687{
4688 struct pci_dev *pdev;
4689 uint32_t vtisochctrl;
4690
4691 /* If there's no Azalia in the system anyway, forget it. */
4692 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4693 if (!pdev)
4694 return;
4695 pci_dev_put(pdev);
4696
4697 /* System Management Registers. Might be hidden, in which case
4698 we can't do the sanity check. But that's OK, because the
4699 known-broken BIOSes _don't_ actually hide it, so far. */
4700 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4701 if (!pdev)
4702 return;
4703
4704 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4705 pci_dev_put(pdev);
4706 return;
4707 }
4708
4709 pci_dev_put(pdev);
4710
4711 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4712 if (vtisochctrl & 1)
4713 return;
4714
4715 /* Drop all bits other than the number of TLB entries */
4716 vtisochctrl &= 0x1c;
4717
4718 /* If we have the recommended number of TLB entries (16), fine. */
4719 if (vtisochctrl == 0x10)
4720 return;
4721
4722 /* Zero TLB entries? You get to ride the short bus to school. */
4723 if (!vtisochctrl) {
4724 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4725 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4726 dmi_get_system_info(DMI_BIOS_VENDOR),
4727 dmi_get_system_info(DMI_BIOS_VERSION),
4728 dmi_get_system_info(DMI_PRODUCT_VERSION));
4729 iommu_identity_mapping |= IDENTMAP_AZALIA;
4730 return;
4731 }
9f10e5bf
JR		 4732
4733 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
e0fc7e0b
DW		 4734 vtisochctrl);
4735}
ubuntu-zesty-kernel mirror