]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/iommu/intel-iommu.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
driver core: Add BUS_NOTIFY_REMOVED_DEVICE event
[mirror_ubuntu-artful-kernel.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 18 */
19
20#include <linux/init.h>
21#include <linux/bitmap.h>
5e0d2a6f 22#include <linux/debugfs.h>
54485c30 23#include <linux/export.h>
ba395927
KA		 24#include <linux/slab.h>
25#include <linux/irq.h>
26#include <linux/interrupt.h>
ba395927
KA		 27#include <linux/spinlock.h>
28#include <linux/pci.h>
29#include <linux/dmar.h>
30#include <linux/dma-mapping.h>
31#include <linux/mempool.h>
75f05569 32#include <linux/memory.h>
5e0d2a6f 33#include <linux/timer.h>
38717946 34#include <linux/iova.h>
5d450806 35#include <linux/iommu.h>
38717946 36#include <linux/intel-iommu.h>
134fac3f 37#include <linux/syscore_ops.h>
69575d38 38#include <linux/tboot.h>
adb2fe02 39#include <linux/dmi.h>
5cdede24 40#include <linux/pci-ats.h>
0ee332c1 41#include <linux/memblock.h>
36746436 42#include <linux/dma-contiguous.h>
8a8f422d 43#include <asm/irq_remapping.h>
ba395927 44#include <asm/cacheflush.h>
46a7fa27 45#include <asm/iommu.h>
ba395927 46
078e1ee2
JR		 47#include "irq_remapping.h"
48
5b6985ce
FY		 49#define ROOT_SIZE VTD_PAGE_SIZE
50#define CONTEXT_SIZE VTD_PAGE_SIZE
51
ba395927
KA		 52#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
53#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 54#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 55
56#define IOAPIC_RANGE_START (0xfee00000)
57#define IOAPIC_RANGE_END (0xfeefffff)
58#define IOVA_START_ADDR (0x1000)
59
60#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
61
4ed0d3e6 62#define MAX_AGAW_WIDTH 64
5c645b35 63#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 64
2ebe3151
DW		 65#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
66#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
67
68/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
69 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
70#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
71 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
72#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 73
f27be03b 74#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 75#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 76#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 77
df08cdc7
AM		 78/* page table handling */
79#define LEVEL_STRIDE (9)
80#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
81
6d1c56a9
OBC		 82/*
83 * This bitmap is used to advertise the page sizes our hardware support
84 * to the IOMMU core, which will then use this information to split
85 * physically contiguous memory regions it is mapping into page sizes
86 * that we support.
87 *
88 * Traditionally the IOMMU core just handed us the mappings directly,
89 * after making sure the size is an order of a 4KiB page and that the
90 * mapping has natural alignment.
91 *
92 * To retain this behavior, we currently advertise that we support
93 * all page sizes that are an order of 4KiB.
94 *
95 * If at some point we'd like to utilize the IOMMU core's new behavior,
96 * we could change this to advertise the real page sizes we support.
97 */
98#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
99
df08cdc7
AM		 100static inline int agaw_to_level(int agaw)
101{
102 return agaw + 2;
103}
104
105static inline int agaw_to_width(int agaw)
106{
5c645b35 107 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 108}
109
110static inline int width_to_agaw(int width)
111{
5c645b35 112 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 113}
114
115static inline unsigned int level_to_offset_bits(int level)
116{
117 return (level - 1) * LEVEL_STRIDE;
118}
119
120static inline int pfn_level_offset(unsigned long pfn, int level)
121{
122 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
123}
124
125static inline unsigned long level_mask(int level)
126{
127 return -1UL << level_to_offset_bits(level);
128}
129
130static inline unsigned long level_size(int level)
131{
132 return 1UL << level_to_offset_bits(level);
133}
134
135static inline unsigned long align_to_level(unsigned long pfn, int level)
136{
137 return (pfn + level_size(level) - 1) & level_mask(level);
138}
fd18de50 139
6dd9a7c7
YS		 140static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
141{
5c645b35 142 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 143}
144
dd4e8319
DW		 145/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
146 are never going to work. */
147static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
148{
149 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
150}
151
152static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
153{
154 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
155}
156static inline unsigned long page_to_dma_pfn(struct page *pg)
157{
158 return mm_to_dma_pfn(page_to_pfn(pg));
159}
160static inline unsigned long virt_to_dma_pfn(void *p)
161{
162 return page_to_dma_pfn(virt_to_page(p));
163}
164
d9630fe9
WH		 165/* global iommu list, set NULL for ignored DMAR units */
166static struct intel_iommu **g_iommus;
167
e0fc7e0b 168static void __init check_tylersburg_isoch(void);
9af88143
DW		 169static int rwbf_quirk;
170
b779260b
JC		 171/*
172 * set to 1 to panic kernel if can't successfully enable VT-d
173 * (used when kernel is launched w/ TXT)
174 */
175static int force_on = 0;
176
46b08e1a
MM		 177/*
178 * 0: Present
179 * 1-11: Reserved
180 * 12-63: Context Ptr (12 - (haw-1))
181 * 64-127: Reserved
182 */
183struct root_entry {
184 u64 val;
185 u64 rsvd1;
186};
187#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
188static inline bool root_present(struct root_entry *root)
189{
190 return (root->val & 1);
191}
192static inline void set_root_present(struct root_entry *root)
193{
194 root->val |= 1;
195}
196static inline void set_root_value(struct root_entry *root, unsigned long value)
197{
198 root->val |= value & VTD_PAGE_MASK;
199}
200
201static inline struct context_entry *
202get_context_addr_from_root(struct root_entry *root)
203{
204 return (struct context_entry *)
205 (root_present(root)?phys_to_virt(
206 root->val & VTD_PAGE_MASK) :
207 NULL);
208}
209
7a8fc25e
MM		 210/*
211 * low 64 bits:
212 * 0: present
213 * 1: fault processing disable
214 * 2-3: translation type
215 * 12-63: address space root
216 * high 64 bits:
217 * 0-2: address width
218 * 3-6: aval
219 * 8-23: domain id
220 */
221struct context_entry {
222 u64 lo;
223 u64 hi;
224};
c07e7d21
MM		 225
226static inline bool context_present(struct context_entry *context)
227{
228 return (context->lo & 1);
229}
230static inline void context_set_present(struct context_entry *context)
231{
232 context->lo |= 1;
233}
234
235static inline void context_set_fault_enable(struct context_entry *context)
236{
237 context->lo &= (((u64)-1) << 2) | 1;
238}
239
c07e7d21
MM		 240static inline void context_set_translation_type(struct context_entry *context,
241 unsigned long value)
242{
243 context->lo &= (((u64)-1) << 4) | 3;
244 context->lo |= (value & 3) << 2;
245}
246
247static inline void context_set_address_root(struct context_entry *context,
248 unsigned long value)
249{
250 context->lo |= value & VTD_PAGE_MASK;
251}
252
253static inline void context_set_address_width(struct context_entry *context,
254 unsigned long value)
255{
256 context->hi |= value & 7;
257}
258
259static inline void context_set_domain_id(struct context_entry *context,
260 unsigned long value)
261{
262 context->hi |= (value & ((1 << 16) - 1)) << 8;
263}
264
265static inline void context_clear_entry(struct context_entry *context)
266{
267 context->lo = 0;
268 context->hi = 0;
269}
7a8fc25e 270
622ba12a
MM		 271/*
272 * 0: readable
273 * 1: writable
274 * 2-6: reserved
275 * 7: super page
9cf06697
SY		 276 * 8-10: available
277 * 11: snoop behavior
622ba12a
MM		 278 * 12-63: Host physcial address
279 */
280struct dma_pte {
281 u64 val;
282};
622ba12a 283
19c239ce
MM		 284static inline void dma_clear_pte(struct dma_pte *pte)
285{
286 pte->val = 0;
287}
288
19c239ce
MM		 289static inline u64 dma_pte_addr(struct dma_pte *pte)
290{
c85994e4
DW		 291#ifdef CONFIG_64BIT
292 return pte->val & VTD_PAGE_MASK;
293#else
294 /* Must have a full atomic 64-bit read */
1a8bd481 295 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 296#endif
19c239ce
MM		 297}
298
19c239ce
MM		 299static inline bool dma_pte_present(struct dma_pte *pte)
300{
301 return (pte->val & 3) != 0;
302}
622ba12a 303
4399c8bf
AK		 304static inline bool dma_pte_superpage(struct dma_pte *pte)
305{
c3c75eb7 306 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 307}
308
75e6bf96
DW		 309static inline int first_pte_in_page(struct dma_pte *pte)
310{
311 return !((unsigned long)pte & ~VTD_PAGE_MASK);
312}
313
2c2e2c38
FY		 314/*
315 * This domain is a statically identity mapping domain.
316 * 1. This domain creats a static 1:1 mapping to all usable memory.
317 * 2. It maps to each iommu if successful.
318 * 3. Each iommu mapps to this domain if successful.
319 */
19943b0e
DW		 320static struct dmar_domain *si_domain;
321static int hw_pass_through = 1;
2c2e2c38 322
1ce28feb
WH		 323/* domain represents a virtual machine, more than one devices
324 * across iommus may be owned in one domain, e.g. kvm guest.
325 */
ab8dfe25 326#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 327
2c2e2c38 328/* si_domain contains mulitple devices */
ab8dfe25 329#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 330
1b198bb0
MT		 331/* define the limit of IOMMUs supported in each domain */
332#ifdef CONFIG_X86
333# define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
334#else
335# define IOMMU_UNITS_SUPPORTED 64
336#endif
337
99126f7c
MM		 338struct dmar_domain {
339 int id; /* domain id */
4c923d47 340 int nid; /* node id */
1b198bb0
MT		 341 DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
342 /* bitmap of iommus this domain uses*/
99126f7c
MM		 343
344 struct list_head devices; /* all devices' list */
345 struct iova_domain iovad; /* iova's that belong to this domain */
346
347 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 348 int gaw; /* max guest address width */
349
350 /* adjusted guest address width, 0 is level 2 30-bit */
351 int agaw;
352
3b5410e7 353 int flags; /* flags to find out type of domain */
8e604097
WH		 354
355 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 356 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 357 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 358 int iommu_superpage;/* Level of superpages supported:
359 0 == 4KiB (no superpages), 1 == 2MiB,
360 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 361 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 362 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 363};
364
a647dacb
MM		 365/* PCI domain-device relationship */
366struct device_domain_info {
367 struct list_head link; /* link to domain siblings */
368 struct list_head global; /* link to global list */
276dbf99 369 u8 bus; /* PCI bus number */
a647dacb 370 u8 devfn; /* PCI devfn number */
0bcb3e28 371 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 372 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 373 struct dmar_domain *domain; /* pointer to domain */
374};
375
b94e4117
JL		 376struct dmar_rmrr_unit {
377 struct list_head list; /* list of rmrr units */
378 struct acpi_dmar_header *hdr; /* ACPI header */
379 u64 base_address; /* reserved base address*/
380 u64 end_address; /* reserved end address */
832bd858 381 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 382 int devices_cnt; /* target device count */
383};
384
385struct dmar_atsr_unit {
386 struct list_head list; /* list of ATSR units */
387 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 388 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 389 int devices_cnt; /* target device count */
390 u8 include_all:1; /* include all ports */
391};
392
393static LIST_HEAD(dmar_atsr_units);
394static LIST_HEAD(dmar_rmrr_units);
395
396#define for_each_rmrr_units(rmrr) \
397 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
398
5e0d2a6f 399static void flush_unmaps_timeout(unsigned long data);
400
b707cb02 401static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 402
80b20dd8 403#define HIGH_WATER_MARK 250
404struct deferred_flush_tables {
405 int next;
406 struct iova *iova[HIGH_WATER_MARK];
407 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 408 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 409};
410
411static struct deferred_flush_tables *deferred_flush;
412
5e0d2a6f 413/* bitmap for indexing intel_iommus */
5e0d2a6f 414static int g_num_of_iommus;
415
416static DEFINE_SPINLOCK(async_umap_flush_lock);
417static LIST_HEAD(unmaps_to_do);
418
419static int timer_on;
420static long list_size;
5e0d2a6f 421
92d03cc8 422static void domain_exit(struct dmar_domain *domain);
ba395927 423static void domain_remove_dev_info(struct dmar_domain *domain);
b94e4117 424static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 425 struct device *dev);
92d03cc8 426static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 427 struct device *dev);
2a46ddf7
JL		 428static int domain_detach_iommu(struct dmar_domain *domain,
429 struct intel_iommu *iommu);
ba395927 430
d3f13810 431#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 432int dmar_disabled = 0;
433#else
434int dmar_disabled = 1;
d3f13810 435#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 436
8bc1f85c
ED		 437int intel_iommu_enabled = 0;
438EXPORT_SYMBOL_GPL(intel_iommu_enabled);
439
2d9e667e 440static int dmar_map_gfx = 1;
7d3b03ce 441static int dmar_forcedac;
5e0d2a6f 442static int intel_iommu_strict;
6dd9a7c7 443static int intel_iommu_superpage = 1;
ba395927 444
c0771df8
DW		 445int intel_iommu_gfx_mapped;
446EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
447
ba395927
KA		 448#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
449static DEFINE_SPINLOCK(device_domain_lock);
450static LIST_HEAD(device_domain_list);
451
b22f6434 452static const struct iommu_ops intel_iommu_ops;
a8bcbb0d 453
ba395927
KA		 454static int __init intel_iommu_setup(char *str)
455{
456 if (!str)
457 return -EINVAL;
458 while (*str) {
0cd5c3c8
KM		 459 if (!strncmp(str, "on", 2)) {
460 dmar_disabled = 0;
461 printk(KERN_INFO "Intel-IOMMU: enabled\n");
462 } else if (!strncmp(str, "off", 3)) {
ba395927 463 dmar_disabled = 1;
0cd5c3c8 464 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 465 } else if (!strncmp(str, "igfx_off", 8)) {
466 dmar_map_gfx = 0;
467 printk(KERN_INFO
468 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 469 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 470 printk(KERN_INFO
7d3b03ce
KA		 471 "Intel-IOMMU: Forcing DAC for PCI devices\n");
472 dmar_forcedac = 1;
5e0d2a6f 473 } else if (!strncmp(str, "strict", 6)) {
474 printk(KERN_INFO
475 "Intel-IOMMU: disable batched IOTLB flush\n");
476 intel_iommu_strict = 1;
6dd9a7c7
YS		 477 } else if (!strncmp(str, "sp_off", 6)) {
478 printk(KERN_INFO
479 "Intel-IOMMU: disable supported super page\n");
480 intel_iommu_superpage = 0;
ba395927
KA		 481 }
482
483 str += strcspn(str, ",");
484 while (*str == ',')
485 str++;
486 }
487 return 0;
488}
489__setup("intel_iommu=", intel_iommu_setup);
490
491static struct kmem_cache *iommu_domain_cache;
492static struct kmem_cache *iommu_devinfo_cache;
493static struct kmem_cache *iommu_iova_cache;
494
4c923d47 495static inline void *alloc_pgtable_page(int node)
eb3fa7cb 496{
4c923d47
SS		 497 struct page *page;
498 void *vaddr = NULL;
eb3fa7cb 499
4c923d47
SS		 500 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
501 if (page)
502 vaddr = page_address(page);
eb3fa7cb 503 return vaddr;
ba395927
KA		 504}
505
506static inline void free_pgtable_page(void *vaddr)
507{
508 free_page((unsigned long)vaddr);
509}
510
511static inline void *alloc_domain_mem(void)
512{
354bb65e 513 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 514}
515
38717946 516static void free_domain_mem(void *vaddr)
ba395927
KA		 517{
518 kmem_cache_free(iommu_domain_cache, vaddr);
519}
520
521static inline void * alloc_devinfo_mem(void)
522{
354bb65e 523 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 524}
525
526static inline void free_devinfo_mem(void *vaddr)
527{
528 kmem_cache_free(iommu_devinfo_cache, vaddr);
529}
530
531struct iova *alloc_iova_mem(void)
532{
354bb65e 533 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
ba395927
KA		 534}
535
536void free_iova_mem(struct iova *iova)
537{
538 kmem_cache_free(iommu_iova_cache, iova);
539}
540
ab8dfe25
JL		 541static inline int domain_type_is_vm(struct dmar_domain *domain)
542{
543 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
544}
545
546static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
547{
548 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
549 DOMAIN_FLAG_STATIC_IDENTITY);
550}
1b573683 551
162d1b10
JL		 552static inline int domain_pfn_supported(struct dmar_domain *domain,
553 unsigned long pfn)
554{
555 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
556
557 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
558}
559
4ed0d3e6 560static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 561{
562 unsigned long sagaw;
563 int agaw = -1;
564
565 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 566 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 567 agaw >= 0; agaw--) {
568 if (test_bit(agaw, &sagaw))
569 break;
570 }
571
572 return agaw;
573}
574
4ed0d3e6
FY		 575/*
576 * Calculate max SAGAW for each iommu.
577 */
578int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
579{
580 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
581}
582
583/*
584 * calculate agaw for each iommu.
585 * "SAGAW" may be different across iommus, use a default agaw, and
586 * get a supported less agaw for iommus that don't support the default agaw.
587 */
588int iommu_calculate_agaw(struct intel_iommu *iommu)
589{
590 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
591}
592
2c2e2c38 593/* This functionin only returns single iommu in a domain */
8c11e798
WH		 594static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
595{
596 int iommu_id;
597
2c2e2c38 598 /* si_domain and vm domain should not get here. */
ab8dfe25 599 BUG_ON(domain_type_is_vm_or_si(domain));
1b198bb0 600 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 601 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
602 return NULL;
603
604 return g_iommus[iommu_id];
605}
606
8e604097
WH		 607static void domain_update_iommu_coherency(struct dmar_domain *domain)
608{
d0501960
DW		 609 struct dmar_drhd_unit *drhd;
610 struct intel_iommu *iommu;
611 int i, found = 0;
2e12bc29 612
d0501960 613 domain->iommu_coherency = 1;
8e604097 614
1b198bb0 615 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
d0501960 616 found = 1;
8e604097
WH		 617 if (!ecap_coherent(g_iommus[i]->ecap)) {
618 domain->iommu_coherency = 0;
619 break;
620 }
8e604097 621 }
d0501960
DW		 622 if (found)
623 return;
624
625 /* No hardware attached; use lowest common denominator */
626 rcu_read_lock();
627 for_each_active_iommu(iommu, drhd) {
628 if (!ecap_coherent(iommu->ecap)) {
629 domain->iommu_coherency = 0;
630 break;
631 }
632 }
633 rcu_read_unlock();
8e604097
WH		 634}
635
161f6934 636static int domain_update_iommu_snooping(struct intel_iommu *skip)
58c610bd 637{
161f6934
JL		 638 struct dmar_drhd_unit *drhd;
639 struct intel_iommu *iommu;
640 int ret = 1;
58c610bd 641
161f6934
JL		 642 rcu_read_lock();
643 for_each_active_iommu(iommu, drhd) {
644 if (iommu != skip) {
645 if (!ecap_sc_support(iommu->ecap)) {
646 ret = 0;
647 break;
648 }
58c610bd 649 }
58c610bd 650 }
161f6934
JL		 651 rcu_read_unlock();
652
653 return ret;
58c610bd
SY		 654}
655
161f6934 656static int domain_update_iommu_superpage(struct intel_iommu *skip)
6dd9a7c7 657{
8140a95d 658 struct dmar_drhd_unit *drhd;
161f6934 659 struct intel_iommu *iommu;
8140a95d 660 int mask = 0xf;
6dd9a7c7
YS		 661
662 if (!intel_iommu_superpage) {
161f6934 663 return 0;
6dd9a7c7
YS		 664 }
665
8140a95d 666 /* set iommu_superpage to the smallest common denominator */
0e242612 667 rcu_read_lock();
8140a95d 668 for_each_active_iommu(iommu, drhd) {
161f6934
JL		 669 if (iommu != skip) {
670 mask &= cap_super_page_val(iommu->cap);
671 if (!mask)
672 break;
6dd9a7c7
YS		 673 }
674 }
0e242612
JL		 675 rcu_read_unlock();
676
161f6934 677 return fls(mask);
6dd9a7c7
YS		 678}
679
58c610bd
SY		 680/* Some capabilities may be different across iommus */
681static void domain_update_iommu_cap(struct dmar_domain *domain)
682{
683 domain_update_iommu_coherency(domain);
161f6934
JL		 684 domain->iommu_snooping = domain_update_iommu_snooping(NULL);
685 domain->iommu_superpage = domain_update_iommu_superpage(NULL);
58c610bd
SY		 686}
687
156baca8 688static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 689{
690 struct dmar_drhd_unit *drhd = NULL;
b683b230 691 struct intel_iommu *iommu;
156baca8
DW		 692 struct device *tmp;
693 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 694 u16 segment = 0;
c7151a8d
WH		 695 int i;
696
156baca8
DW		 697 if (dev_is_pci(dev)) {
698 pdev = to_pci_dev(dev);
699 segment = pci_domain_nr(pdev->bus);
700 } else if (ACPI_COMPANION(dev))
701 dev = &ACPI_COMPANION(dev)->dev;
702
0e242612 703 rcu_read_lock();
b683b230 704 for_each_active_iommu(iommu, drhd) {
156baca8 705 if (pdev && segment != drhd->segment)
276dbf99 706 continue;
c7151a8d 707
b683b230 708 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 709 drhd->devices_cnt, i, tmp) {
710 if (tmp == dev) {
711 *bus = drhd->devices[i].bus;
712 *devfn = drhd->devices[i].devfn;
b683b230 713 goto out;
156baca8
DW		 714 }
715
716 if (!pdev || !dev_is_pci(tmp))
717 continue;
718
719 ptmp = to_pci_dev(tmp);
720 if (ptmp->subordinate &&
721 ptmp->subordinate->number <= pdev->bus->number &&
722 ptmp->subordinate->busn_res.end >= pdev->bus->number)
723 goto got_pdev;
924b6231 724 }
c7151a8d 725
156baca8
DW		 726 if (pdev && drhd->include_all) {
727 got_pdev:
728 *bus = pdev->bus->number;
729 *devfn = pdev->devfn;
b683b230 730 goto out;
156baca8 731 }
c7151a8d 732 }
b683b230 733 iommu = NULL;
156baca8 734 out:
0e242612 735 rcu_read_unlock();
c7151a8d 736
b683b230 737 return iommu;
c7151a8d
WH		 738}
739
5331fe6f
WH		 740static void domain_flush_cache(struct dmar_domain *domain,
741 void *addr, int size)
742{
743 if (!domain->iommu_coherency)
744 clflush_cache_range(addr, size);
745}
746
ba395927
KA		 747/* Gets context entry for a given bus and devfn */
748static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
749 u8 bus, u8 devfn)
750{
751 struct root_entry *root;
752 struct context_entry *context;
753 unsigned long phy_addr;
754 unsigned long flags;
755
756 spin_lock_irqsave(&iommu->lock, flags);
757 root = &iommu->root_entry[bus];
758 context = get_context_addr_from_root(root);
759 if (!context) {
4c923d47
SS		 760 context = (struct context_entry *)
761 alloc_pgtable_page(iommu->node);
ba395927
KA		 762 if (!context) {
763 spin_unlock_irqrestore(&iommu->lock, flags);
764 return NULL;
765 }
5b6985ce 766 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 767 phy_addr = virt_to_phys((void *)context);
768 set_root_value(root, phy_addr);
769 set_root_present(root);
770 __iommu_flush_cache(iommu, root, sizeof(*root));
771 }
772 spin_unlock_irqrestore(&iommu->lock, flags);
773 return &context[devfn];
774}
775
776static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
777{
778 struct root_entry *root;
779 struct context_entry *context;
780 int ret;
781 unsigned long flags;
782
783 spin_lock_irqsave(&iommu->lock, flags);
784 root = &iommu->root_entry[bus];
785 context = get_context_addr_from_root(root);
786 if (!context) {
787 ret = 0;
788 goto out;
789 }
c07e7d21 790 ret = context_present(&context[devfn]);
ba395927
KA		 791out:
792 spin_unlock_irqrestore(&iommu->lock, flags);
793 return ret;
794}
795
796static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
797{
798 struct root_entry *root;
799 struct context_entry *context;
800 unsigned long flags;
801
802 spin_lock_irqsave(&iommu->lock, flags);
803 root = &iommu->root_entry[bus];
804 context = get_context_addr_from_root(root);
805 if (context) {
c07e7d21 806 context_clear_entry(&context[devfn]);
ba395927
KA		 807 __iommu_flush_cache(iommu, &context[devfn], \
808 sizeof(*context));
809 }
810 spin_unlock_irqrestore(&iommu->lock, flags);
811}
812
813static void free_context_table(struct intel_iommu *iommu)
814{
815 struct root_entry *root;
816 int i;
817 unsigned long flags;
818 struct context_entry *context;
819
820 spin_lock_irqsave(&iommu->lock, flags);
821 if (!iommu->root_entry) {
822 goto out;
823 }
824 for (i = 0; i < ROOT_ENTRY_NR; i++) {
825 root = &iommu->root_entry[i];
826 context = get_context_addr_from_root(root);
827 if (context)
828 free_pgtable_page(context);
829 }
830 free_pgtable_page(iommu->root_entry);
831 iommu->root_entry = NULL;
832out:
833 spin_unlock_irqrestore(&iommu->lock, flags);
834}
835
b026fd28 836static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 837 unsigned long pfn, int *target_level)
ba395927 838{
ba395927
KA		 839 struct dma_pte *parent, *pte = NULL;
840 int level = agaw_to_level(domain->agaw);
4399c8bf 841 int offset;
ba395927
KA		 842
843 BUG_ON(!domain->pgd);
f9423606 844
162d1b10 845 if (!domain_pfn_supported(domain, pfn))
f9423606
JS		 846 /* Address beyond IOMMU's addressing capabilities. */
847 return NULL;
848
ba395927
KA		 849 parent = domain->pgd;
850
5cf0a76f 851 while (1) {
ba395927
KA		 852 void *tmp_page;
853
b026fd28 854 offset = pfn_level_offset(pfn, level);
ba395927 855 pte = &parent[offset];
5cf0a76f 856 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 857 break;
5cf0a76f 858 if (level == *target_level)
ba395927
KA		 859 break;
860
19c239ce 861 if (!dma_pte_present(pte)) {
c85994e4
DW		 862 uint64_t pteval;
863
4c923d47 864 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 865
206a73c1 866 if (!tmp_page)
ba395927 867 return NULL;
206a73c1 868
c85994e4 869 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 870 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 871 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 872 /* Someone else set it while we were thinking; use theirs. */
873 free_pgtable_page(tmp_page);
effad4b5 874 else
c85994e4 875 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 876 }
5cf0a76f
DW		 877 if (level == 1)
878 break;
879
19c239ce 880 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 881 level--;
882 }
883
5cf0a76f
DW		 884 if (!*target_level)
885 *target_level = level;
886
ba395927
KA		 887 return pte;
888}
889
6dd9a7c7 890
ba395927 891/* return address's pte at specific level */
90dcfb5e
DW		 892static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
893 unsigned long pfn,
6dd9a7c7 894 int level, int *large_page)
ba395927
KA		 895{
896 struct dma_pte *parent, *pte = NULL;
897 int total = agaw_to_level(domain->agaw);
898 int offset;
899
900 parent = domain->pgd;
901 while (level <= total) {
90dcfb5e 902 offset = pfn_level_offset(pfn, total);
ba395927
KA		 903 pte = &parent[offset];
904 if (level == total)
905 return pte;
906
6dd9a7c7
YS		 907 if (!dma_pte_present(pte)) {
908 *large_page = total;
ba395927 909 break;
6dd9a7c7
YS		 910 }
911
e16922af 912 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 913 *large_page = total;
914 return pte;
915 }
916
19c239ce 917 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 918 total--;
919 }
920 return NULL;
921}
922
ba395927 923/* clear last level pte, a tlb flush should be followed */
5cf0a76f 924static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 925 unsigned long start_pfn,
926 unsigned long last_pfn)
ba395927 927{
6dd9a7c7 928 unsigned int large_page = 1;
310a5ab9 929 struct dma_pte *first_pte, *pte;
66eae846 930
162d1b10
JL		 931 BUG_ON(!domain_pfn_supported(domain, start_pfn));
932 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 933 BUG_ON(start_pfn > last_pfn);
ba395927 934
04b18e65 935 /* we don't need lock here; nobody else touches the iova range */
59c36286 936 do {
6dd9a7c7
YS		 937 large_page = 1;
938 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 939 if (!pte) {
6dd9a7c7 940 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 941 continue;
942 }
6dd9a7c7 943 do {
310a5ab9 944 dma_clear_pte(pte);
6dd9a7c7 945 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 946 pte++;
75e6bf96
DW		 947 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
948
310a5ab9
DW		 949 domain_flush_cache(domain, first_pte,
950 (void *)pte - (void *)first_pte);
59c36286
DW		 951
952 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 953}
954
3269ee0b
AW		 955static void dma_pte_free_level(struct dmar_domain *domain, int level,
956 struct dma_pte *pte, unsigned long pfn,
957 unsigned long start_pfn, unsigned long last_pfn)
958{
959 pfn = max(start_pfn, pfn);
960 pte = &pte[pfn_level_offset(pfn, level)];
961
962 do {
963 unsigned long level_pfn;
964 struct dma_pte *level_pte;
965
966 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
967 goto next;
968
969 level_pfn = pfn & level_mask(level - 1);
970 level_pte = phys_to_virt(dma_pte_addr(pte));
971
972 if (level > 2)
973 dma_pte_free_level(domain, level - 1, level_pte,
974 level_pfn, start_pfn, last_pfn);
975
976 /* If range covers entire pagetable, free it */
977 if (!(start_pfn > level_pfn ||
08336fd2 978 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 979 dma_clear_pte(pte);
980 domain_flush_cache(domain, pte, sizeof(*pte));
981 free_pgtable_page(level_pte);
982 }
983next:
984 pfn += level_size(level);
985 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
986}
987
ba395927
KA		 988/* free page table pages. last level pte should already be cleared */
989static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 990 unsigned long start_pfn,
991 unsigned long last_pfn)
ba395927 992{
162d1b10
JL		 993 BUG_ON(!domain_pfn_supported(domain, start_pfn));
994 BUG_ON(!domain_pfn_supported(domain, last_pfn));
59c36286 995 BUG_ON(start_pfn > last_pfn);
ba395927 996
d41a4adb
JL		 997 dma_pte_clear_range(domain, start_pfn, last_pfn);
998
f3a0a52f 999 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 1000 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
1001 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 1002
ba395927 1003 /* free pgd */
d794dc9b 1004 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 1005 free_pgtable_page(domain->pgd);
1006 domain->pgd = NULL;
1007 }
1008}
1009
ea8ea460
DW		 1010/* When a page at a given level is being unlinked from its parent, we don't
1011 need to *modify* it at all. All we need to do is make a list of all the
1012 pages which can be freed just as soon as we've flushed the IOTLB and we
1013 know the hardware page-walk will no longer touch them.
1014 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1015 be freed. */
1016static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1017 int level, struct dma_pte *pte,
1018 struct page *freelist)
1019{
1020 struct page *pg;
1021
1022 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1023 pg->freelist = freelist;
1024 freelist = pg;
1025
1026 if (level == 1)
1027 return freelist;
1028
adeb2590
JL		 1029 pte = page_address(pg);
1030 do {
ea8ea460
DW		 1031 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1032 freelist = dma_pte_list_pagetables(domain, level - 1,
1033 pte, freelist);
adeb2590
JL		 1034 pte++;
1035 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1036
1037 return freelist;
1038}
1039
1040static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1041 struct dma_pte *pte, unsigned long pfn,
1042 unsigned long start_pfn,
1043 unsigned long last_pfn,
1044 struct page *freelist)
1045{
1046 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1047
1048 pfn = max(start_pfn, pfn);
1049 pte = &pte[pfn_level_offset(pfn, level)];
1050
1051 do {
1052 unsigned long level_pfn;
1053
1054 if (!dma_pte_present(pte))
1055 goto next;
1056
1057 level_pfn = pfn & level_mask(level);
1058
1059 /* If range covers entire pagetable, free it */
1060 if (start_pfn <= level_pfn &&
1061 last_pfn >= level_pfn + level_size(level) - 1) {
1062 /* These suborbinate page tables are going away entirely. Don't
1063 bother to clear them; we're just going to *free* them. */
1064 if (level > 1 && !dma_pte_superpage(pte))
1065 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1066
1067 dma_clear_pte(pte);
1068 if (!first_pte)
1069 first_pte = pte;
1070 last_pte = pte;
1071 } else if (level > 1) {
1072 /* Recurse down into a level that isn't *entirely* obsolete */
1073 freelist = dma_pte_clear_level(domain, level - 1,
1074 phys_to_virt(dma_pte_addr(pte)),
1075 level_pfn, start_pfn, last_pfn,
1076 freelist);
1077 }
1078next:
1079 pfn += level_size(level);
1080 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1081
1082 if (first_pte)
1083 domain_flush_cache(domain, first_pte,
1084 (void *)++last_pte - (void *)first_pte);
1085
1086 return freelist;
1087}
1088
1089/* We can't just free the pages because the IOMMU may still be walking
1090 the page tables, and may have cached the intermediate levels. The
1091 pages can only be freed after the IOTLB flush has been done. */
1092struct page *domain_unmap(struct dmar_domain *domain,
1093 unsigned long start_pfn,
1094 unsigned long last_pfn)
1095{
ea8ea460
DW		 1096 struct page *freelist = NULL;
1097
162d1b10
JL		 1098 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1099 BUG_ON(!domain_pfn_supported(domain, last_pfn));
ea8ea460
DW		 1100 BUG_ON(start_pfn > last_pfn);
1101
1102 /* we don't need lock here; nobody else touches the iova range */
1103 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1104 domain->pgd, 0, start_pfn, last_pfn, NULL);
1105
1106 /* free pgd */
1107 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1108 struct page *pgd_page = virt_to_page(domain->pgd);
1109 pgd_page->freelist = freelist;
1110 freelist = pgd_page;
1111
1112 domain->pgd = NULL;
1113 }
1114
1115 return freelist;
1116}
1117
1118void dma_free_pagelist(struct page *freelist)
1119{
1120 struct page *pg;
1121
1122 while ((pg = freelist)) {
1123 freelist = pg->freelist;
1124 free_pgtable_page(page_address(pg));
1125 }
1126}
1127
ba395927
KA		 1128/* iommu handling */
1129static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1130{
1131 struct root_entry *root;
1132 unsigned long flags;
1133
4c923d47 1134 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ba395927
KA		 1135 if (!root)
1136 return -ENOMEM;
1137
5b6985ce 1138 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1139
1140 spin_lock_irqsave(&iommu->lock, flags);
1141 iommu->root_entry = root;
1142 spin_unlock_irqrestore(&iommu->lock, flags);
1143
1144 return 0;
1145}
1146
ba395927
KA		 1147static void iommu_set_root_entry(struct intel_iommu *iommu)
1148{
1149 void *addr;
c416daa9 1150 u32 sts;
ba395927
KA		 1151 unsigned long flag;
1152
1153 addr = iommu->root_entry;
1154
1f5b3c3f 1155 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1156 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
1157
c416daa9 1158 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1159
1160 /* Make sure hardware complete it */
1161 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1162 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1163
1f5b3c3f 1164 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1165}
1166
1167static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1168{
1169 u32 val;
1170 unsigned long flag;
1171
9af88143 1172 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1173 return;
ba395927 1174
1f5b3c3f 1175 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1176 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1177
1178 /* Make sure hardware complete it */
1179 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1180 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1181
1f5b3c3f 1182 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1183}
1184
1185/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1186static void __iommu_flush_context(struct intel_iommu *iommu,
1187 u16 did, u16 source_id, u8 function_mask,
1188 u64 type)
ba395927
KA		 1189{
1190 u64 val = 0;
1191 unsigned long flag;
1192
ba395927
KA		 1193 switch (type) {
1194 case DMA_CCMD_GLOBAL_INVL:
1195 val = DMA_CCMD_GLOBAL_INVL;
1196 break;
1197 case DMA_CCMD_DOMAIN_INVL:
1198 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1199 break;
1200 case DMA_CCMD_DEVICE_INVL:
1201 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1202 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1203 break;
1204 default:
1205 BUG();
1206 }
1207 val |= DMA_CCMD_ICC;
1208
1f5b3c3f 1209 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1210 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1211
1212 /* Make sure hardware complete it */
1213 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1214 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1215
1f5b3c3f 1216 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1217}
1218
ba395927 1219/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1220static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1221 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1222{
1223 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1224 u64 val = 0, val_iva = 0;
1225 unsigned long flag;
1226
ba395927
KA		 1227 switch (type) {
1228 case DMA_TLB_GLOBAL_FLUSH:
1229 /* global flush doesn't need set IVA_REG */
1230 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1231 break;
1232 case DMA_TLB_DSI_FLUSH:
1233 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1234 break;
1235 case DMA_TLB_PSI_FLUSH:
1236 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1237 /* IH bit is passed in as part of address */
ba395927
KA		 1238 val_iva = size_order | addr;
1239 break;
1240 default:
1241 BUG();
1242 }
1243 /* Note: set drain read/write */
1244#if 0
1245 /*
1246 * This is probably to be super secure.. Looks like we can
1247 * ignore it without any impact.
1248 */
1249 if (cap_read_drain(iommu->cap))
1250 val |= DMA_TLB_READ_DRAIN;
1251#endif
1252 if (cap_write_drain(iommu->cap))
1253 val |= DMA_TLB_WRITE_DRAIN;
1254
1f5b3c3f 1255 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1256 /* Note: Only uses first TLB reg currently */
1257 if (val_iva)
1258 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1259 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1260
1261 /* Make sure hardware complete it */
1262 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1263 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1264
1f5b3c3f 1265 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1266
1267 /* check IOTLB invalidation granularity */
1268 if (DMA_TLB_IAIG(val) == 0)
1269 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1270 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1271 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1272 (unsigned long long)DMA_TLB_IIRG(type),
1273 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1274}
1275
64ae892b
DW		 1276static struct device_domain_info *
1277iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1278 u8 bus, u8 devfn)
93a23a72
YZ		 1279{
1280 int found = 0;
1281 unsigned long flags;
1282 struct device_domain_info *info;
0bcb3e28 1283 struct pci_dev *pdev;
93a23a72
YZ		 1284
1285 if (!ecap_dev_iotlb_support(iommu->ecap))
1286 return NULL;
1287
1288 if (!iommu->qi)
1289 return NULL;
1290
1291 spin_lock_irqsave(&device_domain_lock, flags);
1292 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1293 if (info->iommu == iommu && info->bus == bus &&
1294 info->devfn == devfn) {
93a23a72
YZ		 1295 found = 1;
1296 break;
1297 }
1298 spin_unlock_irqrestore(&device_domain_lock, flags);
1299
0bcb3e28 1300 if (!found || !info->dev || !dev_is_pci(info->dev))
93a23a72
YZ		 1301 return NULL;
1302
0bcb3e28
DW		 1303 pdev = to_pci_dev(info->dev);
1304
1305 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
93a23a72
YZ		 1306 return NULL;
1307
0bcb3e28 1308 if (!dmar_find_matched_atsr_unit(pdev))
93a23a72
YZ		 1309 return NULL;
1310
93a23a72
YZ		 1311 return info;
1312}
1313
1314static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1315{
0bcb3e28 1316 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1317 return;
1318
0bcb3e28 1319 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
93a23a72
YZ		 1320}
1321
1322static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1323{
0bcb3e28
DW		 1324 if (!info->dev || !dev_is_pci(info->dev) ||
1325 !pci_ats_enabled(to_pci_dev(info->dev)))
93a23a72
YZ		 1326 return;
1327
0bcb3e28 1328 pci_disable_ats(to_pci_dev(info->dev));
93a23a72
YZ		 1329}
1330
1331static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1332 u64 addr, unsigned mask)
1333{
1334 u16 sid, qdep;
1335 unsigned long flags;
1336 struct device_domain_info *info;
1337
1338 spin_lock_irqsave(&device_domain_lock, flags);
1339 list_for_each_entry(info, &domain->devices, link) {
0bcb3e28
DW		 1340 struct pci_dev *pdev;
1341 if (!info->dev || !dev_is_pci(info->dev))
1342 continue;
1343
1344 pdev = to_pci_dev(info->dev);
1345 if (!pci_ats_enabled(pdev))
93a23a72
YZ		 1346 continue;
1347
1348 sid = info->bus << 8 | info->devfn;
0bcb3e28 1349 qdep = pci_ats_queue_depth(pdev);
93a23a72
YZ		 1350 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1351 }
1352 spin_unlock_irqrestore(&device_domain_lock, flags);
1353}
1354
1f0ef2aa 1355static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
ea8ea460 1356 unsigned long pfn, unsigned int pages, int ih, int map)
ba395927 1357{
9dd2fe89 1358 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1359 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1360
ba395927
KA		 1361 BUG_ON(pages == 0);
1362
ea8ea460
DW		 1363 if (ih)
1364 ih = 1 << 6;
ba395927 1365 /*
9dd2fe89
YZ		 1366 * Fallback to domain selective flush if no PSI support or the size is
1367 * too big.
ba395927
KA		 1368 * PSI requires page size to be 2 ^ x, and the base address is naturally
1369 * aligned to the size
1370 */
9dd2fe89
YZ		 1371 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1372 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1373 DMA_TLB_DSI_FLUSH);
9dd2fe89 1374 else
ea8ea460 1375 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1376 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1377
1378 /*
82653633
NA		 1379 * In caching mode, changes of pages from non-present to present require
1380 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1381 */
82653633 1382 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1383 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1384}
1385
f8bab735 1386static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1387{
1388 u32 pmen;
1389 unsigned long flags;
1390
1f5b3c3f 1391 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1392 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1393 pmen &= ~DMA_PMEN_EPM;
1394 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1395
1396 /* wait for the protected region status bit to clear */
1397 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1398 readl, !(pmen & DMA_PMEN_PRS), pmen);
1399
1f5b3c3f 1400 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1401}
1402
2a41ccee 1403static void iommu_enable_translation(struct intel_iommu *iommu)
ba395927
KA		 1404{
1405 u32 sts;
1406 unsigned long flags;
1407
1f5b3c3f 1408 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1409 iommu->gcmd |= DMA_GCMD_TE;
1410 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1411
1412 /* Make sure hardware complete it */
1413 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1414 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1415
1f5b3c3f 1416 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1417}
1418
2a41ccee 1419static void iommu_disable_translation(struct intel_iommu *iommu)
ba395927
KA		 1420{
1421 u32 sts;
1422 unsigned long flag;
1423
1f5b3c3f 1424 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1425 iommu->gcmd &= ~DMA_GCMD_TE;
1426 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1427
1428 /* Make sure hardware complete it */
1429 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1430 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1431
1f5b3c3f 1432 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1433}
1434
3460a6d9 1435
ba395927
KA		 1436static int iommu_init_domains(struct intel_iommu *iommu)
1437{
1438 unsigned long ndomains;
1439 unsigned long nlongs;
1440
1441 ndomains = cap_ndoms(iommu->cap);
852bdb04
JL		 1442 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1443 iommu->seq_id, ndomains);
ba395927
KA		 1444 nlongs = BITS_TO_LONGS(ndomains);
1445
94a91b50
DD		 1446 spin_lock_init(&iommu->lock);
1447
ba395927
KA		 1448 /* TBD: there might be 64K domains,
1449 * consider other allocation for future chip
1450 */
1451 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1452 if (!iommu->domain_ids) {
852bdb04
JL		 1453 pr_err("IOMMU%d: allocating domain id array failed\n",
1454 iommu->seq_id);
ba395927
KA		 1455 return -ENOMEM;
1456 }
1457 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1458 GFP_KERNEL);
1459 if (!iommu->domains) {
852bdb04
JL		 1460 pr_err("IOMMU%d: allocating domain array failed\n",
1461 iommu->seq_id);
1462 kfree(iommu->domain_ids);
1463 iommu->domain_ids = NULL;
ba395927
KA		 1464 return -ENOMEM;
1465 }
1466
1467 /*
1468 * if Caching mode is set, then invalid translations are tagged
1469 * with domainid 0. Hence we need to pre-allocate it.
1470 */
1471 if (cap_caching_mode(iommu->cap))
1472 set_bit(0, iommu->domain_ids);
1473 return 0;
1474}
ba395927 1475
a868e6b7 1476static void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1477{
1478 struct dmar_domain *domain;
2a46ddf7 1479 int i;
ba395927 1480
94a91b50 1481 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1482 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
a4eaa86c
JL		 1483 /*
1484 * Domain id 0 is reserved for invalid translation
1485 * if hardware supports caching mode.
1486 */
1487 if (cap_caching_mode(iommu->cap) && i == 0)
1488 continue;
1489
94a91b50
DD		 1490 domain = iommu->domains[i];
1491 clear_bit(i, iommu->domain_ids);
129ad281
JL		 1492 if (domain_detach_iommu(domain, iommu) == 0 &&
1493 !domain_type_is_vm(domain))
92d03cc8 1494 domain_exit(domain);
5e98c4b1 1495 }
ba395927
KA		 1496 }
1497
1498 if (iommu->gcmd & DMA_GCMD_TE)
1499 iommu_disable_translation(iommu);
1500
ba395927
KA		 1501 kfree(iommu->domains);
1502 kfree(iommu->domain_ids);
a868e6b7
JL		 1503 iommu->domains = NULL;
1504 iommu->domain_ids = NULL;
ba395927 1505
d9630fe9
WH		 1506 g_iommus[iommu->seq_id] = NULL;
1507
ba395927
KA		 1508 /* free context mapping */
1509 free_context_table(iommu);
ba395927
KA		 1510}
1511
ab8dfe25 1512static struct dmar_domain *alloc_domain(int flags)
ba395927 1513{
92d03cc8
JL		 1514 /* domain id for virtual machine, it won't be set in context */
1515 static atomic_t vm_domid = ATOMIC_INIT(0);
ba395927 1516 struct dmar_domain *domain;
ba395927
KA		 1517
1518 domain = alloc_domain_mem();
1519 if (!domain)
1520 return NULL;
1521
ab8dfe25 1522 memset(domain, 0, sizeof(*domain));
4c923d47 1523 domain->nid = -1;
ab8dfe25 1524 domain->flags = flags;
92d03cc8
JL		 1525 spin_lock_init(&domain->iommu_lock);
1526 INIT_LIST_HEAD(&domain->devices);
ab8dfe25 1527 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
92d03cc8 1528 domain->id = atomic_inc_return(&vm_domid);
2c2e2c38
FY		 1529
1530 return domain;
1531}
1532
fb170fb4
JL		 1533static int __iommu_attach_domain(struct dmar_domain *domain,
1534 struct intel_iommu *iommu)
2c2e2c38
FY		 1535{
1536 int num;
1537 unsigned long ndomains;
2c2e2c38 1538
ba395927 1539 ndomains = cap_ndoms(iommu->cap);
ba395927 1540 num = find_first_zero_bit(iommu->domain_ids, ndomains);
fb170fb4
JL		 1541 if (num < ndomains) {
1542 set_bit(num, iommu->domain_ids);
1543 iommu->domains[num] = domain;
1544 } else {
1545 num = -ENOSPC;
ba395927
KA		 1546 }
1547
fb170fb4
JL		 1548 return num;
1549}
1550
1551static int iommu_attach_domain(struct dmar_domain *domain,
1552 struct intel_iommu *iommu)
1553{
1554 int num;
1555 unsigned long flags;
1556
1557 spin_lock_irqsave(&iommu->lock, flags);
1558 num = __iommu_attach_domain(domain, iommu);
44bde614 1559 spin_unlock_irqrestore(&iommu->lock, flags);
fb170fb4
JL		 1560 if (num < 0)
1561 pr_err("IOMMU: no free domain ids\n");
ba395927 1562
fb170fb4 1563 return num;
ba395927
KA		 1564}
1565
44bde614
JL		 1566static int iommu_attach_vm_domain(struct dmar_domain *domain,
1567 struct intel_iommu *iommu)
1568{
1569 int num;
1570 unsigned long ndomains;
1571
1572 ndomains = cap_ndoms(iommu->cap);
1573 for_each_set_bit(num, iommu->domain_ids, ndomains)
1574 if (iommu->domains[num] == domain)
1575 return num;
1576
1577 return __iommu_attach_domain(domain, iommu);
1578}
1579
2c2e2c38
FY		 1580static void iommu_detach_domain(struct dmar_domain *domain,
1581 struct intel_iommu *iommu)
ba395927
KA		 1582{
1583 unsigned long flags;
2c2e2c38 1584 int num, ndomains;
ba395927 1585
8c11e798 1586 spin_lock_irqsave(&iommu->lock, flags);
fb170fb4
JL		 1587 if (domain_type_is_vm_or_si(domain)) {
1588 ndomains = cap_ndoms(iommu->cap);
1589 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1590 if (iommu->domains[num] == domain) {
1591 clear_bit(num, iommu->domain_ids);
1592 iommu->domains[num] = NULL;
1593 break;
1594 }
2c2e2c38 1595 }
fb170fb4
JL		 1596 } else {
1597 clear_bit(domain->id, iommu->domain_ids);
1598 iommu->domains[domain->id] = NULL;
2c2e2c38 1599 }
8c11e798 1600 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1601}
1602
fb170fb4
JL		 1603static void domain_attach_iommu(struct dmar_domain *domain,
1604 struct intel_iommu *iommu)
1605{
1606 unsigned long flags;
1607
1608 spin_lock_irqsave(&domain->iommu_lock, flags);
1609 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1610 domain->iommu_count++;
1611 if (domain->iommu_count == 1)
1612 domain->nid = iommu->node;
1613 domain_update_iommu_cap(domain);
1614 }
1615 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1616}
1617
1618static int domain_detach_iommu(struct dmar_domain *domain,
1619 struct intel_iommu *iommu)
1620{
1621 unsigned long flags;
1622 int count = INT_MAX;
1623
1624 spin_lock_irqsave(&domain->iommu_lock, flags);
1625 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1626 count = --domain->iommu_count;
1627 domain_update_iommu_cap(domain);
1628 }
1629 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1630
1631 return count;
1632}
1633
ba395927 1634static struct iova_domain reserved_iova_list;
8a443df4 1635static struct lock_class_key reserved_rbtree_key;
ba395927 1636
51a63e67 1637static int dmar_init_reserved_ranges(void)
ba395927
KA		 1638{
1639 struct pci_dev *pdev = NULL;
1640 struct iova *iova;
1641 int i;
ba395927 1642
f661197e 1643 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1644
8a443df4
MG		 1645 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1646 &reserved_rbtree_key);
1647
ba395927
KA		 1648 /* IOAPIC ranges shouldn't be accessed by DMA */
1649 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1650 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1651 if (!iova) {
ba395927 1652 printk(KERN_ERR "Reserve IOAPIC range failed\n");
51a63e67
JC		 1653 return -ENODEV;
1654 }
ba395927
KA		 1655
1656 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1657 for_each_pci_dev(pdev) {
1658 struct resource *r;
1659
1660 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1661 r = &pdev->resource[i];
1662 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1663 continue;
1a4a4551
DW		 1664 iova = reserve_iova(&reserved_iova_list,
1665 IOVA_PFN(r->start),
1666 IOVA_PFN(r->end));
51a63e67 1667 if (!iova) {
ba395927 1668 printk(KERN_ERR "Reserve iova failed\n");
51a63e67
JC		 1669 return -ENODEV;
1670 }
ba395927
KA		 1671 }
1672 }
51a63e67 1673 return 0;
ba395927
KA		 1674}
1675
1676static void domain_reserve_special_ranges(struct dmar_domain *domain)
1677{
1678 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1679}
1680
1681static inline int guestwidth_to_adjustwidth(int gaw)
1682{
1683 int agaw;
1684 int r = (gaw - 12) % 9;
1685
1686 if (r == 0)
1687 agaw = gaw;
1688 else
1689 agaw = gaw + 9 - r;
1690 if (agaw > 64)
1691 agaw = 64;
1692 return agaw;
1693}
1694
1695static int domain_init(struct dmar_domain *domain, int guest_width)
1696{
1697 struct intel_iommu *iommu;
1698 int adjust_width, agaw;
1699 unsigned long sagaw;
1700
f661197e 1701 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
ba395927
KA		 1702 domain_reserve_special_ranges(domain);
1703
1704 /* calculate AGAW */
8c11e798 1705 iommu = domain_get_iommu(domain);
ba395927
KA		 1706 if (guest_width > cap_mgaw(iommu->cap))
1707 guest_width = cap_mgaw(iommu->cap);
1708 domain->gaw = guest_width;
1709 adjust_width = guestwidth_to_adjustwidth(guest_width);
1710 agaw = width_to_agaw(adjust_width);
1711 sagaw = cap_sagaw(iommu->cap);
1712 if (!test_bit(agaw, &sagaw)) {
1713 /* hardware doesn't support it, choose a bigger one */
1714 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1715 agaw = find_next_bit(&sagaw, 5, agaw);
1716 if (agaw >= 5)
1717 return -ENODEV;
1718 }
1719 domain->agaw = agaw;
ba395927 1720
8e604097
WH		 1721 if (ecap_coherent(iommu->ecap))
1722 domain->iommu_coherency = 1;
1723 else
1724 domain->iommu_coherency = 0;
1725
58c610bd
SY		 1726 if (ecap_sc_support(iommu->ecap))
1727 domain->iommu_snooping = 1;
1728 else
1729 domain->iommu_snooping = 0;
1730
214e39aa
DW		 1731 if (intel_iommu_superpage)
1732 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1733 else
1734 domain->iommu_superpage = 0;
1735
4c923d47 1736 domain->nid = iommu->node;
c7151a8d 1737
ba395927 1738 /* always allocate the top pgd */
4c923d47 1739 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1740 if (!domain->pgd)
1741 return -ENOMEM;
5b6985ce 1742 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1743 return 0;
1744}
1745
1746static void domain_exit(struct dmar_domain *domain)
1747{
2c2e2c38
FY		 1748 struct dmar_drhd_unit *drhd;
1749 struct intel_iommu *iommu;
ea8ea460 1750 struct page *freelist = NULL;
ba395927
KA		 1751
1752 /* Domain 0 is reserved, so dont process it */
1753 if (!domain)
1754 return;
1755
7b668357
AW		 1756 /* Flush any lazy unmaps that may reference this domain */
1757 if (!intel_iommu_strict)
1758 flush_unmaps_timeout(0);
1759
92d03cc8 1760 /* remove associated devices */
ba395927 1761 domain_remove_dev_info(domain);
92d03cc8 1762
ba395927
KA		 1763 /* destroy iovas */
1764 put_iova_domain(&domain->iovad);
ba395927 1765
ea8ea460 1766 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1767
92d03cc8 1768 /* clear attached or cached domains */
0e242612 1769 rcu_read_lock();
2c2e2c38 1770 for_each_active_iommu(iommu, drhd)
fb170fb4 1771 iommu_detach_domain(domain, iommu);
0e242612 1772 rcu_read_unlock();
2c2e2c38 1773
ea8ea460
DW		 1774 dma_free_pagelist(freelist);
1775
ba395927
KA		 1776 free_domain_mem(domain);
1777}
1778
64ae892b
DW		 1779static int domain_context_mapping_one(struct dmar_domain *domain,
1780 struct intel_iommu *iommu,
1781 u8 bus, u8 devfn, int translation)
ba395927
KA		 1782{
1783 struct context_entry *context;
ba395927 1784 unsigned long flags;
ea6606b0 1785 struct dma_pte *pgd;
ea6606b0
WH		 1786 int id;
1787 int agaw;
93a23a72 1788 struct device_domain_info *info = NULL;
ba395927
KA		 1789
1790 pr_debug("Set context mapping for %02x:%02x.%d\n",
1791 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1792
ba395927 1793 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1794 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1795 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1796
ba395927
KA		 1797 context = device_to_context_entry(iommu, bus, devfn);
1798 if (!context)
1799 return -ENOMEM;
1800 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1801 if (context_present(context)) {
ba395927
KA		 1802 spin_unlock_irqrestore(&iommu->lock, flags);
1803 return 0;
1804 }
1805
ea6606b0
WH		 1806 id = domain->id;
1807 pgd = domain->pgd;
1808
ab8dfe25 1809 if (domain_type_is_vm_or_si(domain)) {
44bde614
JL		 1810 if (domain_type_is_vm(domain)) {
1811 id = iommu_attach_vm_domain(domain, iommu);
fb170fb4 1812 if (id < 0) {
ea6606b0 1813 spin_unlock_irqrestore(&iommu->lock, flags);
fb170fb4 1814 pr_err("IOMMU: no free domain ids\n");
ea6606b0
WH		 1815 return -EFAULT;
1816 }
ea6606b0
WH		 1817 }
1818
1819 /* Skip top levels of page tables for
1820 * iommu which has less agaw than default.
1672af11 1821 * Unnecessary for PT mode.
ea6606b0 1822 */
1672af11
CW		 1823 if (translation != CONTEXT_TT_PASS_THROUGH) {
1824 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1825 pgd = phys_to_virt(dma_pte_addr(pgd));
1826 if (!dma_pte_present(pgd)) {
1827 spin_unlock_irqrestore(&iommu->lock, flags);
1828 return -ENOMEM;
1829 }
ea6606b0
WH		 1830 }
1831 }
1832 }
1833
1834 context_set_domain_id(context, id);
4ed0d3e6 1835
93a23a72 1836 if (translation != CONTEXT_TT_PASS_THROUGH) {
64ae892b 1837 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
93a23a72
YZ		 1838 translation = info ? CONTEXT_TT_DEV_IOTLB :
1839 CONTEXT_TT_MULTI_LEVEL;
1840 }
4ed0d3e6
FY		 1841 /*
1842 * In pass through mode, AW must be programmed to indicate the largest
1843 * AGAW value supported by hardware. And ASR is ignored by hardware.
1844 */
93a23a72 1845 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1846 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1847 else {
1848 context_set_address_root(context, virt_to_phys(pgd));
1849 context_set_address_width(context, iommu->agaw);
1850 }
4ed0d3e6
FY		 1851
1852 context_set_translation_type(context, translation);
c07e7d21
MM		 1853 context_set_fault_enable(context);
1854 context_set_present(context);
5331fe6f 1855 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1856
4c25a2c1
DW		 1857 /*
1858 * It's a non-present to present mapping. If hardware doesn't cache
1859 * non-present entry we only need to flush the write-buffer. If the
1860 * _does_ cache non-present entries, then it does so in the special
1861 * domain #0, which we have to flush:
1862 */
1863 if (cap_caching_mode(iommu->cap)) {
1864 iommu->flush.flush_context(iommu, 0,
1865 (((u16)bus) << 8) | devfn,
1866 DMA_CCMD_MASK_NOBIT,
1867 DMA_CCMD_DEVICE_INVL);
18fd779a 1868 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1869 } else {
ba395927 1870 iommu_flush_write_buffer(iommu);
4c25a2c1 1871 }
93a23a72 1872 iommu_enable_dev_iotlb(info);
ba395927 1873 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d 1874
fb170fb4
JL		 1875 domain_attach_iommu(domain, iommu);
1876
ba395927
KA		 1877 return 0;
1878}
1879
579305f7
AW		 1880struct domain_context_mapping_data {
1881 struct dmar_domain *domain;
1882 struct intel_iommu *iommu;
1883 int translation;
1884};
1885
1886static int domain_context_mapping_cb(struct pci_dev *pdev,
1887 u16 alias, void *opaque)
1888{
1889 struct domain_context_mapping_data *data = opaque;
1890
1891 return domain_context_mapping_one(data->domain, data->iommu,
1892 PCI_BUS_NUM(alias), alias & 0xff,
1893 data->translation);
1894}
1895
ba395927 1896static int
e1f167f3
DW		 1897domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1898 int translation)
ba395927 1899{
64ae892b 1900 struct intel_iommu *iommu;
156baca8 1901 u8 bus, devfn;
579305f7 1902 struct domain_context_mapping_data data;
64ae892b 1903
e1f167f3 1904 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 1905 if (!iommu)
1906 return -ENODEV;
ba395927 1907
579305f7
AW		 1908 if (!dev_is_pci(dev))
1909 return domain_context_mapping_one(domain, iommu, bus, devfn,
4ed0d3e6 1910 translation);
579305f7
AW		 1911
1912 data.domain = domain;
1913 data.iommu = iommu;
1914 data.translation = translation;
1915
1916 return pci_for_each_dma_alias(to_pci_dev(dev),
1917 &domain_context_mapping_cb, &data);
1918}
1919
1920static int domain_context_mapped_cb(struct pci_dev *pdev,
1921 u16 alias, void *opaque)
1922{
1923 struct intel_iommu *iommu = opaque;
1924
1925 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 1926}
1927
e1f167f3 1928static int domain_context_mapped(struct device *dev)
ba395927 1929{
5331fe6f 1930 struct intel_iommu *iommu;
156baca8 1931 u8 bus, devfn;
5331fe6f 1932
e1f167f3 1933 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 1934 if (!iommu)
1935 return -ENODEV;
ba395927 1936
579305f7
AW		 1937 if (!dev_is_pci(dev))
1938 return device_context_mapped(iommu, bus, devfn);
e1f167f3 1939
579305f7
AW		 1940 return !pci_for_each_dma_alias(to_pci_dev(dev),
1941 domain_context_mapped_cb, iommu);
ba395927
KA		 1942}
1943
f532959b
FY		 1944/* Returns a number of VTD pages, but aligned to MM page size */
1945static inline unsigned long aligned_nrpages(unsigned long host_addr,
1946 size_t size)
1947{
1948 host_addr &= ~PAGE_MASK;
1949 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1950}
1951
6dd9a7c7
YS		 1952/* Return largest possible superpage level for a given mapping */
1953static inline int hardware_largepage_caps(struct dmar_domain *domain,
1954 unsigned long iov_pfn,
1955 unsigned long phy_pfn,
1956 unsigned long pages)
1957{
1958 int support, level = 1;
1959 unsigned long pfnmerge;
1960
1961 support = domain->iommu_superpage;
1962
1963 /* To use a large page, the virtual *and* physical addresses
1964 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1965 of them will mean we have to use smaller pages. So just
1966 merge them and check both at once. */
1967 pfnmerge = iov_pfn | phy_pfn;
1968
1969 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1970 pages >>= VTD_STRIDE_SHIFT;
1971 if (!pages)
1972 break;
1973 pfnmerge >>= VTD_STRIDE_SHIFT;
1974 level++;
1975 support--;
1976 }
1977 return level;
1978}
1979
9051aa02
DW		 1980static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1981 struct scatterlist *sg, unsigned long phys_pfn,
1982 unsigned long nr_pages, int prot)
e1605495
DW		 1983{
1984 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1985 phys_addr_t uninitialized_var(pteval);
9051aa02 1986 unsigned long sg_res;
6dd9a7c7
YS		 1987 unsigned int largepage_lvl = 0;
1988 unsigned long lvl_pages = 0;
e1605495 1989
162d1b10 1990 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
e1605495
DW		 1991
1992 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1993 return -EINVAL;
1994
1995 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1996
9051aa02
DW		 1997 if (sg)
1998 sg_res = 0;
1999 else {
2000 sg_res = nr_pages + 1;
2001 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
2002 }
2003
6dd9a7c7 2004 while (nr_pages > 0) {
c85994e4
DW		 2005 uint64_t tmp;
2006
e1605495 2007 if (!sg_res) {
f532959b 2008 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 2009 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2010 sg->dma_length = sg->length;
2011 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 2012 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2013 }
6dd9a7c7 2014
e1605495 2015 if (!pte) {
6dd9a7c7
YS		 2016 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2017
5cf0a76f 2018 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2019 if (!pte)
2020 return -ENOMEM;
6dd9a7c7 2021 /* It is large page*/
6491d4d0 2022 if (largepage_lvl > 1) {
6dd9a7c7 2023 pteval |= DMA_PTE_LARGE_PAGE;
d41a4adb
JL		 2024 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2025 /*
2026 * Ensure that old small page tables are
2027 * removed to make room for superpage,
2028 * if they exist.
2029 */
6491d4d0 2030 dma_pte_free_pagetable(domain, iov_pfn,
d41a4adb 2031 iov_pfn + lvl_pages - 1);
6491d4d0 2032 } else {
6dd9a7c7 2033 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2034 }
6dd9a7c7 2035
e1605495
DW		 2036 }
2037 /* We don't need lock here, nobody else
2038 * touches the iova range
2039 */
7766a3fb 2040 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2041 if (tmp) {
1bf20f0d 2042 static int dumps = 5;
c85994e4
DW		 2043 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2044 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2045 if (dumps) {
2046 dumps--;
2047 debug_dma_dump_mappings(NULL);
2048 }
2049 WARN_ON(1);
2050 }
6dd9a7c7
YS		 2051
2052 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2053
2054 BUG_ON(nr_pages < lvl_pages);
2055 BUG_ON(sg_res < lvl_pages);
2056
2057 nr_pages -= lvl_pages;
2058 iov_pfn += lvl_pages;
2059 phys_pfn += lvl_pages;
2060 pteval += lvl_pages * VTD_PAGE_SIZE;
2061 sg_res -= lvl_pages;
2062
2063 /* If the next PTE would be the first in a new page, then we
2064 need to flush the cache on the entries we've just written.
2065 And then we'll need to recalculate 'pte', so clear it and
2066 let it get set again in the if (!pte) block above.
2067
2068 If we're done (!nr_pages) we need to flush the cache too.
2069
2070 Also if we've been setting superpages, we may need to
2071 recalculate 'pte' and switch back to smaller pages for the
2072 end of the mapping, if the trailing size is not enough to
2073 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2074 pte++;
6dd9a7c7
YS		 2075 if (!nr_pages || first_pte_in_page(pte) ||
2076 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2077 domain_flush_cache(domain, first_pte,
2078 (void *)pte - (void *)first_pte);
2079 pte = NULL;
2080 }
6dd9a7c7
YS		 2081
2082 if (!sg_res && nr_pages)
e1605495
DW		 2083 sg = sg_next(sg);
2084 }
2085 return 0;
2086}
2087
9051aa02
DW		 2088static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2089 struct scatterlist *sg, unsigned long nr_pages,
2090 int prot)
ba395927 2091{
9051aa02
DW		 2092 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2093}
6f6a00e4 2094
9051aa02
DW		 2095static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2096 unsigned long phys_pfn, unsigned long nr_pages,
2097 int prot)
2098{
2099 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2100}
2101
c7151a8d 2102static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2103{
c7151a8d
WH		 2104 if (!iommu)
2105 return;
8c11e798
WH		 2106
2107 clear_context_table(iommu, bus, devfn);
2108 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2109 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2110 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2111}
2112
109b9b04
DW		 2113static inline void unlink_domain_info(struct device_domain_info *info)
2114{
2115 assert_spin_locked(&device_domain_lock);
2116 list_del(&info->link);
2117 list_del(&info->global);
2118 if (info->dev)
0bcb3e28 2119 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2120}
2121
ba395927
KA		 2122static void domain_remove_dev_info(struct dmar_domain *domain)
2123{
3a74ca01 2124 struct device_domain_info *info, *tmp;
fb170fb4 2125 unsigned long flags;
ba395927
KA		 2126
2127 spin_lock_irqsave(&device_domain_lock, flags);
3a74ca01 2128 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
109b9b04 2129 unlink_domain_info(info);
ba395927
KA		 2130 spin_unlock_irqrestore(&device_domain_lock, flags);
2131
93a23a72 2132 iommu_disable_dev_iotlb(info);
7c7faa11 2133 iommu_detach_dev(info->iommu, info->bus, info->devfn);
ba395927 2134
ab8dfe25 2135 if (domain_type_is_vm(domain)) {
7c7faa11 2136 iommu_detach_dependent_devices(info->iommu, info->dev);
fb170fb4 2137 domain_detach_iommu(domain, info->iommu);
92d03cc8
JL		 2138 }
2139
2140 free_devinfo_mem(info);
ba395927
KA		 2141 spin_lock_irqsave(&device_domain_lock, flags);
2142 }
2143 spin_unlock_irqrestore(&device_domain_lock, flags);
2144}
2145
2146/*
2147 * find_domain
1525a29a 2148 * Note: we use struct device->archdata.iommu stores the info
ba395927 2149 */
1525a29a 2150static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2151{
2152 struct device_domain_info *info;
2153
2154 /* No lock here, assumes no domain exit in normal case */
1525a29a 2155 info = dev->archdata.iommu;
ba395927
KA		 2156 if (info)
2157 return info->domain;
2158 return NULL;
2159}
2160
5a8f40e8 2161static inline struct device_domain_info *
745f2586
JL		 2162dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2163{
2164 struct device_domain_info *info;
2165
2166 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2167 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2168 info->devfn == devfn)
5a8f40e8 2169 return info;
745f2586
JL		 2170
2171 return NULL;
2172}
2173
5a8f40e8 2174static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
41e80dca 2175 int bus, int devfn,
b718cd3d
DW		 2176 struct device *dev,
2177 struct dmar_domain *domain)
745f2586 2178{
5a8f40e8 2179 struct dmar_domain *found = NULL;
745f2586
JL		 2180 struct device_domain_info *info;
2181 unsigned long flags;
2182
2183 info = alloc_devinfo_mem();
2184 if (!info)
b718cd3d 2185 return NULL;
745f2586 2186
745f2586
JL		 2187 info->bus = bus;
2188 info->devfn = devfn;
2189 info->dev = dev;
2190 info->domain = domain;
5a8f40e8 2191 info->iommu = iommu;
745f2586
JL		 2192
2193 spin_lock_irqsave(&device_domain_lock, flags);
2194 if (dev)
0bcb3e28 2195 found = find_domain(dev);
5a8f40e8
DW		 2196 else {
2197 struct device_domain_info *info2;
41e80dca 2198 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
5a8f40e8
DW		 2199 if (info2)
2200 found = info2->domain;
2201 }
745f2586
JL		 2202 if (found) {
2203 spin_unlock_irqrestore(&device_domain_lock, flags);
2204 free_devinfo_mem(info);
b718cd3d
DW		 2205 /* Caller must free the original domain */
2206 return found;
745f2586
JL		 2207 }
2208
b718cd3d
DW		 2209 list_add(&info->link, &domain->devices);
2210 list_add(&info->global, &device_domain_list);
2211 if (dev)
2212 dev->archdata.iommu = info;
2213 spin_unlock_irqrestore(&device_domain_lock, flags);
2214
2215 return domain;
745f2586
JL		 2216}
2217
579305f7
AW		 2218static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2219{
2220 *(u16 *)opaque = alias;
2221 return 0;
2222}
2223
ba395927 2224/* domain is initialized */
146922ec 2225static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2226{
579305f7
AW		 2227 struct dmar_domain *domain, *tmp;
2228 struct intel_iommu *iommu;
5a8f40e8 2229 struct device_domain_info *info;
579305f7 2230 u16 dma_alias;
ba395927 2231 unsigned long flags;
aa4d066a 2232 u8 bus, devfn;
ba395927 2233
146922ec 2234 domain = find_domain(dev);
ba395927
KA		 2235 if (domain)
2236 return domain;
2237
579305f7
AW		 2238 iommu = device_to_iommu(dev, &bus, &devfn);
2239 if (!iommu)
2240 return NULL;
2241
146922ec
DW		 2242 if (dev_is_pci(dev)) {
2243 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2244
579305f7
AW		 2245 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2246
2247 spin_lock_irqsave(&device_domain_lock, flags);
2248 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2249 PCI_BUS_NUM(dma_alias),
2250 dma_alias & 0xff);
2251 if (info) {
2252 iommu = info->iommu;
2253 domain = info->domain;
5a8f40e8 2254 }
579305f7 2255 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2256
579305f7
AW		 2257 /* DMA alias already has a domain, uses it */
2258 if (info)
2259 goto found_domain;
2260 }
ba395927 2261
146922ec 2262 /* Allocate and initialize new domain for the device */
ab8dfe25 2263 domain = alloc_domain(0);
745f2586 2264 if (!domain)
579305f7 2265 return NULL;
44bde614
JL		 2266 domain->id = iommu_attach_domain(domain, iommu);
2267 if (domain->id < 0) {
2fe9723d 2268 free_domain_mem(domain);
579305f7 2269 return NULL;
2c2e2c38 2270 }
fb170fb4 2271 domain_attach_iommu(domain, iommu);
579305f7
AW		 2272 if (domain_init(domain, gaw)) {
2273 domain_exit(domain);
2274 return NULL;
2c2e2c38 2275 }
ba395927 2276
579305f7
AW		 2277 /* register PCI DMA alias device */
2278 if (dev_is_pci(dev)) {
2279 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2280 dma_alias & 0xff, NULL, domain);
2281
2282 if (!tmp || tmp != domain) {
2283 domain_exit(domain);
2284 domain = tmp;
2285 }
2286
b718cd3d 2287 if (!domain)
579305f7 2288 return NULL;
ba395927
KA		 2289 }
2290
2291found_domain:
579305f7
AW		 2292 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2293
2294 if (!tmp || tmp != domain) {
2295 domain_exit(domain);
2296 domain = tmp;
2297 }
b718cd3d
DW		 2298
2299 return domain;
ba395927
KA		 2300}
2301
2c2e2c38 2302static int iommu_identity_mapping;
e0fc7e0b
DW		 2303#define IDENTMAP_ALL 1
2304#define IDENTMAP_GFX 2
2305#define IDENTMAP_AZALIA 4
2c2e2c38 2306
b213203e
DW		 2307static int iommu_domain_identity_map(struct dmar_domain *domain,
2308 unsigned long long start,
2309 unsigned long long end)
ba395927 2310{
c5395d5c
DW		 2311 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2312 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2313
2314 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2315 dma_to_mm_pfn(last_vpfn))) {
ba395927 2316 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 2317 return -ENOMEM;
ba395927
KA		 2318 }
2319
c5395d5c
DW		 2320 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2321 start, end, domain->id);
ba395927
KA		 2322 /*
2323 * RMRR range might have overlap with physical memory range,
2324 * clear it first
2325 */
c5395d5c 2326 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2327
c5395d5c
DW		 2328 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2329 last_vpfn - first_vpfn + 1,
61df7443 2330 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2331}
2332
0b9d9753 2333static int iommu_prepare_identity_map(struct device *dev,
b213203e
DW		 2334 unsigned long long start,
2335 unsigned long long end)
2336{
2337 struct dmar_domain *domain;
2338 int ret;
2339
0b9d9753 2340 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2341 if (!domain)
2342 return -ENOMEM;
2343
19943b0e
DW		 2344 /* For _hardware_ passthrough, don't bother. But for software
2345 passthrough, we do it anyway -- it may indicate a memory
2346 range which is reserved in E820, so which didn't get set
2347 up to start with in si_domain */
2348 if (domain == si_domain && hw_pass_through) {
2349 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
0b9d9753 2350 dev_name(dev), start, end);
19943b0e
DW		 2351 return 0;
2352 }
2353
2354 printk(KERN_INFO
2355 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
0b9d9753 2356 dev_name(dev), start, end);
2ff729f5 2357
5595b528
DW		 2358 if (end < start) {
2359 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2360 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2361 dmi_get_system_info(DMI_BIOS_VENDOR),
2362 dmi_get_system_info(DMI_BIOS_VERSION),
2363 dmi_get_system_info(DMI_PRODUCT_VERSION));
2364 ret = -EIO;
2365 goto error;
2366 }
2367
2ff729f5
DW		 2368 if (end >> agaw_to_width(domain->agaw)) {
2369 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2370 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2371 agaw_to_width(domain->agaw),
2372 dmi_get_system_info(DMI_BIOS_VENDOR),
2373 dmi_get_system_info(DMI_BIOS_VERSION),
2374 dmi_get_system_info(DMI_PRODUCT_VERSION));
2375 ret = -EIO;
2376 goto error;
2377 }
19943b0e 2378
b213203e 2379 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2380 if (ret)
2381 goto error;
2382
2383 /* context entry init */
0b9d9753 2384 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2385 if (ret)
2386 goto error;
2387
2388 return 0;
2389
2390 error:
ba395927
KA		 2391 domain_exit(domain);
2392 return ret;
ba395927
KA		 2393}
2394
2395static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2396 struct device *dev)
ba395927 2397{
0b9d9753 2398 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2399 return 0;
0b9d9753
DW		 2400 return iommu_prepare_identity_map(dev, rmrr->base_address,
2401 rmrr->end_address);
ba395927
KA		 2402}
2403
d3f13810 2404#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2405static inline void iommu_prepare_isa(void)
2406{
2407 struct pci_dev *pdev;
2408 int ret;
2409
2410 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2411 if (!pdev)
2412 return;
2413
c7ab48d2 2414 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2415 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2416
2417 if (ret)
c7ab48d2
DW		 2418 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2419 "floppy might not work\n");
49a0429e 2420
9b27e82d 2421 pci_dev_put(pdev);
49a0429e
KA		 2422}
2423#else
2424static inline void iommu_prepare_isa(void)
2425{
2426 return;
2427}
d3f13810 2428#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2429
2c2e2c38 2430static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2431
071e1374 2432static int __init si_domain_init(int hw)
2c2e2c38
FY		 2433{
2434 struct dmar_drhd_unit *drhd;
2435 struct intel_iommu *iommu;
c7ab48d2 2436 int nid, ret = 0;
44bde614 2437 bool first = true;
2c2e2c38 2438
ab8dfe25 2439 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2440 if (!si_domain)
2441 return -EFAULT;
2442
2c2e2c38
FY		 2443 for_each_active_iommu(iommu, drhd) {
2444 ret = iommu_attach_domain(si_domain, iommu);
fb170fb4 2445 if (ret < 0) {
2c2e2c38
FY		 2446 domain_exit(si_domain);
2447 return -EFAULT;
44bde614
JL		 2448 } else if (first) {
2449 si_domain->id = ret;
2450 first = false;
2451 } else if (si_domain->id != ret) {
2452 domain_exit(si_domain);
2453 return -EFAULT;
2c2e2c38 2454 }
fb170fb4 2455 domain_attach_iommu(si_domain, iommu);
2c2e2c38
FY		 2456 }
2457
2458 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2459 domain_exit(si_domain);
2460 return -EFAULT;
2461 }
2462
9544c003
JL		 2463 pr_debug("IOMMU: identity mapping domain is domain %d\n",
2464 si_domain->id);
2c2e2c38 2465
19943b0e
DW		 2466 if (hw)
2467 return 0;
2468
c7ab48d2 2469 for_each_online_node(nid) {
5dfe8660
TH		 2470 unsigned long start_pfn, end_pfn;
2471 int i;
2472
2473 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2474 ret = iommu_domain_identity_map(si_domain,
2475 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2476 if (ret)
2477 return ret;
2478 }
c7ab48d2
DW		 2479 }
2480
2c2e2c38
FY		 2481 return 0;
2482}
2483
9b226624 2484static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2485{
2486 struct device_domain_info *info;
2487
2488 if (likely(!iommu_identity_mapping))
2489 return 0;
2490
9b226624 2491 info = dev->archdata.iommu;
cb452a40
MT		 2492 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2493 return (info->domain == si_domain);
2c2e2c38 2494
2c2e2c38
FY		 2495 return 0;
2496}
2497
2498static int domain_add_dev_info(struct dmar_domain *domain,
5913c9bf 2499 struct device *dev, int translation)
2c2e2c38 2500{
0ac72664 2501 struct dmar_domain *ndomain;
5a8f40e8 2502 struct intel_iommu *iommu;
156baca8 2503 u8 bus, devfn;
5fe60f4e 2504 int ret;
2c2e2c38 2505
5913c9bf 2506 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2507 if (!iommu)
2508 return -ENODEV;
2509
5913c9bf 2510 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2511 if (ndomain != domain)
2512 return -EBUSY;
2c2e2c38 2513
5913c9bf 2514 ret = domain_context_mapping(domain, dev, translation);
e2ad23d0 2515 if (ret) {
5913c9bf 2516 domain_remove_one_dev_info(domain, dev);
e2ad23d0
DW		 2517 return ret;
2518 }
2519
2c2e2c38
FY		 2520 return 0;
2521}
2522
0b9d9753 2523static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2524{
2525 struct dmar_rmrr_unit *rmrr;
832bd858 2526 struct device *tmp;
ea2447f7
TM		 2527 int i;
2528
0e242612 2529 rcu_read_lock();
ea2447f7 2530 for_each_rmrr_units(rmrr) {
b683b230
JL		 2531 /*
2532 * Return TRUE if this RMRR contains the device that
2533 * is passed in.
2534 */
2535 for_each_active_dev_scope(rmrr->devices,
2536 rmrr->devices_cnt, i, tmp)
0b9d9753 2537 if (tmp == dev) {
0e242612 2538 rcu_read_unlock();
ea2447f7 2539 return true;
b683b230 2540 }
ea2447f7 2541 }
0e242612 2542 rcu_read_unlock();
ea2447f7
TM		 2543 return false;
2544}
2545
c875d2c1
AW		 2546/*
2547 * There are a couple cases where we need to restrict the functionality of
2548 * devices associated with RMRRs. The first is when evaluating a device for
2549 * identity mapping because problems exist when devices are moved in and out
2550 * of domains and their respective RMRR information is lost. This means that
2551 * a device with associated RMRRs will never be in a "passthrough" domain.
2552 * The second is use of the device through the IOMMU API. This interface
2553 * expects to have full control of the IOVA space for the device. We cannot
2554 * satisfy both the requirement that RMRR access is maintained and have an
2555 * unencumbered IOVA space. We also have no ability to quiesce the device's
2556 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2557 * We therefore prevent devices associated with an RMRR from participating in
2558 * the IOMMU API, which eliminates them from device assignment.
2559 *
2560 * In both cases we assume that PCI USB devices with RMRRs have them largely
2561 * for historical reasons and that the RMRR space is not actively used post
2562 * boot. This exclusion may change if vendors begin to abuse it.
2563 */
2564static bool device_is_rmrr_locked(struct device *dev)
2565{
2566 if (!device_has_rmrr(dev))
2567 return false;
2568
2569 if (dev_is_pci(dev)) {
2570 struct pci_dev *pdev = to_pci_dev(dev);
2571
2572 if ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
2573 return false;
2574 }
2575
2576 return true;
2577}
2578
3bdb2591 2579static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2580{
ea2447f7 2581
3bdb2591
DW		 2582 if (dev_is_pci(dev)) {
2583 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2584
c875d2c1 2585 if (device_is_rmrr_locked(dev))
3bdb2591 2586 return 0;
e0fc7e0b 2587
3bdb2591
DW		 2588 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2589 return 1;
e0fc7e0b 2590
3bdb2591
DW		 2591 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2592 return 1;
6941af28 2593
3bdb2591 2594 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2595 return 0;
3bdb2591
DW		 2596
2597 /*
2598 * We want to start off with all devices in the 1:1 domain, and
2599 * take them out later if we find they can't access all of memory.
2600 *
2601 * However, we can't do this for PCI devices behind bridges,
2602 * because all PCI devices behind the same bridge will end up
2603 * with the same source-id on their transactions.
2604 *
2605 * Practically speaking, we can't change things around for these
2606 * devices at run-time, because we can't be sure there'll be no
2607 * DMA transactions in flight for any of their siblings.
2608 *
2609 * So PCI devices (unless they're on the root bus) as well as
2610 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2611 * the 1:1 domain, just in _case_ one of their siblings turns out
2612 * not to be able to map all of memory.
2613 */
2614 if (!pci_is_pcie(pdev)) {
2615 if (!pci_is_root_bus(pdev->bus))
2616 return 0;
2617 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2618 return 0;
2619 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2620 return 0;
3bdb2591
DW		 2621 } else {
2622 if (device_has_rmrr(dev))
2623 return 0;
2624 }
3dfc813d 2625
3bdb2591 2626 /*
3dfc813d 2627 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2628 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2629 * take them out of the 1:1 domain later.
2630 */
8fcc5372
CW		 2631 if (!startup) {
2632 /*
2633 * If the device's dma_mask is less than the system's memory
2634 * size then this is not a candidate for identity mapping.
2635 */
3bdb2591 2636 u64 dma_mask = *dev->dma_mask;
8fcc5372 2637
3bdb2591
DW		 2638 if (dev->coherent_dma_mask &&
2639 dev->coherent_dma_mask < dma_mask)
2640 dma_mask = dev->coherent_dma_mask;
8fcc5372 2641
3bdb2591 2642 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2643 }
6941af28
DW		 2644
2645 return 1;
2646}
2647
cf04eee8
DW		 2648static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2649{
2650 int ret;
2651
2652 if (!iommu_should_identity_map(dev, 1))
2653 return 0;
2654
2655 ret = domain_add_dev_info(si_domain, dev,
2656 hw ? CONTEXT_TT_PASS_THROUGH :
2657 CONTEXT_TT_MULTI_LEVEL);
2658 if (!ret)
2659 pr_info("IOMMU: %s identity mapping for device %s\n",
2660 hw ? "hardware" : "software", dev_name(dev));
2661 else if (ret == -ENODEV)
2662 /* device not associated with an iommu */
2663 ret = 0;
2664
2665 return ret;
2666}
2667
2668
071e1374 2669static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2670{
2c2e2c38 2671 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2672 struct dmar_drhd_unit *drhd;
2673 struct intel_iommu *iommu;
2674 struct device *dev;
2675 int i;
2676 int ret = 0;
2c2e2c38 2677
19943b0e 2678 ret = si_domain_init(hw);
2c2e2c38
FY		 2679 if (ret)
2680 return -EFAULT;
2681
2c2e2c38 2682 for_each_pci_dev(pdev) {
cf04eee8
DW		 2683 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2684 if (ret)
2685 return ret;
2686 }
2687
2688 for_each_active_iommu(iommu, drhd)
2689 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2690 struct acpi_device_physical_node *pn;
2691 struct acpi_device *adev;
2692
2693 if (dev->bus != &acpi_bus_type)
2694 continue;
2695
2696 adev= to_acpi_device(dev);
2697 mutex_lock(&adev->physical_node_lock);
2698 list_for_each_entry(pn, &adev->physical_node_list, node) {
2699 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2700 if (ret)
2701 break;
eae460b6 2702 }
cf04eee8
DW		 2703 mutex_unlock(&adev->physical_node_lock);
2704 if (ret)
2705 return ret;
62edf5dc 2706 }
2c2e2c38
FY		 2707
2708 return 0;
2709}
2710
b779260b 2711static int __init init_dmars(void)
ba395927
KA		 2712{
2713 struct dmar_drhd_unit *drhd;
2714 struct dmar_rmrr_unit *rmrr;
832bd858 2715 struct device *dev;
ba395927 2716 struct intel_iommu *iommu;
9d783ba0 2717 int i, ret;
2c2e2c38 2718
ba395927
KA		 2719 /*
2720 * for each drhd
2721 * allocate root
2722 * initialize and program root entry to not present
2723 * endfor
2724 */
2725 for_each_drhd_unit(drhd) {
5e0d2a6f 2726 /*
2727 * lock not needed as this is only incremented in the single
2728 * threaded kernel __init code path all other access are read
2729 * only
2730 */
1b198bb0
MT		 2731 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2732 g_num_of_iommus++;
2733 continue;
2734 }
2735 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2736 IOMMU_UNITS_SUPPORTED);
5e0d2a6f 2737 }
2738
d9630fe9
WH		 2739 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2740 GFP_KERNEL);
2741 if (!g_iommus) {
2742 printk(KERN_ERR "Allocating global iommu array failed\n");
2743 ret = -ENOMEM;
2744 goto error;
2745 }
2746
80b20dd8 2747 deferred_flush = kzalloc(g_num_of_iommus *
2748 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2749 if (!deferred_flush) {
5e0d2a6f 2750 ret = -ENOMEM;
989d51fc 2751 goto free_g_iommus;
5e0d2a6f 2752 }
2753
7c919779 2754 for_each_active_iommu(iommu, drhd) {
d9630fe9 2755 g_iommus[iommu->seq_id] = iommu;
ba395927 2756
e61d98d8
SS		 2757 ret = iommu_init_domains(iommu);
2758 if (ret)
989d51fc 2759 goto free_iommu;
e61d98d8 2760
ba395927
KA		 2761 /*
2762 * TBD:
2763 * we could share the same root & context tables
25985edc 2764 * among all IOMMU's. Need to Split it later.
ba395927
KA		 2765 */
2766 ret = iommu_alloc_root_entry(iommu);
2767 if (ret) {
2768 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
989d51fc 2769 goto free_iommu;
ba395927 2770 }
4ed0d3e6 2771 if (!ecap_pass_through(iommu->ecap))
19943b0e 2772 hw_pass_through = 0;
ba395927
KA		 2773 }
2774
1531a6a6
SS		 2775 /*
2776 * Start from the sane iommu hardware state.
2777 */
7c919779 2778 for_each_active_iommu(iommu, drhd) {
1531a6a6
SS		 2779 /*
2780 * If the queued invalidation is already initialized by us
2781 * (for example, while enabling interrupt-remapping) then
2782 * we got the things already rolling from a sane state.
2783 */
2784 if (iommu->qi)
2785 continue;
2786
2787 /*
2788 * Clear any previous faults.
2789 */
2790 dmar_fault(-1, iommu);
2791 /*
2792 * Disable queued invalidation if supported and already enabled
2793 * before OS handover.
2794 */
2795 dmar_disable_qi(iommu);
2796 }
2797
7c919779 2798 for_each_active_iommu(iommu, drhd) {
a77b67d4
YS		 2799 if (dmar_enable_qi(iommu)) {
2800 /*
2801 * Queued Invalidate not enabled, use Register Based
2802 * Invalidate
2803 */
2804 iommu->flush.flush_context = __iommu_flush_context;
2805 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
680a7524 2806 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
b4e0f9eb 2807 "invalidation\n",
680a7524 2808 iommu->seq_id,
b4e0f9eb 2809 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2810 } else {
2811 iommu->flush.flush_context = qi_flush_context;
2812 iommu->flush.flush_iotlb = qi_flush_iotlb;
680a7524 2813 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
b4e0f9eb 2814 "invalidation\n",
680a7524 2815 iommu->seq_id,
b4e0f9eb 2816 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2817 }
2818 }
2819
19943b0e 2820 if (iommu_pass_through)
e0fc7e0b
DW		 2821 iommu_identity_mapping |= IDENTMAP_ALL;
2822
d3f13810 2823#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 2824 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2825#endif
e0fc7e0b
DW		 2826
2827 check_tylersburg_isoch();
2828
ba395927 2829 /*
19943b0e
DW		 2830 * If pass through is not set or not enabled, setup context entries for
2831 * identity mappings for rmrr, gfx, and isa and may fall back to static
2832 * identity mapping if iommu_identity_mapping is set.
ba395927 2833 */
19943b0e
DW		 2834 if (iommu_identity_mapping) {
2835 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2836 if (ret) {
19943b0e 2837 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
989d51fc 2838 goto free_iommu;
ba395927
KA		 2839 }
2840 }
ba395927 2841 /*
19943b0e
DW		 2842 * For each rmrr
2843 * for each dev attached to rmrr
2844 * do
2845 * locate drhd for dev, alloc domain for dev
2846 * allocate free domain
2847 * allocate page table entries for rmrr
2848 * if context not allocated for bus
2849 * allocate and init context
2850 * set present in root table for this bus
2851 * init context with domain, translation etc
2852 * endfor
2853 * endfor
ba395927 2854 */
19943b0e
DW		 2855 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2856 for_each_rmrr_units(rmrr) {
b683b230
JL		 2857 /* some BIOS lists non-exist devices in DMAR table. */
2858 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 2859 i, dev) {
0b9d9753 2860 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e
DW		 2861 if (ret)
2862 printk(KERN_ERR
2863 "IOMMU: mapping reserved region failed\n");
ba395927 2864 }
4ed0d3e6 2865 }
49a0429e 2866
19943b0e
DW		 2867 iommu_prepare_isa();
2868
ba395927
KA		 2869 /*
2870 * for each drhd
2871 * enable fault log
2872 * global invalidate context cache
2873 * global invalidate iotlb
2874 * enable translation
2875 */
7c919779 2876 for_each_iommu(iommu, drhd) {
51a63e67
JC		 2877 if (drhd->ignored) {
2878 /*
2879 * we always have to disable PMRs or DMA may fail on
2880 * this device
2881 */
2882 if (force_on)
7c919779 2883 iommu_disable_protect_mem_regions(iommu);
ba395927 2884 continue;
51a63e67 2885 }
ba395927
KA		 2886
2887 iommu_flush_write_buffer(iommu);
2888
3460a6d9
KA		 2889 ret = dmar_set_interrupt(iommu);
2890 if (ret)
989d51fc 2891 goto free_iommu;
3460a6d9 2892
ba395927
KA		 2893 iommu_set_root_entry(iommu);
2894
4c25a2c1 2895 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2896 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2a41ccee 2897 iommu_enable_translation(iommu);
b94996c9 2898 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2899 }
2900
2901 return 0;
989d51fc
JL		 2902
2903free_iommu:
7c919779 2904 for_each_active_iommu(iommu, drhd)
a868e6b7 2905 free_dmar_iommu(iommu);
9bdc531e 2906 kfree(deferred_flush);
989d51fc 2907free_g_iommus:
d9630fe9 2908 kfree(g_iommus);
989d51fc 2909error:
ba395927
KA		 2910 return ret;
2911}
2912
5a5e02a6 2913/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2914static struct iova *intel_alloc_iova(struct device *dev,
2915 struct dmar_domain *domain,
2916 unsigned long nrpages, uint64_t dma_mask)
ba395927 2917{
ba395927 2918 struct iova *iova = NULL;
ba395927 2919
875764de
DW		 2920 /* Restrict dma_mask to the width that the iommu can handle */
2921 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2922
2923 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2924 /*
2925 * First try to allocate an io virtual address in
284901a9 2926 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2927 * from higher range
ba395927 2928 */
875764de
DW		 2929 iova = alloc_iova(&domain->iovad, nrpages,
2930 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2931 if (iova)
2932 return iova;
2933 }
2934 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2935 if (unlikely(!iova)) {
2936 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
207e3592 2937 nrpages, dev_name(dev));
f76aec76
KA		 2938 return NULL;
2939 }
2940
2941 return iova;
2942}
2943
d4b709f4 2944static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76
KA		 2945{
2946 struct dmar_domain *domain;
2947 int ret;
2948
d4b709f4 2949 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 2950 if (!domain) {
d4b709f4
DW		 2951 printk(KERN_ERR "Allocating domain for %s failed",
2952 dev_name(dev));
4fe05bbc 2953 return NULL;
ba395927
KA		 2954 }
2955
2956 /* make sure context mapping is ok */
d4b709f4
DW		 2957 if (unlikely(!domain_context_mapped(dev))) {
2958 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
f76aec76 2959 if (ret) {
d4b709f4
DW		 2960 printk(KERN_ERR "Domain context map for %s failed",
2961 dev_name(dev));
4fe05bbc 2962 return NULL;
f76aec76 2963 }
ba395927
KA		 2964 }
2965
f76aec76
KA		 2966 return domain;
2967}
2968
d4b709f4 2969static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 2970{
2971 struct device_domain_info *info;
2972
2973 /* No lock here, assumes no domain exit in normal case */
d4b709f4 2974 info = dev->archdata.iommu;
147202aa
DW		 2975 if (likely(info))
2976 return info->domain;
2977
2978 return __get_valid_domain_for_dev(dev);
2979}
2980
3d89194a 2981static int iommu_dummy(struct device *dev)
2c2e2c38 2982{
3d89194a 2983 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2c2e2c38
FY		 2984}
2985
ecb509ec 2986/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 2987static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 2988{
2989 int found;
2990
3d89194a 2991 if (iommu_dummy(dev))
1e4c64c4
DW		 2992 return 1;
2993
2c2e2c38 2994 if (!iommu_identity_mapping)
1e4c64c4 2995 return 0;
2c2e2c38 2996
9b226624 2997 found = identity_mapping(dev);
2c2e2c38 2998 if (found) {
ecb509ec 2999 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 3000 return 1;
3001 else {
3002 /*
3003 * 32 bit DMA is removed from si_domain and fall back
3004 * to non-identity mapping.
3005 */
bf9c9eda 3006 domain_remove_one_dev_info(si_domain, dev);
2c2e2c38 3007 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
ecb509ec 3008 dev_name(dev));
2c2e2c38
FY		 3009 return 0;
3010 }
3011 } else {
3012 /*
3013 * In case of a detached 64 bit DMA device from vm, the device
3014 * is put into si_domain for identity mapping.
3015 */
ecb509ec 3016 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 3017 int ret;
5913c9bf 3018 ret = domain_add_dev_info(si_domain, dev,
5fe60f4e
DW		 3019 hw_pass_through ?
3020 CONTEXT_TT_PASS_THROUGH :
3021 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38
FY		 3022 if (!ret) {
3023 printk(KERN_INFO "64bit %s uses identity mapping\n",
ecb509ec 3024 dev_name(dev));
2c2e2c38
FY		 3025 return 1;
3026 }
3027 }
3028 }
3029
1e4c64c4 3030 return 0;
2c2e2c38
FY		 3031}
3032
5040a918 3033static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3034 size_t size, int dir, u64 dma_mask)
f76aec76 3035{
f76aec76 3036 struct dmar_domain *domain;
5b6985ce 3037 phys_addr_t start_paddr;
f76aec76
KA		 3038 struct iova *iova;
3039 int prot = 0;
6865f0d1 3040 int ret;
8c11e798 3041 struct intel_iommu *iommu;
33041ec0 3042 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3043
3044 BUG_ON(dir == DMA_NONE);
2c2e2c38 3045
5040a918 3046 if (iommu_no_mapping(dev))
6865f0d1 3047 return paddr;
f76aec76 3048
5040a918 3049 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3050 if (!domain)
3051 return 0;
3052
8c11e798 3053 iommu = domain_get_iommu(domain);
88cb6a74 3054 size = aligned_nrpages(paddr, size);
f76aec76 3055
5040a918 3056 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 3057 if (!iova)
3058 goto error;
3059
ba395927
KA		 3060 /*
3061 * Check if DMAR supports zero-length reads on write only
3062 * mappings..
3063 */
3064 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3065 !cap_zlr(iommu->cap))
ba395927
KA		 3066 prot |= DMA_PTE_READ;
3067 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3068 prot |= DMA_PTE_WRITE;
3069 /*
6865f0d1 3070 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3071 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3072 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3073 * is not a big problem
3074 */
0ab36de2 3075 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 3076 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3077 if (ret)
3078 goto error;
3079
1f0ef2aa
DW		 3080 /* it's a non-present to present mapping. Only flush if caching mode */
3081 if (cap_caching_mode(iommu->cap))
ea8ea460 3082 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
1f0ef2aa 3083 else
8c11e798 3084 iommu_flush_write_buffer(iommu);
f76aec76 3085
03d6a246
DW		 3086 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3087 start_paddr += paddr & ~PAGE_MASK;
3088 return start_paddr;
ba395927 3089
ba395927 3090error:
f76aec76
KA		 3091 if (iova)
3092 __free_iova(&domain->iovad, iova);
4cf2e75d 3093 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3094 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3095 return 0;
3096}
3097
ffbbef5c
FT		 3098static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3099 unsigned long offset, size_t size,
3100 enum dma_data_direction dir,
3101 struct dma_attrs *attrs)
bb9e6d65 3102{
ffbbef5c 3103 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3104 dir, *dev->dma_mask);
bb9e6d65
FT		 3105}
3106
5e0d2a6f 3107static void flush_unmaps(void)
3108{
80b20dd8 3109 int i, j;
5e0d2a6f 3110
5e0d2a6f 3111 timer_on = 0;
3112
3113 /* just flush them all */
3114 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3115 struct intel_iommu *iommu = g_iommus[i];
3116 if (!iommu)
3117 continue;
c42d9f32 3118
9dd2fe89
YZ		 3119 if (!deferred_flush[i].next)
3120 continue;
3121
78d5f0f5
NA		 3122 /* In caching mode, global flushes turn emulation expensive */
3123 if (!cap_caching_mode(iommu->cap))
3124 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3125 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3126 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3127 unsigned long mask;
3128 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3129 struct dmar_domain *domain = deferred_flush[i].domain[j];
3130
3131 /* On real hardware multiple invalidations are expensive */
3132 if (cap_caching_mode(iommu->cap))
3133 iommu_flush_iotlb_psi(iommu, domain->id,
a156ef99 3134 iova->pfn_lo, iova_size(iova),
ea8ea460 3135 !deferred_flush[i].freelist[j], 0);
78d5f0f5 3136 else {
a156ef99 3137 mask = ilog2(mm_to_dma_pfn(iova_size(iova)));
78d5f0f5
NA		 3138 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3139 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3140 }
93a23a72 3141 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3142 if (deferred_flush[i].freelist[j])
3143 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3144 }
9dd2fe89 3145 deferred_flush[i].next = 0;
5e0d2a6f 3146 }
3147
5e0d2a6f 3148 list_size = 0;
5e0d2a6f 3149}
3150
3151static void flush_unmaps_timeout(unsigned long data)
3152{
80b20dd8 3153 unsigned long flags;
3154
3155 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3156 flush_unmaps();
80b20dd8 3157 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3158}
3159
ea8ea460 3160static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3161{
3162 unsigned long flags;
80b20dd8 3163 int next, iommu_id;
8c11e798 3164 struct intel_iommu *iommu;
5e0d2a6f 3165
3166 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3167 if (list_size == HIGH_WATER_MARK)
3168 flush_unmaps();
3169
8c11e798
WH		 3170 iommu = domain_get_iommu(dom);
3171 iommu_id = iommu->seq_id;
c42d9f32 3172
80b20dd8 3173 next = deferred_flush[iommu_id].next;
3174 deferred_flush[iommu_id].domain[next] = dom;
3175 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3176 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3177 deferred_flush[iommu_id].next++;
5e0d2a6f 3178
3179 if (!timer_on) {
3180 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3181 timer_on = 1;
3182 }
3183 list_size++;
3184 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3185}
3186
d41a4adb 3187static void intel_unmap(struct device *dev, dma_addr_t dev_addr)
ba395927 3188{
f76aec76 3189 struct dmar_domain *domain;
d794dc9b 3190 unsigned long start_pfn, last_pfn;
ba395927 3191 struct iova *iova;
8c11e798 3192 struct intel_iommu *iommu;
ea8ea460 3193 struct page *freelist;
ba395927 3194
73676832 3195 if (iommu_no_mapping(dev))
f76aec76 3196 return;
2c2e2c38 3197
1525a29a 3198 domain = find_domain(dev);
ba395927
KA		 3199 BUG_ON(!domain);
3200
8c11e798
WH		 3201 iommu = domain_get_iommu(domain);
3202
ba395927 3203 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3204 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3205 (unsigned long long)dev_addr))
ba395927 3206 return;
ba395927 3207
d794dc9b
DW		 3208 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3209 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3210
d794dc9b 3211 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3212 dev_name(dev), start_pfn, last_pfn);
ba395927 3213
ea8ea460 3214 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3215
5e0d2a6f 3216 if (intel_iommu_strict) {
03d6a246 3217 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
ea8ea460 3218 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3219 /* free iova */
3220 __free_iova(&domain->iovad, iova);
ea8ea460 3221 dma_free_pagelist(freelist);
5e0d2a6f 3222 } else {
ea8ea460 3223 add_unmap(domain, iova, freelist);
5e0d2a6f 3224 /*
3225 * queue up the release of the unmap to save the 1/6th of the
3226 * cpu used up by the iotlb flush operation...
3227 */
5e0d2a6f 3228 }
ba395927
KA		 3229}
3230
d41a4adb
JL		 3231static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3232 size_t size, enum dma_data_direction dir,
3233 struct dma_attrs *attrs)
3234{
3235 intel_unmap(dev, dev_addr);
3236}
3237
5040a918 3238static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3239 dma_addr_t *dma_handle, gfp_t flags,
3240 struct dma_attrs *attrs)
ba395927 3241{
36746436 3242 struct page *page = NULL;
ba395927
KA		 3243 int order;
3244
5b6985ce 3245 size = PAGE_ALIGN(size);
ba395927 3246 order = get_order(size);
e8bb910d 3247
5040a918 3248 if (!iommu_no_mapping(dev))
e8bb910d 3249 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3250 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3251 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3252 flags |= GFP_DMA;
3253 else
3254 flags |= GFP_DMA32;
3255 }
ba395927 3256
36746436
AM		 3257 if (flags & __GFP_WAIT) {
3258 unsigned int count = size >> PAGE_SHIFT;
3259
3260 page = dma_alloc_from_contiguous(dev, count, order);
3261 if (page && iommu_no_mapping(dev) &&
3262 page_to_phys(page) + size > dev->coherent_dma_mask) {
3263 dma_release_from_contiguous(dev, page, count);
3264 page = NULL;
3265 }
3266 }
3267
3268 if (!page)
3269 page = alloc_pages(flags, order);
3270 if (!page)
ba395927 3271 return NULL;
36746436 3272 memset(page_address(page), 0, size);
ba395927 3273
36746436 3274 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3275 DMA_BIDIRECTIONAL,
5040a918 3276 dev->coherent_dma_mask);
ba395927 3277 if (*dma_handle)
36746436
AM		 3278 return page_address(page);
3279 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3280 __free_pages(page, order);
3281
ba395927
KA		 3282 return NULL;
3283}
3284
5040a918 3285static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3286 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3287{
3288 int order;
36746436 3289 struct page *page = virt_to_page(vaddr);
ba395927 3290
5b6985ce 3291 size = PAGE_ALIGN(size);
ba395927
KA		 3292 order = get_order(size);
3293
d41a4adb 3294 intel_unmap(dev, dma_handle);
36746436
AM		 3295 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3296 __free_pages(page, order);
ba395927
KA		 3297}
3298
5040a918 3299static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3300 int nelems, enum dma_data_direction dir,
3301 struct dma_attrs *attrs)
ba395927 3302{
d41a4adb 3303 intel_unmap(dev, sglist[0].dma_address);
ba395927
KA		 3304}
3305
ba395927 3306static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3307 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3308{
3309 int i;
c03ab37c 3310 struct scatterlist *sg;
ba395927 3311
c03ab37c 3312 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3313 BUG_ON(!sg_page(sg));
4cf2e75d 3314 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3315 sg->dma_length = sg->length;
ba395927
KA		 3316 }
3317 return nelems;
3318}
3319
5040a918 3320static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3321 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3322{
ba395927 3323 int i;
ba395927 3324 struct dmar_domain *domain;
f76aec76
KA		 3325 size_t size = 0;
3326 int prot = 0;
f76aec76
KA		 3327 struct iova *iova = NULL;
3328 int ret;
c03ab37c 3329 struct scatterlist *sg;
b536d24d 3330 unsigned long start_vpfn;
8c11e798 3331 struct intel_iommu *iommu;
ba395927
KA		 3332
3333 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3334 if (iommu_no_mapping(dev))
3335 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3336
5040a918 3337 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3338 if (!domain)
3339 return 0;
3340
8c11e798
WH		 3341 iommu = domain_get_iommu(domain);
3342
b536d24d 3343 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3344 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3345
5040a918
DW		 3346 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3347 *dev->dma_mask);
f76aec76 3348 if (!iova) {
c03ab37c 3349 sglist->dma_length = 0;
f76aec76
KA		 3350 return 0;
3351 }
3352
3353 /*
3354 * Check if DMAR supports zero-length reads on write only
3355 * mappings..
3356 */
3357 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3358 !cap_zlr(iommu->cap))
f76aec76
KA		 3359 prot |= DMA_PTE_READ;
3360 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3361 prot |= DMA_PTE_WRITE;
3362
b536d24d 3363 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3364
f532959b 3365 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495 3366 if (unlikely(ret)) {
e1605495
DW		 3367 dma_pte_free_pagetable(domain, start_vpfn,
3368 start_vpfn + size - 1);
e1605495
DW		 3369 __free_iova(&domain->iovad, iova);
3370 return 0;
ba395927
KA		 3371 }
3372
1f0ef2aa
DW		 3373 /* it's a non-present to present mapping. Only flush if caching mode */
3374 if (cap_caching_mode(iommu->cap))
ea8ea460 3375 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
1f0ef2aa 3376 else
8c11e798 3377 iommu_flush_write_buffer(iommu);
1f0ef2aa 3378
ba395927
KA		 3379 return nelems;
3380}
3381
dfb805e8
FT		 3382static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3383{
3384 return !dma_addr;
3385}
3386
160c1d8e 3387struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3388 .alloc = intel_alloc_coherent,
3389 .free = intel_free_coherent,
ba395927
KA		 3390 .map_sg = intel_map_sg,
3391 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3392 .map_page = intel_map_page,
3393 .unmap_page = intel_unmap_page,
dfb805e8 3394 .mapping_error = intel_mapping_error,
ba395927
KA		 3395};
3396
3397static inline int iommu_domain_cache_init(void)
3398{
3399 int ret = 0;
3400
3401 iommu_domain_cache = kmem_cache_create("iommu_domain",
3402 sizeof(struct dmar_domain),
3403 0,
3404 SLAB_HWCACHE_ALIGN,
3405
3406 NULL);
3407 if (!iommu_domain_cache) {
3408 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3409 ret = -ENOMEM;
3410 }
3411
3412 return ret;
3413}
3414
3415static inline int iommu_devinfo_cache_init(void)
3416{
3417 int ret = 0;
3418
3419 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3420 sizeof(struct device_domain_info),
3421 0,
3422 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3423 NULL);
3424 if (!iommu_devinfo_cache) {
3425 printk(KERN_ERR "Couldn't create devinfo cache\n");
3426 ret = -ENOMEM;
3427 }
3428
3429 return ret;
3430}
3431
3432static inline int iommu_iova_cache_init(void)
3433{
3434 int ret = 0;
3435
3436 iommu_iova_cache = kmem_cache_create("iommu_iova",
3437 sizeof(struct iova),
3438 0,
3439 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3440 NULL);
3441 if (!iommu_iova_cache) {
3442 printk(KERN_ERR "Couldn't create iova cache\n");
3443 ret = -ENOMEM;
3444 }
3445
3446 return ret;
3447}
3448
3449static int __init iommu_init_mempool(void)
3450{
3451 int ret;
3452 ret = iommu_iova_cache_init();
3453 if (ret)
3454 return ret;
3455
3456 ret = iommu_domain_cache_init();
3457 if (ret)
3458 goto domain_error;
3459
3460 ret = iommu_devinfo_cache_init();
3461 if (!ret)
3462 return ret;
3463
3464 kmem_cache_destroy(iommu_domain_cache);
3465domain_error:
3466 kmem_cache_destroy(iommu_iova_cache);
3467
3468 return -ENOMEM;
3469}
3470
3471static void __init iommu_exit_mempool(void)
3472{
3473 kmem_cache_destroy(iommu_devinfo_cache);
3474 kmem_cache_destroy(iommu_domain_cache);
3475 kmem_cache_destroy(iommu_iova_cache);
3476
3477}
3478
556ab45f
DW		 3479static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3480{
3481 struct dmar_drhd_unit *drhd;
3482 u32 vtbar;
3483 int rc;
3484
3485 /* We know that this device on this chipset has its own IOMMU.
3486 * If we find it under a different IOMMU, then the BIOS is lying
3487 * to us. Hope that the IOMMU for this device is actually
3488 * disabled, and it needs no translation...
3489 */
3490 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3491 if (rc) {
3492 /* "can't" happen */
3493 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3494 return;
3495 }
3496 vtbar &= 0xffff0000;
3497
3498 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3499 drhd = dmar_find_matched_drhd_unit(pdev);
3500 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3501 TAINT_FIRMWARE_WORKAROUND,
3502 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3503 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3504}
3505DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3506
ba395927
KA		 3507static void __init init_no_remapping_devices(void)
3508{
3509 struct dmar_drhd_unit *drhd;
832bd858 3510 struct device *dev;
b683b230 3511 int i;
ba395927
KA		 3512
3513 for_each_drhd_unit(drhd) {
3514 if (!drhd->include_all) {
b683b230
JL		 3515 for_each_active_dev_scope(drhd->devices,
3516 drhd->devices_cnt, i, dev)
3517 break;
832bd858 3518 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3519 if (i == drhd->devices_cnt)
3520 drhd->ignored = 1;
3521 }
3522 }
3523
7c919779 3524 for_each_active_drhd_unit(drhd) {
7c919779 3525 if (drhd->include_all)
ba395927
KA		 3526 continue;
3527
b683b230
JL		 3528 for_each_active_dev_scope(drhd->devices,
3529 drhd->devices_cnt, i, dev)
832bd858 3530 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 3531 break;
ba395927
KA		 3532 if (i < drhd->devices_cnt)
3533 continue;
3534
c0771df8
DW		 3535 /* This IOMMU has *only* gfx devices. Either bypass it or
3536 set the gfx_mapped flag, as appropriate */
3537 if (dmar_map_gfx) {
3538 intel_iommu_gfx_mapped = 1;
3539 } else {
3540 drhd->ignored = 1;
b683b230
JL		 3541 for_each_active_dev_scope(drhd->devices,
3542 drhd->devices_cnt, i, dev)
832bd858 3543 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3544 }
3545 }
3546}
3547
f59c7b69
FY		 3548#ifdef CONFIG_SUSPEND
3549static int init_iommu_hw(void)
3550{
3551 struct dmar_drhd_unit *drhd;
3552 struct intel_iommu *iommu = NULL;
3553
3554 for_each_active_iommu(iommu, drhd)
3555 if (iommu->qi)
3556 dmar_reenable_qi(iommu);
3557
b779260b
JC		 3558 for_each_iommu(iommu, drhd) {
3559 if (drhd->ignored) {
3560 /*
3561 * we always have to disable PMRs or DMA may fail on
3562 * this device
3563 */
3564 if (force_on)
3565 iommu_disable_protect_mem_regions(iommu);
3566 continue;
3567 }
3568
f59c7b69
FY		 3569 iommu_flush_write_buffer(iommu);
3570
3571 iommu_set_root_entry(iommu);
3572
3573 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3574 DMA_CCMD_GLOBAL_INVL);
2a41ccee
JL		 3575 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
3576 iommu_enable_translation(iommu);
b94996c9 3577 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3578 }
3579
3580 return 0;
3581}
3582
3583static void iommu_flush_all(void)
3584{
3585 struct dmar_drhd_unit *drhd;
3586 struct intel_iommu *iommu;
3587
3588 for_each_active_iommu(iommu, drhd) {
3589 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3590 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3591 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3592 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3593 }
3594}
3595
134fac3f 3596static int iommu_suspend(void)
f59c7b69
FY		 3597{
3598 struct dmar_drhd_unit *drhd;
3599 struct intel_iommu *iommu = NULL;
3600 unsigned long flag;
3601
3602 for_each_active_iommu(iommu, drhd) {
3603 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3604 GFP_ATOMIC);
3605 if (!iommu->iommu_state)
3606 goto nomem;
3607 }
3608
3609 iommu_flush_all();
3610
3611 for_each_active_iommu(iommu, drhd) {
3612 iommu_disable_translation(iommu);
3613
1f5b3c3f 3614 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3615
3616 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3617 readl(iommu->reg + DMAR_FECTL_REG);
3618 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3619 readl(iommu->reg + DMAR_FEDATA_REG);
3620 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3621 readl(iommu->reg + DMAR_FEADDR_REG);
3622 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3623 readl(iommu->reg + DMAR_FEUADDR_REG);
3624
1f5b3c3f 3625 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3626 }
3627 return 0;
3628
3629nomem:
3630 for_each_active_iommu(iommu, drhd)
3631 kfree(iommu->iommu_state);
3632
3633 return -ENOMEM;
3634}
3635
134fac3f 3636static void iommu_resume(void)
f59c7b69
FY		 3637{
3638 struct dmar_drhd_unit *drhd;
3639 struct intel_iommu *iommu = NULL;
3640 unsigned long flag;
3641
3642 if (init_iommu_hw()) {
b779260b
JC		 3643 if (force_on)
3644 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3645 else
3646 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3647 return;
f59c7b69
FY		 3648 }
3649
3650 for_each_active_iommu(iommu, drhd) {
3651
1f5b3c3f 3652 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3653
3654 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3655 iommu->reg + DMAR_FECTL_REG);
3656 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3657 iommu->reg + DMAR_FEDATA_REG);
3658 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3659 iommu->reg + DMAR_FEADDR_REG);
3660 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3661 iommu->reg + DMAR_FEUADDR_REG);
3662
1f5b3c3f 3663 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3664 }
3665
3666 for_each_active_iommu(iommu, drhd)
3667 kfree(iommu->iommu_state);
f59c7b69
FY		 3668}
3669
134fac3f 3670static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3671 .resume = iommu_resume,
3672 .suspend = iommu_suspend,
3673};
3674
134fac3f 3675static void __init init_iommu_pm_ops(void)
f59c7b69 3676{
134fac3f 3677 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3678}
3679
3680#else
99592ba4 3681static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3682#endif /* CONFIG_PM */
3683
318fe7df
SS		 3684
3685int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3686{
3687 struct acpi_dmar_reserved_memory *rmrr;
3688 struct dmar_rmrr_unit *rmrru;
3689
3690 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3691 if (!rmrru)
3692 return -ENOMEM;
3693
3694 rmrru->hdr = header;
3695 rmrr = (struct acpi_dmar_reserved_memory *)header;
3696 rmrru->base_address = rmrr->base_address;
3697 rmrru->end_address = rmrr->end_address;
2e455289
JL		 3698 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3699 ((void *)rmrr) + rmrr->header.length,
3700 &rmrru->devices_cnt);
3701 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3702 kfree(rmrru);
3703 return -ENOMEM;
3704 }
318fe7df 3705
2e455289 3706 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 3707
2e455289 3708 return 0;
318fe7df
SS		 3709}
3710
318fe7df
SS		 3711int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3712{
3713 struct acpi_dmar_atsr *atsr;
3714 struct dmar_atsr_unit *atsru;
3715
3716 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3717 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3718 if (!atsru)
3719 return -ENOMEM;
3720
3721 atsru->hdr = hdr;
3722 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 3723 if (!atsru->include_all) {
3724 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3725 (void *)atsr + atsr->header.length,
3726 &atsru->devices_cnt);
3727 if (atsru->devices_cnt && atsru->devices == NULL) {
3728 kfree(atsru);
3729 return -ENOMEM;
3730 }
3731 }
318fe7df 3732
0e242612 3733 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 3734
3735 return 0;
3736}
3737
9bdc531e
JL		 3738static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3739{
3740 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3741 kfree(atsru);
3742}
3743
3744static void intel_iommu_free_dmars(void)
3745{
3746 struct dmar_rmrr_unit *rmrru, *rmrr_n;
3747 struct dmar_atsr_unit *atsru, *atsr_n;
3748
3749 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3750 list_del(&rmrru->list);
3751 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3752 kfree(rmrru);
318fe7df
SS		 3753 }
3754
9bdc531e
JL		 3755 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3756 list_del(&atsru->list);
3757 intel_iommu_free_atsr(atsru);
3758 }
318fe7df
SS		 3759}
3760
3761int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3762{
b683b230 3763 int i, ret = 1;
318fe7df 3764 struct pci_bus *bus;
832bd858
DW		 3765 struct pci_dev *bridge = NULL;
3766 struct device *tmp;
318fe7df
SS		 3767 struct acpi_dmar_atsr *atsr;
3768 struct dmar_atsr_unit *atsru;
3769
3770 dev = pci_physfn(dev);
318fe7df 3771 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 3772 bridge = bus->self;
318fe7df 3773 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 3774 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 3775 return 0;
b5f82ddf 3776 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 3777 break;
318fe7df 3778 }
b5f82ddf
JL		 3779 if (!bridge)
3780 return 0;
318fe7df 3781
0e242612 3782 rcu_read_lock();
b5f82ddf
JL		 3783 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3784 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3785 if (atsr->segment != pci_domain_nr(dev->bus))
3786 continue;
3787
b683b230 3788 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 3789 if (tmp == &bridge->dev)
b683b230 3790 goto out;
b5f82ddf
JL		 3791
3792 if (atsru->include_all)
b683b230 3793 goto out;
b5f82ddf 3794 }
b683b230
JL		 3795 ret = 0;
3796out:
0e242612 3797 rcu_read_unlock();
318fe7df 3798
b683b230 3799 return ret;
318fe7df
SS		 3800}
3801
59ce0515
JL		 3802int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3803{
3804 int ret = 0;
3805 struct dmar_rmrr_unit *rmrru;
3806 struct dmar_atsr_unit *atsru;
3807 struct acpi_dmar_atsr *atsr;
3808 struct acpi_dmar_reserved_memory *rmrr;
3809
3810 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
3811 return 0;
3812
3813 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3814 rmrr = container_of(rmrru->hdr,
3815 struct acpi_dmar_reserved_memory, header);
3816 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3817 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3818 ((void *)rmrr) + rmrr->header.length,
3819 rmrr->segment, rmrru->devices,
3820 rmrru->devices_cnt);
27e24950 3821 if(ret < 0)
59ce0515
JL		 3822 return ret;
3823 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
27e24950
JL		 3824 dmar_remove_dev_scope(info, rmrr->segment,
3825 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 3826 }
3827 }
3828
3829 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3830 if (atsru->include_all)
3831 continue;
3832
3833 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3834 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3835 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3836 (void *)atsr + atsr->header.length,
3837 atsr->segment, atsru->devices,
3838 atsru->devices_cnt);
3839 if (ret > 0)
3840 break;
3841 else if(ret < 0)
3842 return ret;
3843 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3844 if (dmar_remove_dev_scope(info, atsr->segment,
3845 atsru->devices, atsru->devices_cnt))
3846 break;
3847 }
3848 }
3849
3850 return 0;
3851}
3852
99dcaded
FY		 3853/*
3854 * Here we only respond to action of unbound device from driver.
3855 *
3856 * Added device is not attached to its DMAR domain here yet. That will happen
3857 * when mapping the device to iova.
3858 */
3859static int device_notifier(struct notifier_block *nb,
3860 unsigned long action, void *data)
3861{
3862 struct device *dev = data;
99dcaded
FY		 3863 struct dmar_domain *domain;
3864
3d89194a 3865 if (iommu_dummy(dev))
44cd613c
DW		 3866 return 0;
3867
7e7dfab7
JL		 3868 if (action != BUS_NOTIFY_UNBOUND_DRIVER &&
3869 action != BUS_NOTIFY_DEL_DEVICE)
3870 return 0;
3871
1525a29a 3872 domain = find_domain(dev);
99dcaded
FY		 3873 if (!domain)
3874 return 0;
3875
3a5670e8 3876 down_read(&dmar_global_lock);
bf9c9eda 3877 domain_remove_one_dev_info(domain, dev);
ab8dfe25 3878 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 3879 domain_exit(domain);
3a5670e8 3880 up_read(&dmar_global_lock);
a97590e5 3881
99dcaded
FY		 3882 return 0;
3883}
3884
3885static struct notifier_block device_nb = {
3886 .notifier_call = device_notifier,
3887};
3888
75f05569
JL		 3889static int intel_iommu_memory_notifier(struct notifier_block *nb,
3890 unsigned long val, void *v)
3891{
3892 struct memory_notify *mhp = v;
3893 unsigned long long start, end;
3894 unsigned long start_vpfn, last_vpfn;
3895
3896 switch (val) {
3897 case MEM_GOING_ONLINE:
3898 start = mhp->start_pfn << PAGE_SHIFT;
3899 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
3900 if (iommu_domain_identity_map(si_domain, start, end)) {
3901 pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
3902 start, end);
3903 return NOTIFY_BAD;
3904 }
3905 break;
3906
3907 case MEM_OFFLINE:
3908 case MEM_CANCEL_ONLINE:
3909 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
3910 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
3911 while (start_vpfn <= last_vpfn) {
3912 struct iova *iova;
3913 struct dmar_drhd_unit *drhd;
3914 struct intel_iommu *iommu;
ea8ea460 3915 struct page *freelist;
75f05569
JL		 3916
3917 iova = find_iova(&si_domain->iovad, start_vpfn);
3918 if (iova == NULL) {
3919 pr_debug("dmar: failed get IOVA for PFN %lx\n",
3920 start_vpfn);
3921 break;
3922 }
3923
3924 iova = split_and_remove_iova(&si_domain->iovad, iova,
3925 start_vpfn, last_vpfn);
3926 if (iova == NULL) {
3927 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
3928 start_vpfn, last_vpfn);
3929 return NOTIFY_BAD;
3930 }
3931
ea8ea460
DW		 3932 freelist = domain_unmap(si_domain, iova->pfn_lo,
3933 iova->pfn_hi);
3934
75f05569
JL		 3935 rcu_read_lock();
3936 for_each_active_iommu(iommu, drhd)
3937 iommu_flush_iotlb_psi(iommu, si_domain->id,
a156ef99 3938 iova->pfn_lo, iova_size(iova),
ea8ea460 3939 !freelist, 0);
75f05569 3940 rcu_read_unlock();
ea8ea460 3941 dma_free_pagelist(freelist);
75f05569
JL		 3942
3943 start_vpfn = iova->pfn_hi + 1;
3944 free_iova_mem(iova);
3945 }
3946 break;
3947 }
3948
3949 return NOTIFY_OK;
3950}
3951
3952static struct notifier_block intel_iommu_memory_nb = {
3953 .notifier_call = intel_iommu_memory_notifier,
3954 .priority = 0
3955};
3956
a5459cfe
AW		 3957
3958static ssize_t intel_iommu_show_version(struct device *dev,
3959 struct device_attribute *attr,
3960 char *buf)
3961{
3962 struct intel_iommu *iommu = dev_get_drvdata(dev);
3963 u32 ver = readl(iommu->reg + DMAR_VER_REG);
3964 return sprintf(buf, "%d:%d\n",
3965 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
3966}
3967static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
3968
3969static ssize_t intel_iommu_show_address(struct device *dev,
3970 struct device_attribute *attr,
3971 char *buf)
3972{
3973 struct intel_iommu *iommu = dev_get_drvdata(dev);
3974 return sprintf(buf, "%llx\n", iommu->reg_phys);
3975}
3976static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
3977
3978static ssize_t intel_iommu_show_cap(struct device *dev,
3979 struct device_attribute *attr,
3980 char *buf)
3981{
3982 struct intel_iommu *iommu = dev_get_drvdata(dev);
3983 return sprintf(buf, "%llx\n", iommu->cap);
3984}
3985static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
3986
3987static ssize_t intel_iommu_show_ecap(struct device *dev,
3988 struct device_attribute *attr,
3989 char *buf)
3990{
3991 struct intel_iommu *iommu = dev_get_drvdata(dev);
3992 return sprintf(buf, "%llx\n", iommu->ecap);
3993}
3994static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
3995
3996static struct attribute *intel_iommu_attrs[] = {
3997 &dev_attr_version.attr,
3998 &dev_attr_address.attr,
3999 &dev_attr_cap.attr,
4000 &dev_attr_ecap.attr,
4001 NULL,
4002};
4003
4004static struct attribute_group intel_iommu_group = {
4005 .name = "intel-iommu",
4006 .attrs = intel_iommu_attrs,
4007};
4008
4009const struct attribute_group *intel_iommu_groups[] = {
4010 &intel_iommu_group,
4011 NULL,
4012};
4013
ba395927
KA		 4014int __init intel_iommu_init(void)
4015{
9bdc531e 4016 int ret = -ENODEV;
3a93c841 4017 struct dmar_drhd_unit *drhd;
7c919779 4018 struct intel_iommu *iommu;
ba395927 4019
a59b50e9
JC		 4020 /* VT-d is required for a TXT/tboot launch, so enforce that */
4021 force_on = tboot_force_iommu();
4022
3a5670e8
JL		 4023 if (iommu_init_mempool()) {
4024 if (force_on)
4025 panic("tboot: Failed to initialize iommu memory\n");
4026 return -ENOMEM;
4027 }
4028
4029 down_write(&dmar_global_lock);
a59b50e9
JC		 4030 if (dmar_table_init()) {
4031 if (force_on)
4032 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4033 goto out_free_dmar;
a59b50e9 4034 }
ba395927 4035
3a93c841
TI		 4036 /*
4037 * Disable translation if already enabled prior to OS handover.
4038 */
7c919779 4039 for_each_active_iommu(iommu, drhd)
3a93c841
TI		 4040 if (iommu->gcmd & DMA_GCMD_TE)
4041 iommu_disable_translation(iommu);
3a93c841 4042
c2c7286a 4043 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4044 if (force_on)
4045 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4046 goto out_free_dmar;
a59b50e9 4047 }
1886e8a9 4048
75f1cdf1 4049 if (no_iommu || dmar_disabled)
9bdc531e 4050 goto out_free_dmar;
2ae21010 4051
318fe7df
SS		 4052 if (list_empty(&dmar_rmrr_units))
4053 printk(KERN_INFO "DMAR: No RMRR found\n");
4054
4055 if (list_empty(&dmar_atsr_units))
4056 printk(KERN_INFO "DMAR: No ATSR found\n");
4057
51a63e67
JC		 4058 if (dmar_init_reserved_ranges()) {
4059 if (force_on)
4060 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4061 goto out_free_reserved_range;
51a63e67 4062 }
ba395927
KA		 4063
4064 init_no_remapping_devices();
4065
b779260b 4066 ret = init_dmars();
ba395927 4067 if (ret) {
a59b50e9
JC		 4068 if (force_on)
4069 panic("tboot: Failed to initialize DMARs\n");
ba395927 4070 printk(KERN_ERR "IOMMU: dmar init failed\n");
9bdc531e 4071 goto out_free_reserved_range;
ba395927 4072 }
3a5670e8 4073 up_write(&dmar_global_lock);
ba395927
KA		 4074 printk(KERN_INFO
4075 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
4076
5e0d2a6f 4077 init_timer(&unmap_timer);
75f1cdf1
FT		 4078#ifdef CONFIG_SWIOTLB
4079 swiotlb = 0;
4080#endif
19943b0e 4081 dma_ops = &intel_dma_ops;
4ed0d3e6 4082
134fac3f 4083 init_iommu_pm_ops();
a8bcbb0d 4084
a5459cfe
AW		 4085 for_each_active_iommu(iommu, drhd)
4086 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4087 intel_iommu_groups,
4088 iommu->name);
4089
4236d97d 4090 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4091 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4092 if (si_domain && !hw_pass_through)
4093 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 4094
8bc1f85c
ED		 4095 intel_iommu_enabled = 1;
4096
ba395927 4097 return 0;
9bdc531e
JL		 4098
4099out_free_reserved_range:
4100 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4101out_free_dmar:
4102 intel_iommu_free_dmars();
3a5670e8
JL		 4103 up_write(&dmar_global_lock);
4104 iommu_exit_mempool();
9bdc531e 4105 return ret;
ba395927 4106}
e820482c 4107
579305f7
AW		 4108static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4109{
4110 struct intel_iommu *iommu = opaque;
4111
4112 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4113 return 0;
4114}
4115
4116/*
4117 * NB - intel-iommu lacks any sort of reference counting for the users of
4118 * dependent devices. If multiple endpoints have intersecting dependent
4119 * devices, unbinding the driver from any one of them will possibly leave
4120 * the others unable to operate.
4121 */
3199aa6b 4122static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 4123 struct device *dev)
3199aa6b 4124{
0bcb3e28 4125 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4126 return;
4127
579305f7 4128 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
3199aa6b
HW		 4129}
4130
2c2e2c38 4131static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 4132 struct device *dev)
c7151a8d 4133{
bca2b916 4134 struct device_domain_info *info, *tmp;
c7151a8d
WH		 4135 struct intel_iommu *iommu;
4136 unsigned long flags;
4137 int found = 0;
156baca8 4138 u8 bus, devfn;
c7151a8d 4139
bf9c9eda 4140 iommu = device_to_iommu(dev, &bus, &devfn);
c7151a8d
WH		 4141 if (!iommu)
4142 return;
4143
4144 spin_lock_irqsave(&device_domain_lock, flags);
bca2b916 4145 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
bf9c9eda
DW		 4146 if (info->iommu == iommu && info->bus == bus &&
4147 info->devfn == devfn) {
109b9b04 4148 unlink_domain_info(info);
c7151a8d
WH		 4149 spin_unlock_irqrestore(&device_domain_lock, flags);
4150
93a23a72 4151 iommu_disable_dev_iotlb(info);
c7151a8d 4152 iommu_detach_dev(iommu, info->bus, info->devfn);
bf9c9eda 4153 iommu_detach_dependent_devices(iommu, dev);
c7151a8d
WH		 4154 free_devinfo_mem(info);
4155
4156 spin_lock_irqsave(&device_domain_lock, flags);
4157
4158 if (found)
4159 break;
4160 else
4161 continue;
4162 }
4163
4164 /* if there is no other devices under the same iommu
4165 * owned by this domain, clear this iommu in iommu_bmp
4166 * update iommu count and coherency
4167 */
8bbc4410 4168 if (info->iommu == iommu)
c7151a8d
WH		 4169 found = 1;
4170 }
4171
3e7abe25
RD		 4172 spin_unlock_irqrestore(&device_domain_lock, flags);
4173
c7151a8d 4174 if (found == 0) {
fb170fb4
JL		 4175 domain_detach_iommu(domain, iommu);
4176 if (!domain_type_is_vm_or_si(domain))
4177 iommu_detach_domain(domain, iommu);
c7151a8d 4178 }
c7151a8d
WH		 4179}
4180
2c2e2c38 4181static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4182{
4183 int adjust_width;
4184
4185 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
5e98c4b1
WH		 4186 domain_reserve_special_ranges(domain);
4187
4188 /* calculate AGAW */
4189 domain->gaw = guest_width;
4190 adjust_width = guestwidth_to_adjustwidth(guest_width);
4191 domain->agaw = width_to_agaw(adjust_width);
4192
5e98c4b1 4193 domain->iommu_coherency = 0;
c5b15255 4194 domain->iommu_snooping = 0;
6dd9a7c7 4195 domain->iommu_superpage = 0;
fe40f1e0 4196 domain->max_addr = 0;
5e98c4b1
WH		 4197
4198 /* always allocate the top pgd */
4c923d47 4199 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4200 if (!domain->pgd)
4201 return -ENOMEM;
4202 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4203 return 0;
4204}
4205
5d450806 4206static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 4207{
5d450806 4208 struct dmar_domain *dmar_domain;
38717946 4209
ab8dfe25 4210 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4211 if (!dmar_domain) {
38717946 4212 printk(KERN_ERR
5d450806
JR		 4213 "intel_iommu_domain_init: dmar_domain == NULL\n");
4214 return -ENOMEM;
38717946 4215 }
2c2e2c38 4216 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 4217 printk(KERN_ERR
5d450806 4218 "intel_iommu_domain_init() failed\n");
92d03cc8 4219 domain_exit(dmar_domain);
5d450806 4220 return -ENOMEM;
38717946 4221 }
8140a95d 4222 domain_update_iommu_cap(dmar_domain);
5d450806 4223 domain->priv = dmar_domain;
faa3d6f5 4224
8a0e715b
JR		 4225 domain->geometry.aperture_start = 0;
4226 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4227 domain->geometry.force_aperture = true;
4228
5d450806 4229 return 0;
38717946 4230}
38717946 4231
5d450806 4232static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 4233{
5d450806
JR		 4234 struct dmar_domain *dmar_domain = domain->priv;
4235
4236 domain->priv = NULL;
92d03cc8 4237 domain_exit(dmar_domain);
38717946 4238}
38717946 4239
4c5478c9
JR		 4240static int intel_iommu_attach_device(struct iommu_domain *domain,
4241 struct device *dev)
38717946 4242{
4c5478c9 4243 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0
WH		 4244 struct intel_iommu *iommu;
4245 int addr_width;
156baca8 4246 u8 bus, devfn;
faa3d6f5 4247
c875d2c1
AW		 4248 if (device_is_rmrr_locked(dev)) {
4249 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4250 return -EPERM;
4251 }
4252
7207d8f9
DW		 4253 /* normally dev is not mapped */
4254 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4255 struct dmar_domain *old_domain;
4256
1525a29a 4257 old_domain = find_domain(dev);
faa3d6f5 4258 if (old_domain) {
ab8dfe25 4259 if (domain_type_is_vm_or_si(dmar_domain))
bf9c9eda 4260 domain_remove_one_dev_info(old_domain, dev);
faa3d6f5
WH		 4261 else
4262 domain_remove_dev_info(old_domain);
4263 }
4264 }
4265
156baca8 4266 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4267 if (!iommu)
4268 return -ENODEV;
4269
4270 /* check if this iommu agaw is sufficient for max mapped address */
4271 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4272 if (addr_width > cap_mgaw(iommu->cap))
4273 addr_width = cap_mgaw(iommu->cap);
4274
4275 if (dmar_domain->max_addr > (1LL << addr_width)) {
4276 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4277 "sufficient for the mapped address (%llx)\n",
a99c47a2 4278 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4279 return -EFAULT;
4280 }
a99c47a2
TL		 4281 dmar_domain->gaw = addr_width;
4282
4283 /*
4284 * Knock out extra levels of page tables if necessary
4285 */
4286 while (iommu->agaw < dmar_domain->agaw) {
4287 struct dma_pte *pte;
4288
4289 pte = dmar_domain->pgd;
4290 if (dma_pte_present(pte)) {
25cbff16
SY		 4291 dmar_domain->pgd = (struct dma_pte *)
4292 phys_to_virt(dma_pte_addr(pte));
7a661013 4293 free_pgtable_page(pte);
a99c47a2
TL		 4294 }
4295 dmar_domain->agaw--;
4296 }
fe40f1e0 4297
5913c9bf 4298 return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
38717946 4299}
38717946 4300
4c5478c9
JR		 4301static void intel_iommu_detach_device(struct iommu_domain *domain,
4302 struct device *dev)
38717946 4303{
4c5478c9 4304 struct dmar_domain *dmar_domain = domain->priv;
4c5478c9 4305
bf9c9eda 4306 domain_remove_one_dev_info(dmar_domain, dev);
faa3d6f5 4307}
c7151a8d 4308
b146a1c9
JR		 4309static int intel_iommu_map(struct iommu_domain *domain,
4310 unsigned long iova, phys_addr_t hpa,
5009065d 4311 size_t size, int iommu_prot)
faa3d6f5 4312{
dde57a21 4313 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0 4314 u64 max_addr;
dde57a21 4315 int prot = 0;
faa3d6f5 4316 int ret;
fe40f1e0 4317
dde57a21
JR		 4318 if (iommu_prot & IOMMU_READ)
4319 prot |= DMA_PTE_READ;
4320 if (iommu_prot & IOMMU_WRITE)
4321 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4322 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4323 prot |= DMA_PTE_SNP;
dde57a21 4324
163cc52c 4325 max_addr = iova + size;
dde57a21 4326 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4327 u64 end;
4328
4329 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4330 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4331 if (end < max_addr) {
8954da1f 4332 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4333 "sufficient for the mapped address (%llx)\n",
8954da1f 4334 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4335 return -EFAULT;
4336 }
dde57a21 4337 dmar_domain->max_addr = max_addr;
fe40f1e0 4338 }
ad051221
DW		 4339 /* Round up size to next multiple of PAGE_SIZE, if it and
4340 the low bits of hpa would take us onto the next page */
88cb6a74 4341 size = aligned_nrpages(hpa, size);
ad051221
DW		 4342 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4343 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4344 return ret;
38717946 4345}
38717946 4346
5009065d 4347static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4348 unsigned long iova, size_t size)
38717946 4349{
dde57a21 4350 struct dmar_domain *dmar_domain = domain->priv;
ea8ea460
DW		 4351 struct page *freelist = NULL;
4352 struct intel_iommu *iommu;
4353 unsigned long start_pfn, last_pfn;
4354 unsigned int npages;
4355 int iommu_id, num, ndomains, level = 0;
5cf0a76f
DW		 4356
4357 /* Cope with horrid API which requires us to unmap more than the
4358 size argument if it happens to be a large-page mapping. */
4359 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4360 BUG();
4361
4362 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4363 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4364
ea8ea460
DW		 4365 start_pfn = iova >> VTD_PAGE_SHIFT;
4366 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4367
4368 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4369
4370 npages = last_pfn - start_pfn + 1;
4371
4372 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4373 iommu = g_iommus[iommu_id];
4374
4375 /*
4376 * find bit position of dmar_domain
4377 */
4378 ndomains = cap_ndoms(iommu->cap);
4379 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4380 if (iommu->domains[num] == dmar_domain)
4381 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4382 npages, !freelist, 0);
4383 }
4384
4385 }
4386
4387 dma_free_pagelist(freelist);
fe40f1e0 4388
163cc52c
DW		 4389 if (dmar_domain->max_addr == iova + size)
4390 dmar_domain->max_addr = iova;
b146a1c9 4391
5cf0a76f 4392 return size;
38717946 4393}
38717946 4394
d14d6577 4395static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4396 dma_addr_t iova)
38717946 4397{
d14d6577 4398 struct dmar_domain *dmar_domain = domain->priv;
38717946 4399 struct dma_pte *pte;
5cf0a76f 4400 int level = 0;
faa3d6f5 4401 u64 phys = 0;
38717946 4402
5cf0a76f 4403 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4404 if (pte)
faa3d6f5 4405 phys = dma_pte_addr(pte);
38717946 4406
faa3d6f5 4407 return phys;
38717946 4408}
a8bcbb0d 4409
dbb9fd86
SY		 4410static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4411 unsigned long cap)
4412{
4413 struct dmar_domain *dmar_domain = domain->priv;
4414
4415 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4416 return dmar_domain->iommu_snooping;
323f99cb 4417 if (cap == IOMMU_CAP_INTR_REMAP)
95a02e97 4418 return irq_remapping_enabled;
dbb9fd86
SY		 4419
4420 return 0;
4421}
4422
abdfdde2
AW		 4423static int intel_iommu_add_device(struct device *dev)
4424{
a5459cfe 4425 struct intel_iommu *iommu;
abdfdde2 4426 struct iommu_group *group;
156baca8 4427 u8 bus, devfn;
70ae6f0d 4428
a5459cfe
AW		 4429 iommu = device_to_iommu(dev, &bus, &devfn);
4430 if (!iommu)
70ae6f0d
AW		 4431 return -ENODEV;
4432
a5459cfe 4433 iommu_device_link(iommu->iommu_dev, dev);
a4ff1fc2 4434
e17f9ff4 4435 group = iommu_group_get_for_dev(dev);
783f157b 4436
e17f9ff4
AW		 4437 if (IS_ERR(group))
4438 return PTR_ERR(group);
bcb71abe 4439
abdfdde2 4440 iommu_group_put(group);
e17f9ff4 4441 return 0;
abdfdde2 4442}
70ae6f0d 4443
abdfdde2
AW		 4444static void intel_iommu_remove_device(struct device *dev)
4445{
a5459cfe
AW		 4446 struct intel_iommu *iommu;
4447 u8 bus, devfn;
4448
4449 iommu = device_to_iommu(dev, &bus, &devfn);
4450 if (!iommu)
4451 return;
4452
abdfdde2 4453 iommu_group_remove_device(dev);
a5459cfe
AW		 4454
4455 iommu_device_unlink(iommu->iommu_dev, dev);
70ae6f0d
AW		 4456}
4457
b22f6434 4458static const struct iommu_ops intel_iommu_ops = {
a8bcbb0d
JR		 4459 .domain_init = intel_iommu_domain_init,
4460 .domain_destroy = intel_iommu_domain_destroy,
4461 .attach_dev = intel_iommu_attach_device,
4462 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4463 .map = intel_iommu_map,
4464 .unmap = intel_iommu_unmap,
a8bcbb0d 4465 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 4466 .domain_has_cap = intel_iommu_domain_has_cap,
abdfdde2
AW		 4467 .add_device = intel_iommu_add_device,
4468 .remove_device = intel_iommu_remove_device,
6d1c56a9 4469 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4470};
9af88143 4471
9452618e
DV		 4472static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4473{
4474 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4475 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4476 dmar_map_gfx = 0;
4477}
4478
4479DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4480DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4481DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4482DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4483DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4484DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4485DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4486
d34d6517 4487static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4488{
4489 /*
4490 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4491 * but needs it. Same seems to hold for the desktop versions.
9af88143
DW		 4492 */
4493 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4494 rwbf_quirk = 1;
4495}
4496
4497DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4498DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4499DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4500DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4501DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4502DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4503DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4504
eecfd57f
AJ		 4505#define GGC 0x52
4506#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4507#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4508#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4509#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4510#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4511#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4512#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4513#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4514
d34d6517 4515static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4516{
4517 unsigned short ggc;
4518
eecfd57f 4519 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4520 return;
4521
eecfd57f 4522 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9eecabcb
DW		 4523 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4524 dmar_map_gfx = 0;
6fbcfb3e
DW		 4525 } else if (dmar_map_gfx) {
4526 /* we have to ensure the gfx device is idle before we flush */
4527 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4528 intel_iommu_strict = 1;
4529 }
9eecabcb
DW		 4530}
4531DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4532DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4533DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4534DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4535
e0fc7e0b
DW		 4536/* On Tylersburg chipsets, some BIOSes have been known to enable the
4537 ISOCH DMAR unit for the Azalia sound device, but not give it any
4538 TLB entries, which causes it to deadlock. Check for that. We do
4539 this in a function called from init_dmars(), instead of in a PCI
4540 quirk, because we don't want to print the obnoxious "BIOS broken"
4541 message if VT-d is actually disabled.
4542*/
4543static void __init check_tylersburg_isoch(void)
4544{
4545 struct pci_dev *pdev;
4546 uint32_t vtisochctrl;
4547
4548 /* If there's no Azalia in the system anyway, forget it. */
4549 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4550 if (!pdev)
4551 return;
4552 pci_dev_put(pdev);
4553
4554 /* System Management Registers. Might be hidden, in which case
4555 we can't do the sanity check. But that's OK, because the
4556 known-broken BIOSes _don't_ actually hide it, so far. */
4557 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4558 if (!pdev)
4559 return;
4560
4561 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4562 pci_dev_put(pdev);
4563 return;
4564 }
4565
4566 pci_dev_put(pdev);
4567
4568 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4569 if (vtisochctrl & 1)
4570 return;
4571
4572 /* Drop all bits other than the number of TLB entries */
4573 vtisochctrl &= 0x1c;
4574
4575 /* If we have the recommended number of TLB entries (16), fine. */
4576 if (vtisochctrl == 0x10)
4577 return;
4578
4579 /* Zero TLB entries? You get to ride the short bus to school. */
4580 if (!vtisochctrl) {
4581 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4582 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4583 dmi_get_system_info(DMI_BIOS_VENDOR),
4584 dmi_get_system_info(DMI_BIOS_VERSION),
4585 dmi_get_system_info(DMI_PRODUCT_VERSION));
4586 iommu_identity_mapping |= IDENTMAP_AZALIA;
4587 return;
4588 }
4589
4590 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4591 vtisochctrl);
4592}
Ubuntu Artful kernel mirror