]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - drivers/iommu/intel-iommu.c
projects / mirror_ubuntu-zesty-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
iommu/vt-d: Allocate dynamic domain id for virtual domains only
[mirror_ubuntu-zesty-kernel.git] / drivers / iommu / intel-iommu.c
CommitLineData
ba395927 1/*
ea8ea460 2 * Copyright © 2006-2014 Intel Corporation.
ba395927
KA		 3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
ea8ea460
DW		 13 * Authors: David Woodhouse <dwmw2@infradead.org>,
14 * Ashok Raj <ashok.raj@intel.com>,
15 * Shaohua Li <shaohua.li@intel.com>,
16 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
17 * Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 18 */
19
20#include <linux/init.h>
21#include <linux/bitmap.h>
5e0d2a6f 22#include <linux/debugfs.h>
54485c30 23#include <linux/export.h>
ba395927
KA		 24#include <linux/slab.h>
25#include <linux/irq.h>
26#include <linux/interrupt.h>
ba395927
KA		 27#include <linux/spinlock.h>
28#include <linux/pci.h>
29#include <linux/dmar.h>
30#include <linux/dma-mapping.h>
31#include <linux/mempool.h>
75f05569 32#include <linux/memory.h>
5e0d2a6f 33#include <linux/timer.h>
38717946 34#include <linux/iova.h>
5d450806 35#include <linux/iommu.h>
38717946 36#include <linux/intel-iommu.h>
134fac3f 37#include <linux/syscore_ops.h>
69575d38 38#include <linux/tboot.h>
adb2fe02 39#include <linux/dmi.h>
5cdede24 40#include <linux/pci-ats.h>
0ee332c1 41#include <linux/memblock.h>
36746436 42#include <linux/dma-contiguous.h>
8a8f422d 43#include <asm/irq_remapping.h>
ba395927 44#include <asm/cacheflush.h>
46a7fa27 45#include <asm/iommu.h>
ba395927 46
078e1ee2
JR		 47#include "irq_remapping.h"
48
5b6985ce
FY		 49#define ROOT_SIZE VTD_PAGE_SIZE
50#define CONTEXT_SIZE VTD_PAGE_SIZE
51
ba395927
KA		 52#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
53#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 54#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 55
56#define IOAPIC_RANGE_START (0xfee00000)
57#define IOAPIC_RANGE_END (0xfeefffff)
58#define IOVA_START_ADDR (0x1000)
59
60#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
61
4ed0d3e6 62#define MAX_AGAW_WIDTH 64
5c645b35 63#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
4ed0d3e6 64
2ebe3151
DW		 65#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
66#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
67
68/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
69 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
70#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
71 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
72#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 73
f27be03b 74#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 75#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 76#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 77
df08cdc7
AM		 78/* page table handling */
79#define LEVEL_STRIDE (9)
80#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
81
6d1c56a9
OBC		 82/*
83 * This bitmap is used to advertise the page sizes our hardware support
84 * to the IOMMU core, which will then use this information to split
85 * physically contiguous memory regions it is mapping into page sizes
86 * that we support.
87 *
88 * Traditionally the IOMMU core just handed us the mappings directly,
89 * after making sure the size is an order of a 4KiB page and that the
90 * mapping has natural alignment.
91 *
92 * To retain this behavior, we currently advertise that we support
93 * all page sizes that are an order of 4KiB.
94 *
95 * If at some point we'd like to utilize the IOMMU core's new behavior,
96 * we could change this to advertise the real page sizes we support.
97 */
98#define INTEL_IOMMU_PGSIZES (~0xFFFUL)
99
df08cdc7
AM		 100static inline int agaw_to_level(int agaw)
101{
102 return agaw + 2;
103}
104
105static inline int agaw_to_width(int agaw)
106{
5c645b35 107 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
df08cdc7
AM		 108}
109
110static inline int width_to_agaw(int width)
111{
5c645b35 112 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
df08cdc7
AM		 113}
114
115static inline unsigned int level_to_offset_bits(int level)
116{
117 return (level - 1) * LEVEL_STRIDE;
118}
119
120static inline int pfn_level_offset(unsigned long pfn, int level)
121{
122 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
123}
124
125static inline unsigned long level_mask(int level)
126{
127 return -1UL << level_to_offset_bits(level);
128}
129
130static inline unsigned long level_size(int level)
131{
132 return 1UL << level_to_offset_bits(level);
133}
134
135static inline unsigned long align_to_level(unsigned long pfn, int level)
136{
137 return (pfn + level_size(level) - 1) & level_mask(level);
138}
fd18de50 139
6dd9a7c7
YS		 140static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
141{
5c645b35 142 return 1 << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
6dd9a7c7
YS		 143}
144
dd4e8319
DW		 145/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
146 are never going to work. */
147static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
148{
149 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
150}
151
152static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
153{
154 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
155}
156static inline unsigned long page_to_dma_pfn(struct page *pg)
157{
158 return mm_to_dma_pfn(page_to_pfn(pg));
159}
160static inline unsigned long virt_to_dma_pfn(void *p)
161{
162 return page_to_dma_pfn(virt_to_page(p));
163}
164
d9630fe9
WH		 165/* global iommu list, set NULL for ignored DMAR units */
166static struct intel_iommu **g_iommus;
167
e0fc7e0b 168static void __init check_tylersburg_isoch(void);
9af88143
DW		 169static int rwbf_quirk;
170
b779260b
JC		 171/*
172 * set to 1 to panic kernel if can't successfully enable VT-d
173 * (used when kernel is launched w/ TXT)
174 */
175static int force_on = 0;
176
46b08e1a
MM		 177/*
178 * 0: Present
179 * 1-11: Reserved
180 * 12-63: Context Ptr (12 - (haw-1))
181 * 64-127: Reserved
182 */
183struct root_entry {
184 u64 val;
185 u64 rsvd1;
186};
187#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
188static inline bool root_present(struct root_entry *root)
189{
190 return (root->val & 1);
191}
192static inline void set_root_present(struct root_entry *root)
193{
194 root->val |= 1;
195}
196static inline void set_root_value(struct root_entry *root, unsigned long value)
197{
198 root->val |= value & VTD_PAGE_MASK;
199}
200
201static inline struct context_entry *
202get_context_addr_from_root(struct root_entry *root)
203{
204 return (struct context_entry *)
205 (root_present(root)?phys_to_virt(
206 root->val & VTD_PAGE_MASK) :
207 NULL);
208}
209
7a8fc25e
MM		 210/*
211 * low 64 bits:
212 * 0: present
213 * 1: fault processing disable
214 * 2-3: translation type
215 * 12-63: address space root
216 * high 64 bits:
217 * 0-2: address width
218 * 3-6: aval
219 * 8-23: domain id
220 */
221struct context_entry {
222 u64 lo;
223 u64 hi;
224};
c07e7d21
MM		 225
226static inline bool context_present(struct context_entry *context)
227{
228 return (context->lo & 1);
229}
230static inline void context_set_present(struct context_entry *context)
231{
232 context->lo |= 1;
233}
234
235static inline void context_set_fault_enable(struct context_entry *context)
236{
237 context->lo &= (((u64)-1) << 2) | 1;
238}
239
c07e7d21
MM		 240static inline void context_set_translation_type(struct context_entry *context,
241 unsigned long value)
242{
243 context->lo &= (((u64)-1) << 4) | 3;
244 context->lo |= (value & 3) << 2;
245}
246
247static inline void context_set_address_root(struct context_entry *context,
248 unsigned long value)
249{
250 context->lo |= value & VTD_PAGE_MASK;
251}
252
253static inline void context_set_address_width(struct context_entry *context,
254 unsigned long value)
255{
256 context->hi |= value & 7;
257}
258
259static inline void context_set_domain_id(struct context_entry *context,
260 unsigned long value)
261{
262 context->hi |= (value & ((1 << 16) - 1)) << 8;
263}
264
265static inline void context_clear_entry(struct context_entry *context)
266{
267 context->lo = 0;
268 context->hi = 0;
269}
7a8fc25e 270
622ba12a
MM		 271/*
272 * 0: readable
273 * 1: writable
274 * 2-6: reserved
275 * 7: super page
9cf06697
SY		 276 * 8-10: available
277 * 11: snoop behavior
622ba12a
MM		 278 * 12-63: Host physcial address
279 */
280struct dma_pte {
281 u64 val;
282};
622ba12a 283
19c239ce
MM		 284static inline void dma_clear_pte(struct dma_pte *pte)
285{
286 pte->val = 0;
287}
288
19c239ce
MM		 289static inline u64 dma_pte_addr(struct dma_pte *pte)
290{
c85994e4
DW		 291#ifdef CONFIG_64BIT
292 return pte->val & VTD_PAGE_MASK;
293#else
294 /* Must have a full atomic 64-bit read */
1a8bd481 295 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
c85994e4 296#endif
19c239ce
MM		 297}
298
19c239ce
MM		 299static inline bool dma_pte_present(struct dma_pte *pte)
300{
301 return (pte->val & 3) != 0;
302}
622ba12a 303
4399c8bf
AK		 304static inline bool dma_pte_superpage(struct dma_pte *pte)
305{
c3c75eb7 306 return (pte->val & DMA_PTE_LARGE_PAGE);
4399c8bf
AK		 307}
308
75e6bf96
DW		 309static inline int first_pte_in_page(struct dma_pte *pte)
310{
311 return !((unsigned long)pte & ~VTD_PAGE_MASK);
312}
313
2c2e2c38
FY		 314/*
315 * This domain is a statically identity mapping domain.
316 * 1. This domain creats a static 1:1 mapping to all usable memory.
317 * 2. It maps to each iommu if successful.
318 * 3. Each iommu mapps to this domain if successful.
319 */
19943b0e
DW		 320static struct dmar_domain *si_domain;
321static int hw_pass_through = 1;
2c2e2c38 322
1ce28feb
WH		 323/* domain represents a virtual machine, more than one devices
324 * across iommus may be owned in one domain, e.g. kvm guest.
325 */
ab8dfe25 326#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 0)
1ce28feb 327
2c2e2c38 328/* si_domain contains mulitple devices */
ab8dfe25 329#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 1)
2c2e2c38 330
1b198bb0
MT		 331/* define the limit of IOMMUs supported in each domain */
332#ifdef CONFIG_X86
333# define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
334#else
335# define IOMMU_UNITS_SUPPORTED 64
336#endif
337
99126f7c
MM		 338struct dmar_domain {
339 int id; /* domain id */
4c923d47 340 int nid; /* node id */
1b198bb0
MT		 341 DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
342 /* bitmap of iommus this domain uses*/
99126f7c
MM		 343
344 struct list_head devices; /* all devices' list */
345 struct iova_domain iovad; /* iova's that belong to this domain */
346
347 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 348 int gaw; /* max guest address width */
349
350 /* adjusted guest address width, 0 is level 2 30-bit */
351 int agaw;
352
3b5410e7 353 int flags; /* flags to find out type of domain */
8e604097
WH		 354
355 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 356 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d 357 int iommu_count; /* reference count of iommu */
6dd9a7c7
YS		 358 int iommu_superpage;/* Level of superpages supported:
359 0 == 4KiB (no superpages), 1 == 2MiB,
360 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
c7151a8d 361 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 362 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 363};
364
a647dacb
MM		 365/* PCI domain-device relationship */
366struct device_domain_info {
367 struct list_head link; /* link to domain siblings */
368 struct list_head global; /* link to global list */
276dbf99 369 u8 bus; /* PCI bus number */
a647dacb 370 u8 devfn; /* PCI devfn number */
0bcb3e28 371 struct device *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 372 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 373 struct dmar_domain *domain; /* pointer to domain */
374};
375
b94e4117
JL		 376struct dmar_rmrr_unit {
377 struct list_head list; /* list of rmrr units */
378 struct acpi_dmar_header *hdr; /* ACPI header */
379 u64 base_address; /* reserved base address*/
380 u64 end_address; /* reserved end address */
832bd858 381 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 382 int devices_cnt; /* target device count */
383};
384
385struct dmar_atsr_unit {
386 struct list_head list; /* list of ATSR units */
387 struct acpi_dmar_header *hdr; /* ACPI header */
832bd858 388 struct dmar_dev_scope *devices; /* target devices */
b94e4117
JL		 389 int devices_cnt; /* target device count */
390 u8 include_all:1; /* include all ports */
391};
392
393static LIST_HEAD(dmar_atsr_units);
394static LIST_HEAD(dmar_rmrr_units);
395
396#define for_each_rmrr_units(rmrr) \
397 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
398
5e0d2a6f 399static void flush_unmaps_timeout(unsigned long data);
400
b707cb02 401static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
5e0d2a6f 402
80b20dd8 403#define HIGH_WATER_MARK 250
404struct deferred_flush_tables {
405 int next;
406 struct iova *iova[HIGH_WATER_MARK];
407 struct dmar_domain *domain[HIGH_WATER_MARK];
ea8ea460 408 struct page *freelist[HIGH_WATER_MARK];
80b20dd8 409};
410
411static struct deferred_flush_tables *deferred_flush;
412
5e0d2a6f 413/* bitmap for indexing intel_iommus */
5e0d2a6f 414static int g_num_of_iommus;
415
416static DEFINE_SPINLOCK(async_umap_flush_lock);
417static LIST_HEAD(unmaps_to_do);
418
419static int timer_on;
420static long list_size;
5e0d2a6f 421
92d03cc8 422static void domain_exit(struct dmar_domain *domain);
ba395927 423static void domain_remove_dev_info(struct dmar_domain *domain);
b94e4117 424static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 425 struct device *dev);
92d03cc8 426static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 427 struct device *dev);
ba395927 428
d3f13810 429#ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
0cd5c3c8
KM		 430int dmar_disabled = 0;
431#else
432int dmar_disabled = 1;
d3f13810 433#endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
0cd5c3c8 434
8bc1f85c
ED		 435int intel_iommu_enabled = 0;
436EXPORT_SYMBOL_GPL(intel_iommu_enabled);
437
2d9e667e 438static int dmar_map_gfx = 1;
7d3b03ce 439static int dmar_forcedac;
5e0d2a6f 440static int intel_iommu_strict;
6dd9a7c7 441static int intel_iommu_superpage = 1;
ba395927 442
c0771df8
DW		 443int intel_iommu_gfx_mapped;
444EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
445
ba395927
KA		 446#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
447static DEFINE_SPINLOCK(device_domain_lock);
448static LIST_HEAD(device_domain_list);
449
b22f6434 450static const struct iommu_ops intel_iommu_ops;
a8bcbb0d 451
ba395927
KA		 452static int __init intel_iommu_setup(char *str)
453{
454 if (!str)
455 return -EINVAL;
456 while (*str) {
0cd5c3c8
KM		 457 if (!strncmp(str, "on", 2)) {
458 dmar_disabled = 0;
459 printk(KERN_INFO "Intel-IOMMU: enabled\n");
460 } else if (!strncmp(str, "off", 3)) {
ba395927 461 dmar_disabled = 1;
0cd5c3c8 462 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 463 } else if (!strncmp(str, "igfx_off", 8)) {
464 dmar_map_gfx = 0;
465 printk(KERN_INFO
466 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 467 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 468 printk(KERN_INFO
7d3b03ce
KA		 469 "Intel-IOMMU: Forcing DAC for PCI devices\n");
470 dmar_forcedac = 1;
5e0d2a6f 471 } else if (!strncmp(str, "strict", 6)) {
472 printk(KERN_INFO
473 "Intel-IOMMU: disable batched IOTLB flush\n");
474 intel_iommu_strict = 1;
6dd9a7c7
YS		 475 } else if (!strncmp(str, "sp_off", 6)) {
476 printk(KERN_INFO
477 "Intel-IOMMU: disable supported super page\n");
478 intel_iommu_superpage = 0;
ba395927
KA		 479 }
480
481 str += strcspn(str, ",");
482 while (*str == ',')
483 str++;
484 }
485 return 0;
486}
487__setup("intel_iommu=", intel_iommu_setup);
488
489static struct kmem_cache *iommu_domain_cache;
490static struct kmem_cache *iommu_devinfo_cache;
491static struct kmem_cache *iommu_iova_cache;
492
4c923d47 493static inline void *alloc_pgtable_page(int node)
eb3fa7cb 494{
4c923d47
SS		 495 struct page *page;
496 void *vaddr = NULL;
eb3fa7cb 497
4c923d47
SS		 498 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
499 if (page)
500 vaddr = page_address(page);
eb3fa7cb 501 return vaddr;
ba395927
KA		 502}
503
504static inline void free_pgtable_page(void *vaddr)
505{
506 free_page((unsigned long)vaddr);
507}
508
509static inline void *alloc_domain_mem(void)
510{
354bb65e 511 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 512}
513
38717946 514static void free_domain_mem(void *vaddr)
ba395927
KA		 515{
516 kmem_cache_free(iommu_domain_cache, vaddr);
517}
518
519static inline void * alloc_devinfo_mem(void)
520{
354bb65e 521 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 522}
523
524static inline void free_devinfo_mem(void *vaddr)
525{
526 kmem_cache_free(iommu_devinfo_cache, vaddr);
527}
528
529struct iova *alloc_iova_mem(void)
530{
354bb65e 531 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
ba395927
KA		 532}
533
534void free_iova_mem(struct iova *iova)
535{
536 kmem_cache_free(iommu_iova_cache, iova);
537}
538
ab8dfe25
JL		 539static inline int domain_type_is_vm(struct dmar_domain *domain)
540{
541 return domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE;
542}
543
544static inline int domain_type_is_vm_or_si(struct dmar_domain *domain)
545{
546 return domain->flags & (DOMAIN_FLAG_VIRTUAL_MACHINE |
547 DOMAIN_FLAG_STATIC_IDENTITY);
548}
1b573683 549
4ed0d3e6 550static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 551{
552 unsigned long sagaw;
553 int agaw = -1;
554
555 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 556 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 557 agaw >= 0; agaw--) {
558 if (test_bit(agaw, &sagaw))
559 break;
560 }
561
562 return agaw;
563}
564
4ed0d3e6
FY		 565/*
566 * Calculate max SAGAW for each iommu.
567 */
568int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
569{
570 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
571}
572
573/*
574 * calculate agaw for each iommu.
575 * "SAGAW" may be different across iommus, use a default agaw, and
576 * get a supported less agaw for iommus that don't support the default agaw.
577 */
578int iommu_calculate_agaw(struct intel_iommu *iommu)
579{
580 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
581}
582
2c2e2c38 583/* This functionin only returns single iommu in a domain */
8c11e798
WH		 584static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
585{
586 int iommu_id;
587
2c2e2c38 588 /* si_domain and vm domain should not get here. */
ab8dfe25 589 BUG_ON(domain_type_is_vm_or_si(domain));
1b198bb0 590 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
8c11e798
WH		 591 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
592 return NULL;
593
594 return g_iommus[iommu_id];
595}
596
8e604097
WH		 597static void domain_update_iommu_coherency(struct dmar_domain *domain)
598{
d0501960
DW		 599 struct dmar_drhd_unit *drhd;
600 struct intel_iommu *iommu;
601 int i, found = 0;
2e12bc29 602
d0501960 603 domain->iommu_coherency = 1;
8e604097 604
1b198bb0 605 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
d0501960 606 found = 1;
8e604097
WH		 607 if (!ecap_coherent(g_iommus[i]->ecap)) {
608 domain->iommu_coherency = 0;
609 break;
610 }
8e604097 611 }
d0501960
DW		 612 if (found)
613 return;
614
615 /* No hardware attached; use lowest common denominator */
616 rcu_read_lock();
617 for_each_active_iommu(iommu, drhd) {
618 if (!ecap_coherent(iommu->ecap)) {
619 domain->iommu_coherency = 0;
620 break;
621 }
622 }
623 rcu_read_unlock();
8e604097
WH		 624}
625
58c610bd
SY		 626static void domain_update_iommu_snooping(struct dmar_domain *domain)
627{
628 int i;
629
630 domain->iommu_snooping = 1;
631
1b198bb0 632 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
58c610bd
SY		 633 if (!ecap_sc_support(g_iommus[i]->ecap)) {
634 domain->iommu_snooping = 0;
635 break;
636 }
58c610bd
SY		 637 }
638}
639
6dd9a7c7
YS		 640static void domain_update_iommu_superpage(struct dmar_domain *domain)
641{
8140a95d
AK		 642 struct dmar_drhd_unit *drhd;
643 struct intel_iommu *iommu = NULL;
644 int mask = 0xf;
6dd9a7c7
YS		 645
646 if (!intel_iommu_superpage) {
647 domain->iommu_superpage = 0;
648 return;
649 }
650
8140a95d 651 /* set iommu_superpage to the smallest common denominator */
0e242612 652 rcu_read_lock();
8140a95d
AK		 653 for_each_active_iommu(iommu, drhd) {
654 mask &= cap_super_page_val(iommu->cap);
6dd9a7c7
YS		 655 if (!mask) {
656 break;
657 }
658 }
0e242612
JL		 659 rcu_read_unlock();
660
6dd9a7c7
YS		 661 domain->iommu_superpage = fls(mask);
662}
663
58c610bd
SY		 664/* Some capabilities may be different across iommus */
665static void domain_update_iommu_cap(struct dmar_domain *domain)
666{
667 domain_update_iommu_coherency(domain);
668 domain_update_iommu_snooping(domain);
6dd9a7c7 669 domain_update_iommu_superpage(domain);
58c610bd
SY		 670}
671
156baca8 672static struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
c7151a8d
WH		 673{
674 struct dmar_drhd_unit *drhd = NULL;
b683b230 675 struct intel_iommu *iommu;
156baca8
DW		 676 struct device *tmp;
677 struct pci_dev *ptmp, *pdev = NULL;
aa4d066a 678 u16 segment = 0;
c7151a8d
WH		 679 int i;
680
156baca8
DW		 681 if (dev_is_pci(dev)) {
682 pdev = to_pci_dev(dev);
683 segment = pci_domain_nr(pdev->bus);
684 } else if (ACPI_COMPANION(dev))
685 dev = &ACPI_COMPANION(dev)->dev;
686
0e242612 687 rcu_read_lock();
b683b230 688 for_each_active_iommu(iommu, drhd) {
156baca8 689 if (pdev && segment != drhd->segment)
276dbf99 690 continue;
c7151a8d 691
b683b230 692 for_each_active_dev_scope(drhd->devices,
156baca8
DW		 693 drhd->devices_cnt, i, tmp) {
694 if (tmp == dev) {
695 *bus = drhd->devices[i].bus;
696 *devfn = drhd->devices[i].devfn;
b683b230 697 goto out;
156baca8
DW		 698 }
699
700 if (!pdev || !dev_is_pci(tmp))
701 continue;
702
703 ptmp = to_pci_dev(tmp);
704 if (ptmp->subordinate &&
705 ptmp->subordinate->number <= pdev->bus->number &&
706 ptmp->subordinate->busn_res.end >= pdev->bus->number)
707 goto got_pdev;
924b6231 708 }
c7151a8d 709
156baca8
DW		 710 if (pdev && drhd->include_all) {
711 got_pdev:
712 *bus = pdev->bus->number;
713 *devfn = pdev->devfn;
b683b230 714 goto out;
156baca8 715 }
c7151a8d 716 }
b683b230 717 iommu = NULL;
156baca8 718 out:
0e242612 719 rcu_read_unlock();
c7151a8d 720
b683b230 721 return iommu;
c7151a8d
WH		 722}
723
5331fe6f
WH		 724static void domain_flush_cache(struct dmar_domain *domain,
725 void *addr, int size)
726{
727 if (!domain->iommu_coherency)
728 clflush_cache_range(addr, size);
729}
730
ba395927
KA		 731/* Gets context entry for a given bus and devfn */
732static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
733 u8 bus, u8 devfn)
734{
735 struct root_entry *root;
736 struct context_entry *context;
737 unsigned long phy_addr;
738 unsigned long flags;
739
740 spin_lock_irqsave(&iommu->lock, flags);
741 root = &iommu->root_entry[bus];
742 context = get_context_addr_from_root(root);
743 if (!context) {
4c923d47
SS		 744 context = (struct context_entry *)
745 alloc_pgtable_page(iommu->node);
ba395927
KA		 746 if (!context) {
747 spin_unlock_irqrestore(&iommu->lock, flags);
748 return NULL;
749 }
5b6985ce 750 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 751 phy_addr = virt_to_phys((void *)context);
752 set_root_value(root, phy_addr);
753 set_root_present(root);
754 __iommu_flush_cache(iommu, root, sizeof(*root));
755 }
756 spin_unlock_irqrestore(&iommu->lock, flags);
757 return &context[devfn];
758}
759
760static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
761{
762 struct root_entry *root;
763 struct context_entry *context;
764 int ret;
765 unsigned long flags;
766
767 spin_lock_irqsave(&iommu->lock, flags);
768 root = &iommu->root_entry[bus];
769 context = get_context_addr_from_root(root);
770 if (!context) {
771 ret = 0;
772 goto out;
773 }
c07e7d21 774 ret = context_present(&context[devfn]);
ba395927
KA		 775out:
776 spin_unlock_irqrestore(&iommu->lock, flags);
777 return ret;
778}
779
780static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
781{
782 struct root_entry *root;
783 struct context_entry *context;
784 unsigned long flags;
785
786 spin_lock_irqsave(&iommu->lock, flags);
787 root = &iommu->root_entry[bus];
788 context = get_context_addr_from_root(root);
789 if (context) {
c07e7d21 790 context_clear_entry(&context[devfn]);
ba395927
KA		 791 __iommu_flush_cache(iommu, &context[devfn], \
792 sizeof(*context));
793 }
794 spin_unlock_irqrestore(&iommu->lock, flags);
795}
796
797static void free_context_table(struct intel_iommu *iommu)
798{
799 struct root_entry *root;
800 int i;
801 unsigned long flags;
802 struct context_entry *context;
803
804 spin_lock_irqsave(&iommu->lock, flags);
805 if (!iommu->root_entry) {
806 goto out;
807 }
808 for (i = 0; i < ROOT_ENTRY_NR; i++) {
809 root = &iommu->root_entry[i];
810 context = get_context_addr_from_root(root);
811 if (context)
812 free_pgtable_page(context);
813 }
814 free_pgtable_page(iommu->root_entry);
815 iommu->root_entry = NULL;
816out:
817 spin_unlock_irqrestore(&iommu->lock, flags);
818}
819
b026fd28 820static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
5cf0a76f 821 unsigned long pfn, int *target_level)
ba395927 822{
b026fd28 823 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927
KA		 824 struct dma_pte *parent, *pte = NULL;
825 int level = agaw_to_level(domain->agaw);
4399c8bf 826 int offset;
ba395927
KA		 827
828 BUG_ON(!domain->pgd);
f9423606
JS		 829
830 if (addr_width < BITS_PER_LONG && pfn >> addr_width)
831 /* Address beyond IOMMU's addressing capabilities. */
832 return NULL;
833
ba395927
KA		 834 parent = domain->pgd;
835
5cf0a76f 836 while (1) {
ba395927
KA		 837 void *tmp_page;
838
b026fd28 839 offset = pfn_level_offset(pfn, level);
ba395927 840 pte = &parent[offset];
5cf0a76f 841 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
6dd9a7c7 842 break;
5cf0a76f 843 if (level == *target_level)
ba395927
KA		 844 break;
845
19c239ce 846 if (!dma_pte_present(pte)) {
c85994e4
DW		 847 uint64_t pteval;
848
4c923d47 849 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 850
206a73c1 851 if (!tmp_page)
ba395927 852 return NULL;
206a73c1 853
c85994e4 854 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 855 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
effad4b5 856 if (cmpxchg64(&pte->val, 0ULL, pteval))
c85994e4
DW		 857 /* Someone else set it while we were thinking; use theirs. */
858 free_pgtable_page(tmp_page);
effad4b5 859 else
c85994e4 860 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 861 }
5cf0a76f
DW		 862 if (level == 1)
863 break;
864
19c239ce 865 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 866 level--;
867 }
868
5cf0a76f
DW		 869 if (!*target_level)
870 *target_level = level;
871
ba395927
KA		 872 return pte;
873}
874
6dd9a7c7 875
ba395927 876/* return address's pte at specific level */
90dcfb5e
DW		 877static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
878 unsigned long pfn,
6dd9a7c7 879 int level, int *large_page)
ba395927
KA		 880{
881 struct dma_pte *parent, *pte = NULL;
882 int total = agaw_to_level(domain->agaw);
883 int offset;
884
885 parent = domain->pgd;
886 while (level <= total) {
90dcfb5e 887 offset = pfn_level_offset(pfn, total);
ba395927
KA		 888 pte = &parent[offset];
889 if (level == total)
890 return pte;
891
6dd9a7c7
YS		 892 if (!dma_pte_present(pte)) {
893 *large_page = total;
ba395927 894 break;
6dd9a7c7
YS		 895 }
896
e16922af 897 if (dma_pte_superpage(pte)) {
6dd9a7c7
YS		 898 *large_page = total;
899 return pte;
900 }
901
19c239ce 902 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 903 total--;
904 }
905 return NULL;
906}
907
ba395927 908/* clear last level pte, a tlb flush should be followed */
5cf0a76f 909static void dma_pte_clear_range(struct dmar_domain *domain,
595badf5
DW		 910 unsigned long start_pfn,
911 unsigned long last_pfn)
ba395927 912{
04b18e65 913 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
6dd9a7c7 914 unsigned int large_page = 1;
310a5ab9 915 struct dma_pte *first_pte, *pte;
66eae846 916
04b18e65 917 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
595badf5 918 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 919 BUG_ON(start_pfn > last_pfn);
ba395927 920
04b18e65 921 /* we don't need lock here; nobody else touches the iova range */
59c36286 922 do {
6dd9a7c7
YS		 923 large_page = 1;
924 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
310a5ab9 925 if (!pte) {
6dd9a7c7 926 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
310a5ab9
DW		 927 continue;
928 }
6dd9a7c7 929 do {
310a5ab9 930 dma_clear_pte(pte);
6dd9a7c7 931 start_pfn += lvl_to_nr_pages(large_page);
310a5ab9 932 pte++;
75e6bf96
DW		 933 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
934
310a5ab9
DW		 935 domain_flush_cache(domain, first_pte,
936 (void *)pte - (void *)first_pte);
59c36286
DW		 937
938 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 939}
940
3269ee0b
AW		 941static void dma_pte_free_level(struct dmar_domain *domain, int level,
942 struct dma_pte *pte, unsigned long pfn,
943 unsigned long start_pfn, unsigned long last_pfn)
944{
945 pfn = max(start_pfn, pfn);
946 pte = &pte[pfn_level_offset(pfn, level)];
947
948 do {
949 unsigned long level_pfn;
950 struct dma_pte *level_pte;
951
952 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
953 goto next;
954
955 level_pfn = pfn & level_mask(level - 1);
956 level_pte = phys_to_virt(dma_pte_addr(pte));
957
958 if (level > 2)
959 dma_pte_free_level(domain, level - 1, level_pte,
960 level_pfn, start_pfn, last_pfn);
961
962 /* If range covers entire pagetable, free it */
963 if (!(start_pfn > level_pfn ||
08336fd2 964 last_pfn < level_pfn + level_size(level) - 1)) {
3269ee0b
AW		 965 dma_clear_pte(pte);
966 domain_flush_cache(domain, pte, sizeof(*pte));
967 free_pgtable_page(level_pte);
968 }
969next:
970 pfn += level_size(level);
971 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
972}
973
ba395927
KA		 974/* free page table pages. last level pte should already be cleared */
975static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 976 unsigned long start_pfn,
977 unsigned long last_pfn)
ba395927 978{
6660c63a 979 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927 980
6660c63a
DW		 981 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
982 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 983 BUG_ON(start_pfn > last_pfn);
ba395927 984
f3a0a52f 985 /* We don't need lock here; nobody else touches the iova range */
3269ee0b
AW		 986 dma_pte_free_level(domain, agaw_to_level(domain->agaw),
987 domain->pgd, 0, start_pfn, last_pfn);
6660c63a 988
ba395927 989 /* free pgd */
d794dc9b 990 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 991 free_pgtable_page(domain->pgd);
992 domain->pgd = NULL;
993 }
994}
995
ea8ea460
DW		 996/* When a page at a given level is being unlinked from its parent, we don't
997 need to *modify* it at all. All we need to do is make a list of all the
998 pages which can be freed just as soon as we've flushed the IOTLB and we
999 know the hardware page-walk will no longer touch them.
1000 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1001 be freed. */
1002static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,
1003 int level, struct dma_pte *pte,
1004 struct page *freelist)
1005{
1006 struct page *pg;
1007
1008 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1009 pg->freelist = freelist;
1010 freelist = pg;
1011
1012 if (level == 1)
1013 return freelist;
1014
adeb2590
JL		 1015 pte = page_address(pg);
1016 do {
ea8ea460
DW		 1017 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1018 freelist = dma_pte_list_pagetables(domain, level - 1,
1019 pte, freelist);
adeb2590
JL		 1020 pte++;
1021 } while (!first_pte_in_page(pte));
ea8ea460
DW		 1022
1023 return freelist;
1024}
1025
1026static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,
1027 struct dma_pte *pte, unsigned long pfn,
1028 unsigned long start_pfn,
1029 unsigned long last_pfn,
1030 struct page *freelist)
1031{
1032 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1033
1034 pfn = max(start_pfn, pfn);
1035 pte = &pte[pfn_level_offset(pfn, level)];
1036
1037 do {
1038 unsigned long level_pfn;
1039
1040 if (!dma_pte_present(pte))
1041 goto next;
1042
1043 level_pfn = pfn & level_mask(level);
1044
1045 /* If range covers entire pagetable, free it */
1046 if (start_pfn <= level_pfn &&
1047 last_pfn >= level_pfn + level_size(level) - 1) {
1048 /* These suborbinate page tables are going away entirely. Don't
1049 bother to clear them; we're just going to *free* them. */
1050 if (level > 1 && !dma_pte_superpage(pte))
1051 freelist = dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1052
1053 dma_clear_pte(pte);
1054 if (!first_pte)
1055 first_pte = pte;
1056 last_pte = pte;
1057 } else if (level > 1) {
1058 /* Recurse down into a level that isn't *entirely* obsolete */
1059 freelist = dma_pte_clear_level(domain, level - 1,
1060 phys_to_virt(dma_pte_addr(pte)),
1061 level_pfn, start_pfn, last_pfn,
1062 freelist);
1063 }
1064next:
1065 pfn += level_size(level);
1066 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1067
1068 if (first_pte)
1069 domain_flush_cache(domain, first_pte,
1070 (void *)++last_pte - (void *)first_pte);
1071
1072 return freelist;
1073}
1074
1075/* We can't just free the pages because the IOMMU may still be walking
1076 the page tables, and may have cached the intermediate levels. The
1077 pages can only be freed after the IOTLB flush has been done. */
1078struct page *domain_unmap(struct dmar_domain *domain,
1079 unsigned long start_pfn,
1080 unsigned long last_pfn)
1081{
1082 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1083 struct page *freelist = NULL;
1084
1085 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
1086 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
1087 BUG_ON(start_pfn > last_pfn);
1088
1089 /* we don't need lock here; nobody else touches the iova range */
1090 freelist = dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1091 domain->pgd, 0, start_pfn, last_pfn, NULL);
1092
1093 /* free pgd */
1094 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1095 struct page *pgd_page = virt_to_page(domain->pgd);
1096 pgd_page->freelist = freelist;
1097 freelist = pgd_page;
1098
1099 domain->pgd = NULL;
1100 }
1101
1102 return freelist;
1103}
1104
1105void dma_free_pagelist(struct page *freelist)
1106{
1107 struct page *pg;
1108
1109 while ((pg = freelist)) {
1110 freelist = pg->freelist;
1111 free_pgtable_page(page_address(pg));
1112 }
1113}
1114
ba395927
KA		 1115/* iommu handling */
1116static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1117{
1118 struct root_entry *root;
1119 unsigned long flags;
1120
4c923d47 1121 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ba395927
KA		 1122 if (!root)
1123 return -ENOMEM;
1124
5b6985ce 1125 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 1126
1127 spin_lock_irqsave(&iommu->lock, flags);
1128 iommu->root_entry = root;
1129 spin_unlock_irqrestore(&iommu->lock, flags);
1130
1131 return 0;
1132}
1133
ba395927
KA		 1134static void iommu_set_root_entry(struct intel_iommu *iommu)
1135{
1136 void *addr;
c416daa9 1137 u32 sts;
ba395927
KA		 1138 unsigned long flag;
1139
1140 addr = iommu->root_entry;
1141
1f5b3c3f 1142 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1143 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
1144
c416daa9 1145 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1146
1147 /* Make sure hardware complete it */
1148 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1149 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927 1150
1f5b3c3f 1151 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1152}
1153
1154static void iommu_flush_write_buffer(struct intel_iommu *iommu)
1155{
1156 u32 val;
1157 unsigned long flag;
1158
9af88143 1159 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 1160 return;
ba395927 1161
1f5b3c3f 1162 raw_spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 1163 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1164
1165 /* Make sure hardware complete it */
1166 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1167 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927 1168
1f5b3c3f 1169 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1170}
1171
1172/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 1173static void __iommu_flush_context(struct intel_iommu *iommu,
1174 u16 did, u16 source_id, u8 function_mask,
1175 u64 type)
ba395927
KA		 1176{
1177 u64 val = 0;
1178 unsigned long flag;
1179
ba395927
KA		 1180 switch (type) {
1181 case DMA_CCMD_GLOBAL_INVL:
1182 val = DMA_CCMD_GLOBAL_INVL;
1183 break;
1184 case DMA_CCMD_DOMAIN_INVL:
1185 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1186 break;
1187 case DMA_CCMD_DEVICE_INVL:
1188 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1189 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1190 break;
1191 default:
1192 BUG();
1193 }
1194 val |= DMA_CCMD_ICC;
1195
1f5b3c3f 1196 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1197 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1198
1199 /* Make sure hardware complete it */
1200 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1201 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1202
1f5b3c3f 1203 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1204}
1205
ba395927 1206/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 1207static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1208 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 1209{
1210 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1211 u64 val = 0, val_iva = 0;
1212 unsigned long flag;
1213
ba395927
KA		 1214 switch (type) {
1215 case DMA_TLB_GLOBAL_FLUSH:
1216 /* global flush doesn't need set IVA_REG */
1217 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1218 break;
1219 case DMA_TLB_DSI_FLUSH:
1220 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1221 break;
1222 case DMA_TLB_PSI_FLUSH:
1223 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
ea8ea460 1224 /* IH bit is passed in as part of address */
ba395927
KA		 1225 val_iva = size_order | addr;
1226 break;
1227 default:
1228 BUG();
1229 }
1230 /* Note: set drain read/write */
1231#if 0
1232 /*
1233 * This is probably to be super secure.. Looks like we can
1234 * ignore it without any impact.
1235 */
1236 if (cap_read_drain(iommu->cap))
1237 val |= DMA_TLB_READ_DRAIN;
1238#endif
1239 if (cap_write_drain(iommu->cap))
1240 val |= DMA_TLB_WRITE_DRAIN;
1241
1f5b3c3f 1242 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1243 /* Note: Only uses first TLB reg currently */
1244 if (val_iva)
1245 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1246 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1247
1248 /* Make sure hardware complete it */
1249 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1250 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1251
1f5b3c3f 1252 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1253
1254 /* check IOTLB invalidation granularity */
1255 if (DMA_TLB_IAIG(val) == 0)
1256 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1257 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1258 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 1259 (unsigned long long)DMA_TLB_IIRG(type),
1260 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 1261}
1262
64ae892b
DW		 1263static struct device_domain_info *
1264iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,
1265 u8 bus, u8 devfn)
93a23a72
YZ		 1266{
1267 int found = 0;
1268 unsigned long flags;
1269 struct device_domain_info *info;
0bcb3e28 1270 struct pci_dev *pdev;
93a23a72
YZ		 1271
1272 if (!ecap_dev_iotlb_support(iommu->ecap))
1273 return NULL;
1274
1275 if (!iommu->qi)
1276 return NULL;
1277
1278 spin_lock_irqsave(&device_domain_lock, flags);
1279 list_for_each_entry(info, &domain->devices, link)
c3b497c6
JL		 1280 if (info->iommu == iommu && info->bus == bus &&
1281 info->devfn == devfn) {
93a23a72
YZ		 1282 found = 1;
1283 break;
1284 }
1285 spin_unlock_irqrestore(&device_domain_lock, flags);
1286
0bcb3e28 1287 if (!found || !info->dev || !dev_is_pci(info->dev))
93a23a72
YZ		 1288 return NULL;
1289
0bcb3e28
DW		 1290 pdev = to_pci_dev(info->dev);
1291
1292 if (!pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ATS))
93a23a72
YZ		 1293 return NULL;
1294
0bcb3e28 1295 if (!dmar_find_matched_atsr_unit(pdev))
93a23a72
YZ		 1296 return NULL;
1297
93a23a72
YZ		 1298 return info;
1299}
1300
1301static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1302{
0bcb3e28 1303 if (!info || !dev_is_pci(info->dev))
93a23a72
YZ		 1304 return;
1305
0bcb3e28 1306 pci_enable_ats(to_pci_dev(info->dev), VTD_PAGE_SHIFT);
93a23a72
YZ		 1307}
1308
1309static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1310{
0bcb3e28
DW		 1311 if (!info->dev || !dev_is_pci(info->dev) ||
1312 !pci_ats_enabled(to_pci_dev(info->dev)))
93a23a72
YZ		 1313 return;
1314
0bcb3e28 1315 pci_disable_ats(to_pci_dev(info->dev));
93a23a72
YZ		 1316}
1317
1318static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1319 u64 addr, unsigned mask)
1320{
1321 u16 sid, qdep;
1322 unsigned long flags;
1323 struct device_domain_info *info;
1324
1325 spin_lock_irqsave(&device_domain_lock, flags);
1326 list_for_each_entry(info, &domain->devices, link) {
0bcb3e28
DW		 1327 struct pci_dev *pdev;
1328 if (!info->dev || !dev_is_pci(info->dev))
1329 continue;
1330
1331 pdev = to_pci_dev(info->dev);
1332 if (!pci_ats_enabled(pdev))
93a23a72
YZ		 1333 continue;
1334
1335 sid = info->bus << 8 | info->devfn;
0bcb3e28 1336 qdep = pci_ats_queue_depth(pdev);
93a23a72
YZ		 1337 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1338 }
1339 spin_unlock_irqrestore(&device_domain_lock, flags);
1340}
1341
1f0ef2aa 1342static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
ea8ea460 1343 unsigned long pfn, unsigned int pages, int ih, int map)
ba395927 1344{
9dd2fe89 1345 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1346 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1347
ba395927
KA		 1348 BUG_ON(pages == 0);
1349
ea8ea460
DW		 1350 if (ih)
1351 ih = 1 << 6;
ba395927 1352 /*
9dd2fe89
YZ		 1353 * Fallback to domain selective flush if no PSI support or the size is
1354 * too big.
ba395927
KA		 1355 * PSI requires page size to be 2 ^ x, and the base address is naturally
1356 * aligned to the size
1357 */
9dd2fe89
YZ		 1358 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1359 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1360 DMA_TLB_DSI_FLUSH);
9dd2fe89 1361 else
ea8ea460 1362 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
9dd2fe89 1363 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1364
1365 /*
82653633
NA		 1366 * In caching mode, changes of pages from non-present to present require
1367 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1368 */
82653633 1369 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1370 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1371}
1372
f8bab735 1373static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1374{
1375 u32 pmen;
1376 unsigned long flags;
1377
1f5b3c3f 1378 raw_spin_lock_irqsave(&iommu->register_lock, flags);
f8bab735 1379 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1380 pmen &= ~DMA_PMEN_EPM;
1381 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1382
1383 /* wait for the protected region status bit to clear */
1384 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1385 readl, !(pmen & DMA_PMEN_PRS), pmen);
1386
1f5b3c3f 1387 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
f8bab735 1388}
1389
ba395927
KA		 1390static int iommu_enable_translation(struct intel_iommu *iommu)
1391{
1392 u32 sts;
1393 unsigned long flags;
1394
1f5b3c3f 1395 raw_spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1396 iommu->gcmd |= DMA_GCMD_TE;
1397 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1398
1399 /* Make sure hardware complete it */
1400 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1401 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1402
1f5b3c3f 1403 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
ba395927
KA		 1404 return 0;
1405}
1406
1407static int iommu_disable_translation(struct intel_iommu *iommu)
1408{
1409 u32 sts;
1410 unsigned long flag;
1411
1f5b3c3f 1412 raw_spin_lock_irqsave(&iommu->register_lock, flag);
ba395927
KA		 1413 iommu->gcmd &= ~DMA_GCMD_TE;
1414 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1415
1416 /* Make sure hardware complete it */
1417 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1418 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927 1419
1f5b3c3f 1420 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 1421 return 0;
1422}
1423
3460a6d9 1424
ba395927
KA		 1425static int iommu_init_domains(struct intel_iommu *iommu)
1426{
1427 unsigned long ndomains;
1428 unsigned long nlongs;
1429
1430 ndomains = cap_ndoms(iommu->cap);
852bdb04
JL		 1431 pr_debug("IOMMU%d: Number of Domains supported <%ld>\n",
1432 iommu->seq_id, ndomains);
ba395927
KA		 1433 nlongs = BITS_TO_LONGS(ndomains);
1434
94a91b50
DD		 1435 spin_lock_init(&iommu->lock);
1436
ba395927
KA		 1437 /* TBD: there might be 64K domains,
1438 * consider other allocation for future chip
1439 */
1440 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1441 if (!iommu->domain_ids) {
852bdb04
JL		 1442 pr_err("IOMMU%d: allocating domain id array failed\n",
1443 iommu->seq_id);
ba395927
KA		 1444 return -ENOMEM;
1445 }
1446 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1447 GFP_KERNEL);
1448 if (!iommu->domains) {
852bdb04
JL		 1449 pr_err("IOMMU%d: allocating domain array failed\n",
1450 iommu->seq_id);
1451 kfree(iommu->domain_ids);
1452 iommu->domain_ids = NULL;
ba395927
KA		 1453 return -ENOMEM;
1454 }
1455
1456 /*
1457 * if Caching mode is set, then invalid translations are tagged
1458 * with domainid 0. Hence we need to pre-allocate it.
1459 */
1460 if (cap_caching_mode(iommu->cap))
1461 set_bit(0, iommu->domain_ids);
1462 return 0;
1463}
ba395927 1464
a868e6b7 1465static void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1466{
1467 struct dmar_domain *domain;
5ced12af 1468 int i, count;
c7151a8d 1469 unsigned long flags;
ba395927 1470
94a91b50 1471 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1472 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
a4eaa86c
JL		 1473 /*
1474 * Domain id 0 is reserved for invalid translation
1475 * if hardware supports caching mode.
1476 */
1477 if (cap_caching_mode(iommu->cap) && i == 0)
1478 continue;
1479
94a91b50
DD		 1480 domain = iommu->domains[i];
1481 clear_bit(i, iommu->domain_ids);
1482
1483 spin_lock_irqsave(&domain->iommu_lock, flags);
5ced12af
JL		 1484 count = --domain->iommu_count;
1485 spin_unlock_irqrestore(&domain->iommu_lock, flags);
92d03cc8
JL		 1486 if (count == 0)
1487 domain_exit(domain);
5e98c4b1 1488 }
ba395927
KA		 1489 }
1490
1491 if (iommu->gcmd & DMA_GCMD_TE)
1492 iommu_disable_translation(iommu);
1493
ba395927
KA		 1494 kfree(iommu->domains);
1495 kfree(iommu->domain_ids);
a868e6b7
JL		 1496 iommu->domains = NULL;
1497 iommu->domain_ids = NULL;
ba395927 1498
d9630fe9
WH		 1499 g_iommus[iommu->seq_id] = NULL;
1500
ba395927
KA		 1501 /* free context mapping */
1502 free_context_table(iommu);
ba395927
KA		 1503}
1504
ab8dfe25 1505static struct dmar_domain *alloc_domain(int flags)
ba395927 1506{
92d03cc8
JL		 1507 /* domain id for virtual machine, it won't be set in context */
1508 static atomic_t vm_domid = ATOMIC_INIT(0);
ba395927 1509 struct dmar_domain *domain;
ba395927
KA		 1510
1511 domain = alloc_domain_mem();
1512 if (!domain)
1513 return NULL;
1514
ab8dfe25 1515 memset(domain, 0, sizeof(*domain));
4c923d47 1516 domain->nid = -1;
ab8dfe25 1517 domain->flags = flags;
92d03cc8
JL		 1518 spin_lock_init(&domain->iommu_lock);
1519 INIT_LIST_HEAD(&domain->devices);
ab8dfe25 1520 if (flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
92d03cc8 1521 domain->id = atomic_inc_return(&vm_domid);
2c2e2c38
FY		 1522
1523 return domain;
1524}
1525
fb170fb4
JL		 1526static int __iommu_attach_domain(struct dmar_domain *domain,
1527 struct intel_iommu *iommu)
2c2e2c38
FY		 1528{
1529 int num;
1530 unsigned long ndomains;
2c2e2c38 1531
ba395927 1532 ndomains = cap_ndoms(iommu->cap);
ba395927 1533 num = find_first_zero_bit(iommu->domain_ids, ndomains);
fb170fb4
JL		 1534 if (num < ndomains) {
1535 set_bit(num, iommu->domain_ids);
1536 iommu->domains[num] = domain;
1537 } else {
1538 num = -ENOSPC;
ba395927
KA		 1539 }
1540
fb170fb4
JL		 1541 return num;
1542}
1543
1544static int iommu_attach_domain(struct dmar_domain *domain,
1545 struct intel_iommu *iommu)
1546{
1547 int num;
1548 unsigned long flags;
1549
1550 spin_lock_irqsave(&iommu->lock, flags);
1551 num = __iommu_attach_domain(domain, iommu);
44bde614 1552 spin_unlock_irqrestore(&iommu->lock, flags);
fb170fb4
JL		 1553 if (num < 0)
1554 pr_err("IOMMU: no free domain ids\n");
ba395927 1555
fb170fb4 1556 return num;
ba395927
KA		 1557}
1558
44bde614
JL		 1559static int iommu_attach_vm_domain(struct dmar_domain *domain,
1560 struct intel_iommu *iommu)
1561{
1562 int num;
1563 unsigned long ndomains;
1564
1565 ndomains = cap_ndoms(iommu->cap);
1566 for_each_set_bit(num, iommu->domain_ids, ndomains)
1567 if (iommu->domains[num] == domain)
1568 return num;
1569
1570 return __iommu_attach_domain(domain, iommu);
1571}
1572
2c2e2c38
FY		 1573static void iommu_detach_domain(struct dmar_domain *domain,
1574 struct intel_iommu *iommu)
ba395927
KA		 1575{
1576 unsigned long flags;
2c2e2c38 1577 int num, ndomains;
ba395927 1578
8c11e798 1579 spin_lock_irqsave(&iommu->lock, flags);
fb170fb4
JL		 1580 if (domain_type_is_vm_or_si(domain)) {
1581 ndomains = cap_ndoms(iommu->cap);
1582 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1583 if (iommu->domains[num] == domain) {
1584 clear_bit(num, iommu->domain_ids);
1585 iommu->domains[num] = NULL;
1586 break;
1587 }
2c2e2c38 1588 }
fb170fb4
JL		 1589 } else {
1590 clear_bit(domain->id, iommu->domain_ids);
1591 iommu->domains[domain->id] = NULL;
2c2e2c38 1592 }
8c11e798 1593 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1594}
1595
fb170fb4
JL		 1596static void domain_attach_iommu(struct dmar_domain *domain,
1597 struct intel_iommu *iommu)
1598{
1599 unsigned long flags;
1600
1601 spin_lock_irqsave(&domain->iommu_lock, flags);
1602 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1603 domain->iommu_count++;
1604 if (domain->iommu_count == 1)
1605 domain->nid = iommu->node;
1606 domain_update_iommu_cap(domain);
1607 }
1608 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1609}
1610
1611static int domain_detach_iommu(struct dmar_domain *domain,
1612 struct intel_iommu *iommu)
1613{
1614 unsigned long flags;
1615 int count = INT_MAX;
1616
1617 spin_lock_irqsave(&domain->iommu_lock, flags);
1618 if (test_and_clear_bit(iommu->seq_id, domain->iommu_bmp)) {
1619 count = --domain->iommu_count;
1620 domain_update_iommu_cap(domain);
1621 }
1622 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1623
1624 return count;
1625}
1626
ba395927 1627static struct iova_domain reserved_iova_list;
8a443df4 1628static struct lock_class_key reserved_rbtree_key;
ba395927 1629
51a63e67 1630static int dmar_init_reserved_ranges(void)
ba395927
KA		 1631{
1632 struct pci_dev *pdev = NULL;
1633 struct iova *iova;
1634 int i;
ba395927 1635
f661197e 1636 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1637
8a443df4
MG		 1638 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1639 &reserved_rbtree_key);
1640
ba395927
KA		 1641 /* IOAPIC ranges shouldn't be accessed by DMA */
1642 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1643 IOVA_PFN(IOAPIC_RANGE_END));
51a63e67 1644 if (!iova) {
ba395927 1645 printk(KERN_ERR "Reserve IOAPIC range failed\n");
51a63e67
JC		 1646 return -ENODEV;
1647 }
ba395927
KA		 1648
1649 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1650 for_each_pci_dev(pdev) {
1651 struct resource *r;
1652
1653 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1654 r = &pdev->resource[i];
1655 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1656 continue;
1a4a4551
DW		 1657 iova = reserve_iova(&reserved_iova_list,
1658 IOVA_PFN(r->start),
1659 IOVA_PFN(r->end));
51a63e67 1660 if (!iova) {
ba395927 1661 printk(KERN_ERR "Reserve iova failed\n");
51a63e67
JC		 1662 return -ENODEV;
1663 }
ba395927
KA		 1664 }
1665 }
51a63e67 1666 return 0;
ba395927
KA		 1667}
1668
1669static void domain_reserve_special_ranges(struct dmar_domain *domain)
1670{
1671 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1672}
1673
1674static inline int guestwidth_to_adjustwidth(int gaw)
1675{
1676 int agaw;
1677 int r = (gaw - 12) % 9;
1678
1679 if (r == 0)
1680 agaw = gaw;
1681 else
1682 agaw = gaw + 9 - r;
1683 if (agaw > 64)
1684 agaw = 64;
1685 return agaw;
1686}
1687
1688static int domain_init(struct dmar_domain *domain, int guest_width)
1689{
1690 struct intel_iommu *iommu;
1691 int adjust_width, agaw;
1692 unsigned long sagaw;
1693
f661197e 1694 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
ba395927
KA		 1695 domain_reserve_special_ranges(domain);
1696
1697 /* calculate AGAW */
8c11e798 1698 iommu = domain_get_iommu(domain);
ba395927
KA		 1699 if (guest_width > cap_mgaw(iommu->cap))
1700 guest_width = cap_mgaw(iommu->cap);
1701 domain->gaw = guest_width;
1702 adjust_width = guestwidth_to_adjustwidth(guest_width);
1703 agaw = width_to_agaw(adjust_width);
1704 sagaw = cap_sagaw(iommu->cap);
1705 if (!test_bit(agaw, &sagaw)) {
1706 /* hardware doesn't support it, choose a bigger one */
1707 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1708 agaw = find_next_bit(&sagaw, 5, agaw);
1709 if (agaw >= 5)
1710 return -ENODEV;
1711 }
1712 domain->agaw = agaw;
ba395927 1713
8e604097
WH		 1714 if (ecap_coherent(iommu->ecap))
1715 domain->iommu_coherency = 1;
1716 else
1717 domain->iommu_coherency = 0;
1718
58c610bd
SY		 1719 if (ecap_sc_support(iommu->ecap))
1720 domain->iommu_snooping = 1;
1721 else
1722 domain->iommu_snooping = 0;
1723
214e39aa
DW		 1724 if (intel_iommu_superpage)
1725 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1726 else
1727 domain->iommu_superpage = 0;
1728
4c923d47 1729 domain->nid = iommu->node;
c7151a8d 1730
ba395927 1731 /* always allocate the top pgd */
4c923d47 1732 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1733 if (!domain->pgd)
1734 return -ENOMEM;
5b6985ce 1735 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1736 return 0;
1737}
1738
1739static void domain_exit(struct dmar_domain *domain)
1740{
2c2e2c38
FY		 1741 struct dmar_drhd_unit *drhd;
1742 struct intel_iommu *iommu;
ea8ea460 1743 struct page *freelist = NULL;
ba395927
KA		 1744
1745 /* Domain 0 is reserved, so dont process it */
1746 if (!domain)
1747 return;
1748
7b668357
AW		 1749 /* Flush any lazy unmaps that may reference this domain */
1750 if (!intel_iommu_strict)
1751 flush_unmaps_timeout(0);
1752
92d03cc8 1753 /* remove associated devices */
ba395927 1754 domain_remove_dev_info(domain);
92d03cc8 1755
ba395927
KA		 1756 /* destroy iovas */
1757 put_iova_domain(&domain->iovad);
ba395927 1758
ea8ea460 1759 freelist = domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1760
92d03cc8 1761 /* clear attached or cached domains */
0e242612 1762 rcu_read_lock();
2c2e2c38 1763 for_each_active_iommu(iommu, drhd)
fb170fb4 1764 iommu_detach_domain(domain, iommu);
0e242612 1765 rcu_read_unlock();
2c2e2c38 1766
ea8ea460
DW		 1767 dma_free_pagelist(freelist);
1768
ba395927
KA		 1769 free_domain_mem(domain);
1770}
1771
64ae892b
DW		 1772static int domain_context_mapping_one(struct dmar_domain *domain,
1773 struct intel_iommu *iommu,
1774 u8 bus, u8 devfn, int translation)
ba395927
KA		 1775{
1776 struct context_entry *context;
ba395927 1777 unsigned long flags;
ea6606b0 1778 struct dma_pte *pgd;
ea6606b0
WH		 1779 int id;
1780 int agaw;
93a23a72 1781 struct device_domain_info *info = NULL;
ba395927
KA		 1782
1783 pr_debug("Set context mapping for %02x:%02x.%d\n",
1784 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1785
ba395927 1786 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1787 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1788 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1789
ba395927
KA		 1790 context = device_to_context_entry(iommu, bus, devfn);
1791 if (!context)
1792 return -ENOMEM;
1793 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1794 if (context_present(context)) {
ba395927
KA		 1795 spin_unlock_irqrestore(&iommu->lock, flags);
1796 return 0;
1797 }
1798
ea6606b0
WH		 1799 id = domain->id;
1800 pgd = domain->pgd;
1801
ab8dfe25 1802 if (domain_type_is_vm_or_si(domain)) {
44bde614
JL		 1803 if (domain_type_is_vm(domain)) {
1804 id = iommu_attach_vm_domain(domain, iommu);
fb170fb4 1805 if (id < 0) {
ea6606b0 1806 spin_unlock_irqrestore(&iommu->lock, flags);
fb170fb4 1807 pr_err("IOMMU: no free domain ids\n");
ea6606b0
WH		 1808 return -EFAULT;
1809 }
ea6606b0
WH		 1810 }
1811
1812 /* Skip top levels of page tables for
1813 * iommu which has less agaw than default.
1672af11 1814 * Unnecessary for PT mode.
ea6606b0 1815 */
1672af11
CW		 1816 if (translation != CONTEXT_TT_PASS_THROUGH) {
1817 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1818 pgd = phys_to_virt(dma_pte_addr(pgd));
1819 if (!dma_pte_present(pgd)) {
1820 spin_unlock_irqrestore(&iommu->lock, flags);
1821 return -ENOMEM;
1822 }
ea6606b0
WH		 1823 }
1824 }
1825 }
1826
1827 context_set_domain_id(context, id);
4ed0d3e6 1828
93a23a72 1829 if (translation != CONTEXT_TT_PASS_THROUGH) {
64ae892b 1830 info = iommu_support_dev_iotlb(domain, iommu, bus, devfn);
93a23a72
YZ		 1831 translation = info ? CONTEXT_TT_DEV_IOTLB :
1832 CONTEXT_TT_MULTI_LEVEL;
1833 }
4ed0d3e6
FY		 1834 /*
1835 * In pass through mode, AW must be programmed to indicate the largest
1836 * AGAW value supported by hardware. And ASR is ignored by hardware.
1837 */
93a23a72 1838 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1839 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1840 else {
1841 context_set_address_root(context, virt_to_phys(pgd));
1842 context_set_address_width(context, iommu->agaw);
1843 }
4ed0d3e6
FY		 1844
1845 context_set_translation_type(context, translation);
c07e7d21
MM		 1846 context_set_fault_enable(context);
1847 context_set_present(context);
5331fe6f 1848 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1849
4c25a2c1
DW		 1850 /*
1851 * It's a non-present to present mapping. If hardware doesn't cache
1852 * non-present entry we only need to flush the write-buffer. If the
1853 * _does_ cache non-present entries, then it does so in the special
1854 * domain #0, which we have to flush:
1855 */
1856 if (cap_caching_mode(iommu->cap)) {
1857 iommu->flush.flush_context(iommu, 0,
1858 (((u16)bus) << 8) | devfn,
1859 DMA_CCMD_MASK_NOBIT,
1860 DMA_CCMD_DEVICE_INVL);
18fd779a 1861 iommu->flush.flush_iotlb(iommu, id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1862 } else {
ba395927 1863 iommu_flush_write_buffer(iommu);
4c25a2c1 1864 }
93a23a72 1865 iommu_enable_dev_iotlb(info);
ba395927 1866 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d 1867
fb170fb4
JL		 1868 domain_attach_iommu(domain, iommu);
1869
ba395927
KA		 1870 return 0;
1871}
1872
579305f7
AW		 1873struct domain_context_mapping_data {
1874 struct dmar_domain *domain;
1875 struct intel_iommu *iommu;
1876 int translation;
1877};
1878
1879static int domain_context_mapping_cb(struct pci_dev *pdev,
1880 u16 alias, void *opaque)
1881{
1882 struct domain_context_mapping_data *data = opaque;
1883
1884 return domain_context_mapping_one(data->domain, data->iommu,
1885 PCI_BUS_NUM(alias), alias & 0xff,
1886 data->translation);
1887}
1888
ba395927 1889static int
e1f167f3
DW		 1890domain_context_mapping(struct dmar_domain *domain, struct device *dev,
1891 int translation)
ba395927 1892{
64ae892b 1893 struct intel_iommu *iommu;
156baca8 1894 u8 bus, devfn;
579305f7 1895 struct domain_context_mapping_data data;
64ae892b 1896
e1f167f3 1897 iommu = device_to_iommu(dev, &bus, &devfn);
64ae892b
DW		 1898 if (!iommu)
1899 return -ENODEV;
ba395927 1900
579305f7
AW		 1901 if (!dev_is_pci(dev))
1902 return domain_context_mapping_one(domain, iommu, bus, devfn,
4ed0d3e6 1903 translation);
579305f7
AW		 1904
1905 data.domain = domain;
1906 data.iommu = iommu;
1907 data.translation = translation;
1908
1909 return pci_for_each_dma_alias(to_pci_dev(dev),
1910 &domain_context_mapping_cb, &data);
1911}
1912
1913static int domain_context_mapped_cb(struct pci_dev *pdev,
1914 u16 alias, void *opaque)
1915{
1916 struct intel_iommu *iommu = opaque;
1917
1918 return !device_context_mapped(iommu, PCI_BUS_NUM(alias), alias & 0xff);
ba395927
KA		 1919}
1920
e1f167f3 1921static int domain_context_mapped(struct device *dev)
ba395927 1922{
5331fe6f 1923 struct intel_iommu *iommu;
156baca8 1924 u8 bus, devfn;
5331fe6f 1925
e1f167f3 1926 iommu = device_to_iommu(dev, &bus, &devfn);
5331fe6f
WH		 1927 if (!iommu)
1928 return -ENODEV;
ba395927 1929
579305f7
AW		 1930 if (!dev_is_pci(dev))
1931 return device_context_mapped(iommu, bus, devfn);
e1f167f3 1932
579305f7
AW		 1933 return !pci_for_each_dma_alias(to_pci_dev(dev),
1934 domain_context_mapped_cb, iommu);
ba395927
KA		 1935}
1936
f532959b
FY		 1937/* Returns a number of VTD pages, but aligned to MM page size */
1938static inline unsigned long aligned_nrpages(unsigned long host_addr,
1939 size_t size)
1940{
1941 host_addr &= ~PAGE_MASK;
1942 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1943}
1944
6dd9a7c7
YS		 1945/* Return largest possible superpage level for a given mapping */
1946static inline int hardware_largepage_caps(struct dmar_domain *domain,
1947 unsigned long iov_pfn,
1948 unsigned long phy_pfn,
1949 unsigned long pages)
1950{
1951 int support, level = 1;
1952 unsigned long pfnmerge;
1953
1954 support = domain->iommu_superpage;
1955
1956 /* To use a large page, the virtual *and* physical addresses
1957 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1958 of them will mean we have to use smaller pages. So just
1959 merge them and check both at once. */
1960 pfnmerge = iov_pfn | phy_pfn;
1961
1962 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1963 pages >>= VTD_STRIDE_SHIFT;
1964 if (!pages)
1965 break;
1966 pfnmerge >>= VTD_STRIDE_SHIFT;
1967 level++;
1968 support--;
1969 }
1970 return level;
1971}
1972
9051aa02
DW		 1973static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1974 struct scatterlist *sg, unsigned long phys_pfn,
1975 unsigned long nr_pages, int prot)
e1605495
DW		 1976{
1977 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1978 phys_addr_t uninitialized_var(pteval);
e1605495 1979 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
9051aa02 1980 unsigned long sg_res;
6dd9a7c7
YS		 1981 unsigned int largepage_lvl = 0;
1982 unsigned long lvl_pages = 0;
e1605495
DW		 1983
1984 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1985
1986 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1987 return -EINVAL;
1988
1989 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1990
9051aa02
DW		 1991 if (sg)
1992 sg_res = 0;
1993 else {
1994 sg_res = nr_pages + 1;
1995 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1996 }
1997
6dd9a7c7 1998 while (nr_pages > 0) {
c85994e4
DW		 1999 uint64_t tmp;
2000
e1605495 2001 if (!sg_res) {
f532959b 2002 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 2003 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
2004 sg->dma_length = sg->length;
2005 pteval = page_to_phys(sg_page(sg)) | prot;
6dd9a7c7 2006 phys_pfn = pteval >> VTD_PAGE_SHIFT;
e1605495 2007 }
6dd9a7c7 2008
e1605495 2009 if (!pte) {
6dd9a7c7
YS		 2010 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
2011
5cf0a76f 2012 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
e1605495
DW		 2013 if (!pte)
2014 return -ENOMEM;
6dd9a7c7 2015 /* It is large page*/
6491d4d0 2016 if (largepage_lvl > 1) {
6dd9a7c7 2017 pteval |= DMA_PTE_LARGE_PAGE;
6491d4d0
WD		 2018 /* Ensure that old small page tables are removed to make room
2019 for superpage, if they exist. */
2020 dma_pte_clear_range(domain, iov_pfn,
2021 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
2022 dma_pte_free_pagetable(domain, iov_pfn,
2023 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
2024 } else {
6dd9a7c7 2025 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
6491d4d0 2026 }
6dd9a7c7 2027
e1605495
DW		 2028 }
2029 /* We don't need lock here, nobody else
2030 * touches the iova range
2031 */
7766a3fb 2032 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 2033 if (tmp) {
1bf20f0d 2034 static int dumps = 5;
c85994e4
DW		 2035 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2036 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 2037 if (dumps) {
2038 dumps--;
2039 debug_dma_dump_mappings(NULL);
2040 }
2041 WARN_ON(1);
2042 }
6dd9a7c7
YS		 2043
2044 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2045
2046 BUG_ON(nr_pages < lvl_pages);
2047 BUG_ON(sg_res < lvl_pages);
2048
2049 nr_pages -= lvl_pages;
2050 iov_pfn += lvl_pages;
2051 phys_pfn += lvl_pages;
2052 pteval += lvl_pages * VTD_PAGE_SIZE;
2053 sg_res -= lvl_pages;
2054
2055 /* If the next PTE would be the first in a new page, then we
2056 need to flush the cache on the entries we've just written.
2057 And then we'll need to recalculate 'pte', so clear it and
2058 let it get set again in the if (!pte) block above.
2059
2060 If we're done (!nr_pages) we need to flush the cache too.
2061
2062 Also if we've been setting superpages, we may need to
2063 recalculate 'pte' and switch back to smaller pages for the
2064 end of the mapping, if the trailing size is not enough to
2065 use another superpage (i.e. sg_res < lvl_pages). */
e1605495 2066 pte++;
6dd9a7c7
YS		 2067 if (!nr_pages || first_pte_in_page(pte) ||
2068 (largepage_lvl > 1 && sg_res < lvl_pages)) {
e1605495
DW		 2069 domain_flush_cache(domain, first_pte,
2070 (void *)pte - (void *)first_pte);
2071 pte = NULL;
2072 }
6dd9a7c7
YS		 2073
2074 if (!sg_res && nr_pages)
e1605495
DW		 2075 sg = sg_next(sg);
2076 }
2077 return 0;
2078}
2079
9051aa02
DW		 2080static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2081 struct scatterlist *sg, unsigned long nr_pages,
2082 int prot)
ba395927 2083{
9051aa02
DW		 2084 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
2085}
6f6a00e4 2086
9051aa02
DW		 2087static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2088 unsigned long phys_pfn, unsigned long nr_pages,
2089 int prot)
2090{
2091 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 2092}
2093
c7151a8d 2094static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 2095{
c7151a8d
WH		 2096 if (!iommu)
2097 return;
8c11e798
WH		 2098
2099 clear_context_table(iommu, bus, devfn);
2100 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 2101 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2102 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 2103}
2104
109b9b04
DW		 2105static inline void unlink_domain_info(struct device_domain_info *info)
2106{
2107 assert_spin_locked(&device_domain_lock);
2108 list_del(&info->link);
2109 list_del(&info->global);
2110 if (info->dev)
0bcb3e28 2111 info->dev->archdata.iommu = NULL;
109b9b04
DW		 2112}
2113
ba395927
KA		 2114static void domain_remove_dev_info(struct dmar_domain *domain)
2115{
3a74ca01 2116 struct device_domain_info *info, *tmp;
fb170fb4 2117 unsigned long flags;
ba395927
KA		 2118
2119 spin_lock_irqsave(&device_domain_lock, flags);
3a74ca01 2120 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
109b9b04 2121 unlink_domain_info(info);
ba395927
KA		 2122 spin_unlock_irqrestore(&device_domain_lock, flags);
2123
93a23a72 2124 iommu_disable_dev_iotlb(info);
7c7faa11 2125 iommu_detach_dev(info->iommu, info->bus, info->devfn);
ba395927 2126
ab8dfe25 2127 if (domain_type_is_vm(domain)) {
7c7faa11 2128 iommu_detach_dependent_devices(info->iommu, info->dev);
fb170fb4 2129 domain_detach_iommu(domain, info->iommu);
92d03cc8
JL		 2130 }
2131
2132 free_devinfo_mem(info);
ba395927
KA		 2133 spin_lock_irqsave(&device_domain_lock, flags);
2134 }
2135 spin_unlock_irqrestore(&device_domain_lock, flags);
2136}
2137
2138/*
2139 * find_domain
1525a29a 2140 * Note: we use struct device->archdata.iommu stores the info
ba395927 2141 */
1525a29a 2142static struct dmar_domain *find_domain(struct device *dev)
ba395927
KA		 2143{
2144 struct device_domain_info *info;
2145
2146 /* No lock here, assumes no domain exit in normal case */
1525a29a 2147 info = dev->archdata.iommu;
ba395927
KA		 2148 if (info)
2149 return info->domain;
2150 return NULL;
2151}
2152
5a8f40e8 2153static inline struct device_domain_info *
745f2586
JL		 2154dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
2155{
2156 struct device_domain_info *info;
2157
2158 list_for_each_entry(info, &device_domain_list, global)
41e80dca 2159 if (info->iommu->segment == segment && info->bus == bus &&
745f2586 2160 info->devfn == devfn)
5a8f40e8 2161 return info;
745f2586
JL		 2162
2163 return NULL;
2164}
2165
5a8f40e8 2166static struct dmar_domain *dmar_insert_dev_info(struct intel_iommu *iommu,
41e80dca 2167 int bus, int devfn,
b718cd3d
DW		 2168 struct device *dev,
2169 struct dmar_domain *domain)
745f2586 2170{
5a8f40e8 2171 struct dmar_domain *found = NULL;
745f2586
JL		 2172 struct device_domain_info *info;
2173 unsigned long flags;
2174
2175 info = alloc_devinfo_mem();
2176 if (!info)
b718cd3d 2177 return NULL;
745f2586 2178
745f2586
JL		 2179 info->bus = bus;
2180 info->devfn = devfn;
2181 info->dev = dev;
2182 info->domain = domain;
5a8f40e8 2183 info->iommu = iommu;
745f2586
JL		 2184
2185 spin_lock_irqsave(&device_domain_lock, flags);
2186 if (dev)
0bcb3e28 2187 found = find_domain(dev);
5a8f40e8
DW		 2188 else {
2189 struct device_domain_info *info2;
41e80dca 2190 info2 = dmar_search_domain_by_dev_info(iommu->segment, bus, devfn);
5a8f40e8
DW		 2191 if (info2)
2192 found = info2->domain;
2193 }
745f2586
JL		 2194 if (found) {
2195 spin_unlock_irqrestore(&device_domain_lock, flags);
2196 free_devinfo_mem(info);
b718cd3d
DW		 2197 /* Caller must free the original domain */
2198 return found;
745f2586
JL		 2199 }
2200
b718cd3d
DW		 2201 list_add(&info->link, &domain->devices);
2202 list_add(&info->global, &device_domain_list);
2203 if (dev)
2204 dev->archdata.iommu = info;
2205 spin_unlock_irqrestore(&device_domain_lock, flags);
2206
2207 return domain;
745f2586
JL		 2208}
2209
579305f7
AW		 2210static int get_last_alias(struct pci_dev *pdev, u16 alias, void *opaque)
2211{
2212 *(u16 *)opaque = alias;
2213 return 0;
2214}
2215
ba395927 2216/* domain is initialized */
146922ec 2217static struct dmar_domain *get_domain_for_dev(struct device *dev, int gaw)
ba395927 2218{
579305f7
AW		 2219 struct dmar_domain *domain, *tmp;
2220 struct intel_iommu *iommu;
5a8f40e8 2221 struct device_domain_info *info;
579305f7 2222 u16 dma_alias;
ba395927 2223 unsigned long flags;
aa4d066a 2224 u8 bus, devfn;
ba395927 2225
146922ec 2226 domain = find_domain(dev);
ba395927
KA		 2227 if (domain)
2228 return domain;
2229
579305f7
AW		 2230 iommu = device_to_iommu(dev, &bus, &devfn);
2231 if (!iommu)
2232 return NULL;
2233
146922ec
DW		 2234 if (dev_is_pci(dev)) {
2235 struct pci_dev *pdev = to_pci_dev(dev);
276dbf99 2236
579305f7
AW		 2237 pci_for_each_dma_alias(pdev, get_last_alias, &dma_alias);
2238
2239 spin_lock_irqsave(&device_domain_lock, flags);
2240 info = dmar_search_domain_by_dev_info(pci_domain_nr(pdev->bus),
2241 PCI_BUS_NUM(dma_alias),
2242 dma_alias & 0xff);
2243 if (info) {
2244 iommu = info->iommu;
2245 domain = info->domain;
5a8f40e8 2246 }
579305f7 2247 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 2248
579305f7
AW		 2249 /* DMA alias already has a domain, uses it */
2250 if (info)
2251 goto found_domain;
2252 }
ba395927 2253
146922ec 2254 /* Allocate and initialize new domain for the device */
ab8dfe25 2255 domain = alloc_domain(0);
745f2586 2256 if (!domain)
579305f7 2257 return NULL;
44bde614
JL		 2258 domain->id = iommu_attach_domain(domain, iommu);
2259 if (domain->id < 0) {
2fe9723d 2260 free_domain_mem(domain);
579305f7 2261 return NULL;
2c2e2c38 2262 }
fb170fb4 2263 domain_attach_iommu(domain, iommu);
579305f7
AW		 2264 if (domain_init(domain, gaw)) {
2265 domain_exit(domain);
2266 return NULL;
2c2e2c38 2267 }
ba395927 2268
579305f7
AW		 2269 /* register PCI DMA alias device */
2270 if (dev_is_pci(dev)) {
2271 tmp = dmar_insert_dev_info(iommu, PCI_BUS_NUM(dma_alias),
2272 dma_alias & 0xff, NULL, domain);
2273
2274 if (!tmp || tmp != domain) {
2275 domain_exit(domain);
2276 domain = tmp;
2277 }
2278
b718cd3d 2279 if (!domain)
579305f7 2280 return NULL;
ba395927
KA		 2281 }
2282
2283found_domain:
579305f7
AW		 2284 tmp = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
2285
2286 if (!tmp || tmp != domain) {
2287 domain_exit(domain);
2288 domain = tmp;
2289 }
b718cd3d
DW		 2290
2291 return domain;
ba395927
KA		 2292}
2293
2c2e2c38 2294static int iommu_identity_mapping;
e0fc7e0b
DW		 2295#define IDENTMAP_ALL 1
2296#define IDENTMAP_GFX 2
2297#define IDENTMAP_AZALIA 4
2c2e2c38 2298
b213203e
DW		 2299static int iommu_domain_identity_map(struct dmar_domain *domain,
2300 unsigned long long start,
2301 unsigned long long end)
ba395927 2302{
c5395d5c
DW		 2303 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2304 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2305
2306 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2307 dma_to_mm_pfn(last_vpfn))) {
ba395927 2308 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 2309 return -ENOMEM;
ba395927
KA		 2310 }
2311
c5395d5c
DW		 2312 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2313 start, end, domain->id);
ba395927
KA		 2314 /*
2315 * RMRR range might have overlap with physical memory range,
2316 * clear it first
2317 */
c5395d5c 2318 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 2319
c5395d5c
DW		 2320 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2321 last_vpfn - first_vpfn + 1,
61df7443 2322 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 2323}
2324
0b9d9753 2325static int iommu_prepare_identity_map(struct device *dev,
b213203e
DW		 2326 unsigned long long start,
2327 unsigned long long end)
2328{
2329 struct dmar_domain *domain;
2330 int ret;
2331
0b9d9753 2332 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 2333 if (!domain)
2334 return -ENOMEM;
2335
19943b0e
DW		 2336 /* For _hardware_ passthrough, don't bother. But for software
2337 passthrough, we do it anyway -- it may indicate a memory
2338 range which is reserved in E820, so which didn't get set
2339 up to start with in si_domain */
2340 if (domain == si_domain && hw_pass_through) {
2341 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
0b9d9753 2342 dev_name(dev), start, end);
19943b0e
DW		 2343 return 0;
2344 }
2345
2346 printk(KERN_INFO
2347 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
0b9d9753 2348 dev_name(dev), start, end);
2ff729f5 2349
5595b528
DW		 2350 if (end < start) {
2351 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2352 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2353 dmi_get_system_info(DMI_BIOS_VENDOR),
2354 dmi_get_system_info(DMI_BIOS_VERSION),
2355 dmi_get_system_info(DMI_PRODUCT_VERSION));
2356 ret = -EIO;
2357 goto error;
2358 }
2359
2ff729f5
DW		 2360 if (end >> agaw_to_width(domain->agaw)) {
2361 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2362 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2363 agaw_to_width(domain->agaw),
2364 dmi_get_system_info(DMI_BIOS_VENDOR),
2365 dmi_get_system_info(DMI_BIOS_VERSION),
2366 dmi_get_system_info(DMI_PRODUCT_VERSION));
2367 ret = -EIO;
2368 goto error;
2369 }
19943b0e 2370
b213203e 2371 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 2372 if (ret)
2373 goto error;
2374
2375 /* context entry init */
0b9d9753 2376 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2377 if (ret)
2378 goto error;
2379
2380 return 0;
2381
2382 error:
ba395927
KA		 2383 domain_exit(domain);
2384 return ret;
ba395927
KA		 2385}
2386
2387static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
0b9d9753 2388 struct device *dev)
ba395927 2389{
0b9d9753 2390 if (dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927 2391 return 0;
0b9d9753
DW		 2392 return iommu_prepare_identity_map(dev, rmrr->base_address,
2393 rmrr->end_address);
ba395927
KA		 2394}
2395
d3f13810 2396#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
49a0429e
KA		 2397static inline void iommu_prepare_isa(void)
2398{
2399 struct pci_dev *pdev;
2400 int ret;
2401
2402 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2403 if (!pdev)
2404 return;
2405
c7ab48d2 2406 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
0b9d9753 2407 ret = iommu_prepare_identity_map(&pdev->dev, 0, 16*1024*1024 - 1);
49a0429e
KA		 2408
2409 if (ret)
c7ab48d2
DW		 2410 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2411 "floppy might not work\n");
49a0429e 2412
9b27e82d 2413 pci_dev_put(pdev);
49a0429e
KA		 2414}
2415#else
2416static inline void iommu_prepare_isa(void)
2417{
2418 return;
2419}
d3f13810 2420#endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
49a0429e 2421
2c2e2c38 2422static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2 2423
071e1374 2424static int __init si_domain_init(int hw)
2c2e2c38
FY		 2425{
2426 struct dmar_drhd_unit *drhd;
2427 struct intel_iommu *iommu;
c7ab48d2 2428 int nid, ret = 0;
44bde614 2429 bool first = true;
2c2e2c38 2430
ab8dfe25 2431 si_domain = alloc_domain(DOMAIN_FLAG_STATIC_IDENTITY);
2c2e2c38
FY		 2432 if (!si_domain)
2433 return -EFAULT;
2434
2c2e2c38
FY		 2435 for_each_active_iommu(iommu, drhd) {
2436 ret = iommu_attach_domain(si_domain, iommu);
fb170fb4 2437 if (ret < 0) {
2c2e2c38
FY		 2438 domain_exit(si_domain);
2439 return -EFAULT;
44bde614
JL		 2440 } else if (first) {
2441 si_domain->id = ret;
2442 first = false;
2443 } else if (si_domain->id != ret) {
2444 domain_exit(si_domain);
2445 return -EFAULT;
2c2e2c38 2446 }
fb170fb4 2447 domain_attach_iommu(si_domain, iommu);
2c2e2c38
FY		 2448 }
2449
2450 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2451 domain_exit(si_domain);
2452 return -EFAULT;
2453 }
2454
9544c003
JL		 2455 pr_debug("IOMMU: identity mapping domain is domain %d\n",
2456 si_domain->id);
2c2e2c38 2457
19943b0e
DW		 2458 if (hw)
2459 return 0;
2460
c7ab48d2 2461 for_each_online_node(nid) {
5dfe8660
TH		 2462 unsigned long start_pfn, end_pfn;
2463 int i;
2464
2465 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2466 ret = iommu_domain_identity_map(si_domain,
2467 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2468 if (ret)
2469 return ret;
2470 }
c7ab48d2
DW		 2471 }
2472
2c2e2c38
FY		 2473 return 0;
2474}
2475
9b226624 2476static int identity_mapping(struct device *dev)
2c2e2c38
FY		 2477{
2478 struct device_domain_info *info;
2479
2480 if (likely(!iommu_identity_mapping))
2481 return 0;
2482
9b226624 2483 info = dev->archdata.iommu;
cb452a40
MT		 2484 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2485 return (info->domain == si_domain);
2c2e2c38 2486
2c2e2c38
FY		 2487 return 0;
2488}
2489
2490static int domain_add_dev_info(struct dmar_domain *domain,
5913c9bf 2491 struct device *dev, int translation)
2c2e2c38 2492{
0ac72664 2493 struct dmar_domain *ndomain;
5a8f40e8 2494 struct intel_iommu *iommu;
156baca8 2495 u8 bus, devfn;
5fe60f4e 2496 int ret;
2c2e2c38 2497
5913c9bf 2498 iommu = device_to_iommu(dev, &bus, &devfn);
5a8f40e8
DW		 2499 if (!iommu)
2500 return -ENODEV;
2501
5913c9bf 2502 ndomain = dmar_insert_dev_info(iommu, bus, devfn, dev, domain);
0ac72664
DW		 2503 if (ndomain != domain)
2504 return -EBUSY;
2c2e2c38 2505
5913c9bf 2506 ret = domain_context_mapping(domain, dev, translation);
e2ad23d0 2507 if (ret) {
5913c9bf 2508 domain_remove_one_dev_info(domain, dev);
e2ad23d0
DW		 2509 return ret;
2510 }
2511
2c2e2c38
FY		 2512 return 0;
2513}
2514
0b9d9753 2515static bool device_has_rmrr(struct device *dev)
ea2447f7
TM		 2516{
2517 struct dmar_rmrr_unit *rmrr;
832bd858 2518 struct device *tmp;
ea2447f7
TM		 2519 int i;
2520
0e242612 2521 rcu_read_lock();
ea2447f7 2522 for_each_rmrr_units(rmrr) {
b683b230
JL		 2523 /*
2524 * Return TRUE if this RMRR contains the device that
2525 * is passed in.
2526 */
2527 for_each_active_dev_scope(rmrr->devices,
2528 rmrr->devices_cnt, i, tmp)
0b9d9753 2529 if (tmp == dev) {
0e242612 2530 rcu_read_unlock();
ea2447f7 2531 return true;
b683b230 2532 }
ea2447f7 2533 }
0e242612 2534 rcu_read_unlock();
ea2447f7
TM		 2535 return false;
2536}
2537
3bdb2591 2538static int iommu_should_identity_map(struct device *dev, int startup)
6941af28 2539{
ea2447f7 2540
3bdb2591
DW		 2541 if (dev_is_pci(dev)) {
2542 struct pci_dev *pdev = to_pci_dev(dev);
ea2447f7 2543
3bdb2591
DW		 2544 /*
2545 * We want to prevent any device associated with an RMRR from
2546 * getting placed into the SI Domain. This is done because
2547 * problems exist when devices are moved in and out of domains
2548 * and their respective RMRR info is lost. We exempt USB devices
2549 * from this process due to their usage of RMRRs that are known
2550 * to not be needed after BIOS hand-off to OS.
2551 */
2552 if (device_has_rmrr(dev) &&
2553 (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2554 return 0;
e0fc7e0b 2555
3bdb2591
DW		 2556 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2557 return 1;
e0fc7e0b 2558
3bdb2591
DW		 2559 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2560 return 1;
6941af28 2561
3bdb2591 2562 if (!(iommu_identity_mapping & IDENTMAP_ALL))
3dfc813d 2563 return 0;
3bdb2591
DW		 2564
2565 /*
2566 * We want to start off with all devices in the 1:1 domain, and
2567 * take them out later if we find they can't access all of memory.
2568 *
2569 * However, we can't do this for PCI devices behind bridges,
2570 * because all PCI devices behind the same bridge will end up
2571 * with the same source-id on their transactions.
2572 *
2573 * Practically speaking, we can't change things around for these
2574 * devices at run-time, because we can't be sure there'll be no
2575 * DMA transactions in flight for any of their siblings.
2576 *
2577 * So PCI devices (unless they're on the root bus) as well as
2578 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2579 * the 1:1 domain, just in _case_ one of their siblings turns out
2580 * not to be able to map all of memory.
2581 */
2582 if (!pci_is_pcie(pdev)) {
2583 if (!pci_is_root_bus(pdev->bus))
2584 return 0;
2585 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2586 return 0;
2587 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
3dfc813d 2588 return 0;
3bdb2591
DW		 2589 } else {
2590 if (device_has_rmrr(dev))
2591 return 0;
2592 }
3dfc813d 2593
3bdb2591 2594 /*
3dfc813d 2595 * At boot time, we don't yet know if devices will be 64-bit capable.
3bdb2591 2596 * Assume that they will — if they turn out not to be, then we can
3dfc813d
DW		 2597 * take them out of the 1:1 domain later.
2598 */
8fcc5372
CW		 2599 if (!startup) {
2600 /*
2601 * If the device's dma_mask is less than the system's memory
2602 * size then this is not a candidate for identity mapping.
2603 */
3bdb2591 2604 u64 dma_mask = *dev->dma_mask;
8fcc5372 2605
3bdb2591
DW		 2606 if (dev->coherent_dma_mask &&
2607 dev->coherent_dma_mask < dma_mask)
2608 dma_mask = dev->coherent_dma_mask;
8fcc5372 2609
3bdb2591 2610 return dma_mask >= dma_get_required_mask(dev);
8fcc5372 2611 }
6941af28
DW		 2612
2613 return 1;
2614}
2615
cf04eee8
DW		 2616static int __init dev_prepare_static_identity_mapping(struct device *dev, int hw)
2617{
2618 int ret;
2619
2620 if (!iommu_should_identity_map(dev, 1))
2621 return 0;
2622
2623 ret = domain_add_dev_info(si_domain, dev,
2624 hw ? CONTEXT_TT_PASS_THROUGH :
2625 CONTEXT_TT_MULTI_LEVEL);
2626 if (!ret)
2627 pr_info("IOMMU: %s identity mapping for device %s\n",
2628 hw ? "hardware" : "software", dev_name(dev));
2629 else if (ret == -ENODEV)
2630 /* device not associated with an iommu */
2631 ret = 0;
2632
2633 return ret;
2634}
2635
2636
071e1374 2637static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2638{
2c2e2c38 2639 struct pci_dev *pdev = NULL;
cf04eee8
DW		 2640 struct dmar_drhd_unit *drhd;
2641 struct intel_iommu *iommu;
2642 struct device *dev;
2643 int i;
2644 int ret = 0;
2c2e2c38 2645
19943b0e 2646 ret = si_domain_init(hw);
2c2e2c38
FY		 2647 if (ret)
2648 return -EFAULT;
2649
2c2e2c38 2650 for_each_pci_dev(pdev) {
cf04eee8
DW		 2651 ret = dev_prepare_static_identity_mapping(&pdev->dev, hw);
2652 if (ret)
2653 return ret;
2654 }
2655
2656 for_each_active_iommu(iommu, drhd)
2657 for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) {
2658 struct acpi_device_physical_node *pn;
2659 struct acpi_device *adev;
2660
2661 if (dev->bus != &acpi_bus_type)
2662 continue;
2663
2664 adev= to_acpi_device(dev);
2665 mutex_lock(&adev->physical_node_lock);
2666 list_for_each_entry(pn, &adev->physical_node_list, node) {
2667 ret = dev_prepare_static_identity_mapping(pn->dev, hw);
2668 if (ret)
2669 break;
eae460b6 2670 }
cf04eee8
DW		 2671 mutex_unlock(&adev->physical_node_lock);
2672 if (ret)
2673 return ret;
62edf5dc 2674 }
2c2e2c38
FY		 2675
2676 return 0;
2677}
2678
b779260b 2679static int __init init_dmars(void)
ba395927
KA		 2680{
2681 struct dmar_drhd_unit *drhd;
2682 struct dmar_rmrr_unit *rmrr;
832bd858 2683 struct device *dev;
ba395927 2684 struct intel_iommu *iommu;
9d783ba0 2685 int i, ret;
2c2e2c38 2686
ba395927
KA		 2687 /*
2688 * for each drhd
2689 * allocate root
2690 * initialize and program root entry to not present
2691 * endfor
2692 */
2693 for_each_drhd_unit(drhd) {
5e0d2a6f 2694 /*
2695 * lock not needed as this is only incremented in the single
2696 * threaded kernel __init code path all other access are read
2697 * only
2698 */
1b198bb0
MT		 2699 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2700 g_num_of_iommus++;
2701 continue;
2702 }
2703 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2704 IOMMU_UNITS_SUPPORTED);
5e0d2a6f 2705 }
2706
d9630fe9
WH		 2707 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2708 GFP_KERNEL);
2709 if (!g_iommus) {
2710 printk(KERN_ERR "Allocating global iommu array failed\n");
2711 ret = -ENOMEM;
2712 goto error;
2713 }
2714
80b20dd8 2715 deferred_flush = kzalloc(g_num_of_iommus *
2716 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2717 if (!deferred_flush) {
5e0d2a6f 2718 ret = -ENOMEM;
989d51fc 2719 goto free_g_iommus;
5e0d2a6f 2720 }
2721
7c919779 2722 for_each_active_iommu(iommu, drhd) {
d9630fe9 2723 g_iommus[iommu->seq_id] = iommu;
ba395927 2724
e61d98d8
SS		 2725 ret = iommu_init_domains(iommu);
2726 if (ret)
989d51fc 2727 goto free_iommu;
e61d98d8 2728
ba395927
KA		 2729 /*
2730 * TBD:
2731 * we could share the same root & context tables
25985edc 2732 * among all IOMMU's. Need to Split it later.
ba395927
KA		 2733 */
2734 ret = iommu_alloc_root_entry(iommu);
2735 if (ret) {
2736 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
989d51fc 2737 goto free_iommu;
ba395927 2738 }
4ed0d3e6 2739 if (!ecap_pass_through(iommu->ecap))
19943b0e 2740 hw_pass_through = 0;
ba395927
KA		 2741 }
2742
1531a6a6
SS		 2743 /*
2744 * Start from the sane iommu hardware state.
2745 */
7c919779 2746 for_each_active_iommu(iommu, drhd) {
1531a6a6
SS		 2747 /*
2748 * If the queued invalidation is already initialized by us
2749 * (for example, while enabling interrupt-remapping) then
2750 * we got the things already rolling from a sane state.
2751 */
2752 if (iommu->qi)
2753 continue;
2754
2755 /*
2756 * Clear any previous faults.
2757 */
2758 dmar_fault(-1, iommu);
2759 /*
2760 * Disable queued invalidation if supported and already enabled
2761 * before OS handover.
2762 */
2763 dmar_disable_qi(iommu);
2764 }
2765
7c919779 2766 for_each_active_iommu(iommu, drhd) {
a77b67d4
YS		 2767 if (dmar_enable_qi(iommu)) {
2768 /*
2769 * Queued Invalidate not enabled, use Register Based
2770 * Invalidate
2771 */
2772 iommu->flush.flush_context = __iommu_flush_context;
2773 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
680a7524 2774 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
b4e0f9eb 2775 "invalidation\n",
680a7524 2776 iommu->seq_id,
b4e0f9eb 2777 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2778 } else {
2779 iommu->flush.flush_context = qi_flush_context;
2780 iommu->flush.flush_iotlb = qi_flush_iotlb;
680a7524 2781 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
b4e0f9eb 2782 "invalidation\n",
680a7524 2783 iommu->seq_id,
b4e0f9eb 2784 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2785 }
2786 }
2787
19943b0e 2788 if (iommu_pass_through)
e0fc7e0b
DW		 2789 iommu_identity_mapping |= IDENTMAP_ALL;
2790
d3f13810 2791#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
e0fc7e0b 2792 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2793#endif
e0fc7e0b
DW		 2794
2795 check_tylersburg_isoch();
2796
ba395927 2797 /*
19943b0e
DW		 2798 * If pass through is not set or not enabled, setup context entries for
2799 * identity mappings for rmrr, gfx, and isa and may fall back to static
2800 * identity mapping if iommu_identity_mapping is set.
ba395927 2801 */
19943b0e
DW		 2802 if (iommu_identity_mapping) {
2803 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2804 if (ret) {
19943b0e 2805 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
989d51fc 2806 goto free_iommu;
ba395927
KA		 2807 }
2808 }
ba395927 2809 /*
19943b0e
DW		 2810 * For each rmrr
2811 * for each dev attached to rmrr
2812 * do
2813 * locate drhd for dev, alloc domain for dev
2814 * allocate free domain
2815 * allocate page table entries for rmrr
2816 * if context not allocated for bus
2817 * allocate and init context
2818 * set present in root table for this bus
2819 * init context with domain, translation etc
2820 * endfor
2821 * endfor
ba395927 2822 */
19943b0e
DW		 2823 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2824 for_each_rmrr_units(rmrr) {
b683b230
JL		 2825 /* some BIOS lists non-exist devices in DMAR table. */
2826 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
832bd858 2827 i, dev) {
0b9d9753 2828 ret = iommu_prepare_rmrr_dev(rmrr, dev);
19943b0e
DW		 2829 if (ret)
2830 printk(KERN_ERR
2831 "IOMMU: mapping reserved region failed\n");
ba395927 2832 }
4ed0d3e6 2833 }
49a0429e 2834
19943b0e
DW		 2835 iommu_prepare_isa();
2836
ba395927
KA		 2837 /*
2838 * for each drhd
2839 * enable fault log
2840 * global invalidate context cache
2841 * global invalidate iotlb
2842 * enable translation
2843 */
7c919779 2844 for_each_iommu(iommu, drhd) {
51a63e67
JC		 2845 if (drhd->ignored) {
2846 /*
2847 * we always have to disable PMRs or DMA may fail on
2848 * this device
2849 */
2850 if (force_on)
7c919779 2851 iommu_disable_protect_mem_regions(iommu);
ba395927 2852 continue;
51a63e67 2853 }
ba395927
KA		 2854
2855 iommu_flush_write_buffer(iommu);
2856
3460a6d9
KA		 2857 ret = dmar_set_interrupt(iommu);
2858 if (ret)
989d51fc 2859 goto free_iommu;
3460a6d9 2860
ba395927
KA		 2861 iommu_set_root_entry(iommu);
2862
4c25a2c1 2863 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2864 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
f8bab735 2865
ba395927
KA		 2866 ret = iommu_enable_translation(iommu);
2867 if (ret)
989d51fc 2868 goto free_iommu;
b94996c9
DW		 2869
2870 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2871 }
2872
2873 return 0;
989d51fc
JL		 2874
2875free_iommu:
7c919779 2876 for_each_active_iommu(iommu, drhd)
a868e6b7 2877 free_dmar_iommu(iommu);
9bdc531e 2878 kfree(deferred_flush);
989d51fc 2879free_g_iommus:
d9630fe9 2880 kfree(g_iommus);
989d51fc 2881error:
ba395927
KA		 2882 return ret;
2883}
2884
5a5e02a6 2885/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2886static struct iova *intel_alloc_iova(struct device *dev,
2887 struct dmar_domain *domain,
2888 unsigned long nrpages, uint64_t dma_mask)
ba395927 2889{
ba395927 2890 struct iova *iova = NULL;
ba395927 2891
875764de
DW		 2892 /* Restrict dma_mask to the width that the iommu can handle */
2893 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2894
2895 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2896 /*
2897 * First try to allocate an io virtual address in
284901a9 2898 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2899 * from higher range
ba395927 2900 */
875764de
DW		 2901 iova = alloc_iova(&domain->iovad, nrpages,
2902 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2903 if (iova)
2904 return iova;
2905 }
2906 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2907 if (unlikely(!iova)) {
2908 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
207e3592 2909 nrpages, dev_name(dev));
f76aec76
KA		 2910 return NULL;
2911 }
2912
2913 return iova;
2914}
2915
d4b709f4 2916static struct dmar_domain *__get_valid_domain_for_dev(struct device *dev)
f76aec76
KA		 2917{
2918 struct dmar_domain *domain;
2919 int ret;
2920
d4b709f4 2921 domain = get_domain_for_dev(dev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
f76aec76 2922 if (!domain) {
d4b709f4
DW		 2923 printk(KERN_ERR "Allocating domain for %s failed",
2924 dev_name(dev));
4fe05bbc 2925 return NULL;
ba395927
KA		 2926 }
2927
2928 /* make sure context mapping is ok */
d4b709f4
DW		 2929 if (unlikely(!domain_context_mapped(dev))) {
2930 ret = domain_context_mapping(domain, dev, CONTEXT_TT_MULTI_LEVEL);
f76aec76 2931 if (ret) {
d4b709f4
DW		 2932 printk(KERN_ERR "Domain context map for %s failed",
2933 dev_name(dev));
4fe05bbc 2934 return NULL;
f76aec76 2935 }
ba395927
KA		 2936 }
2937
f76aec76
KA		 2938 return domain;
2939}
2940
d4b709f4 2941static inline struct dmar_domain *get_valid_domain_for_dev(struct device *dev)
147202aa
DW		 2942{
2943 struct device_domain_info *info;
2944
2945 /* No lock here, assumes no domain exit in normal case */
d4b709f4 2946 info = dev->archdata.iommu;
147202aa
DW		 2947 if (likely(info))
2948 return info->domain;
2949
2950 return __get_valid_domain_for_dev(dev);
2951}
2952
3d89194a 2953static int iommu_dummy(struct device *dev)
2c2e2c38 2954{
3d89194a 2955 return dev->archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2c2e2c38
FY		 2956}
2957
ecb509ec 2958/* Check if the dev needs to go through non-identity map and unmap process.*/
73676832 2959static int iommu_no_mapping(struct device *dev)
2c2e2c38
FY		 2960{
2961 int found;
2962
3d89194a 2963 if (iommu_dummy(dev))
1e4c64c4
DW		 2964 return 1;
2965
2c2e2c38 2966 if (!iommu_identity_mapping)
1e4c64c4 2967 return 0;
2c2e2c38 2968
9b226624 2969 found = identity_mapping(dev);
2c2e2c38 2970 if (found) {
ecb509ec 2971 if (iommu_should_identity_map(dev, 0))
2c2e2c38
FY		 2972 return 1;
2973 else {
2974 /*
2975 * 32 bit DMA is removed from si_domain and fall back
2976 * to non-identity mapping.
2977 */
bf9c9eda 2978 domain_remove_one_dev_info(si_domain, dev);
2c2e2c38 2979 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
ecb509ec 2980 dev_name(dev));
2c2e2c38
FY		 2981 return 0;
2982 }
2983 } else {
2984 /*
2985 * In case of a detached 64 bit DMA device from vm, the device
2986 * is put into si_domain for identity mapping.
2987 */
ecb509ec 2988 if (iommu_should_identity_map(dev, 0)) {
2c2e2c38 2989 int ret;
5913c9bf 2990 ret = domain_add_dev_info(si_domain, dev,
5fe60f4e
DW		 2991 hw_pass_through ?
2992 CONTEXT_TT_PASS_THROUGH :
2993 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38
FY		 2994 if (!ret) {
2995 printk(KERN_INFO "64bit %s uses identity mapping\n",
ecb509ec 2996 dev_name(dev));
2c2e2c38
FY		 2997 return 1;
2998 }
2999 }
3000 }
3001
1e4c64c4 3002 return 0;
2c2e2c38
FY		 3003}
3004
5040a918 3005static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
bb9e6d65 3006 size_t size, int dir, u64 dma_mask)
f76aec76 3007{
f76aec76 3008 struct dmar_domain *domain;
5b6985ce 3009 phys_addr_t start_paddr;
f76aec76
KA		 3010 struct iova *iova;
3011 int prot = 0;
6865f0d1 3012 int ret;
8c11e798 3013 struct intel_iommu *iommu;
33041ec0 3014 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 3015
3016 BUG_ON(dir == DMA_NONE);
2c2e2c38 3017
5040a918 3018 if (iommu_no_mapping(dev))
6865f0d1 3019 return paddr;
f76aec76 3020
5040a918 3021 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3022 if (!domain)
3023 return 0;
3024
8c11e798 3025 iommu = domain_get_iommu(domain);
88cb6a74 3026 size = aligned_nrpages(paddr, size);
f76aec76 3027
5040a918 3028 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
f76aec76
KA		 3029 if (!iova)
3030 goto error;
3031
ba395927
KA		 3032 /*
3033 * Check if DMAR supports zero-length reads on write only
3034 * mappings..
3035 */
3036 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3037 !cap_zlr(iommu->cap))
ba395927
KA		 3038 prot |= DMA_PTE_READ;
3039 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3040 prot |= DMA_PTE_WRITE;
3041 /*
6865f0d1 3042 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 3043 * page. Note: if two part of one page are separately mapped, we
6865f0d1 3044 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 3045 * is not a big problem
3046 */
0ab36de2 3047 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 3048 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 3049 if (ret)
3050 goto error;
3051
1f0ef2aa
DW		 3052 /* it's a non-present to present mapping. Only flush if caching mode */
3053 if (cap_caching_mode(iommu->cap))
ea8ea460 3054 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 0, 1);
1f0ef2aa 3055 else
8c11e798 3056 iommu_flush_write_buffer(iommu);
f76aec76 3057
03d6a246
DW		 3058 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
3059 start_paddr += paddr & ~PAGE_MASK;
3060 return start_paddr;
ba395927 3061
ba395927 3062error:
f76aec76
KA		 3063 if (iova)
3064 __free_iova(&domain->iovad, iova);
4cf2e75d 3065 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5040a918 3066 dev_name(dev), size, (unsigned long long)paddr, dir);
ba395927
KA		 3067 return 0;
3068}
3069
ffbbef5c
FT		 3070static dma_addr_t intel_map_page(struct device *dev, struct page *page,
3071 unsigned long offset, size_t size,
3072 enum dma_data_direction dir,
3073 struct dma_attrs *attrs)
bb9e6d65 3074{
ffbbef5c 3075 return __intel_map_single(dev, page_to_phys(page) + offset, size,
46333e37 3076 dir, *dev->dma_mask);
bb9e6d65
FT		 3077}
3078
5e0d2a6f 3079static void flush_unmaps(void)
3080{
80b20dd8 3081 int i, j;
5e0d2a6f 3082
5e0d2a6f 3083 timer_on = 0;
3084
3085 /* just flush them all */
3086 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 3087 struct intel_iommu *iommu = g_iommus[i];
3088 if (!iommu)
3089 continue;
c42d9f32 3090
9dd2fe89
YZ		 3091 if (!deferred_flush[i].next)
3092 continue;
3093
78d5f0f5
NA		 3094 /* In caching mode, global flushes turn emulation expensive */
3095 if (!cap_caching_mode(iommu->cap))
3096 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 3097 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 3098 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 3099 unsigned long mask;
3100 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 3101 struct dmar_domain *domain = deferred_flush[i].domain[j];
3102
3103 /* On real hardware multiple invalidations are expensive */
3104 if (cap_caching_mode(iommu->cap))
3105 iommu_flush_iotlb_psi(iommu, domain->id,
ea8ea460
DW		 3106 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1,
3107 !deferred_flush[i].freelist[j], 0);
78d5f0f5
NA		 3108 else {
3109 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
3110 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
3111 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
3112 }
93a23a72 3113 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
ea8ea460
DW		 3114 if (deferred_flush[i].freelist[j])
3115 dma_free_pagelist(deferred_flush[i].freelist[j]);
80b20dd8 3116 }
9dd2fe89 3117 deferred_flush[i].next = 0;
5e0d2a6f 3118 }
3119
5e0d2a6f 3120 list_size = 0;
5e0d2a6f 3121}
3122
3123static void flush_unmaps_timeout(unsigned long data)
3124{
80b20dd8 3125 unsigned long flags;
3126
3127 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 3128 flush_unmaps();
80b20dd8 3129 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 3130}
3131
ea8ea460 3132static void add_unmap(struct dmar_domain *dom, struct iova *iova, struct page *freelist)
5e0d2a6f 3133{
3134 unsigned long flags;
80b20dd8 3135 int next, iommu_id;
8c11e798 3136 struct intel_iommu *iommu;
5e0d2a6f 3137
3138 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 3139 if (list_size == HIGH_WATER_MARK)
3140 flush_unmaps();
3141
8c11e798
WH		 3142 iommu = domain_get_iommu(dom);
3143 iommu_id = iommu->seq_id;
c42d9f32 3144
80b20dd8 3145 next = deferred_flush[iommu_id].next;
3146 deferred_flush[iommu_id].domain[next] = dom;
3147 deferred_flush[iommu_id].iova[next] = iova;
ea8ea460 3148 deferred_flush[iommu_id].freelist[next] = freelist;
80b20dd8 3149 deferred_flush[iommu_id].next++;
5e0d2a6f 3150
3151 if (!timer_on) {
3152 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
3153 timer_on = 1;
3154 }
3155 list_size++;
3156 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
3157}
3158
ffbbef5c
FT		 3159static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
3160 size_t size, enum dma_data_direction dir,
3161 struct dma_attrs *attrs)
ba395927 3162{
f76aec76 3163 struct dmar_domain *domain;
d794dc9b 3164 unsigned long start_pfn, last_pfn;
ba395927 3165 struct iova *iova;
8c11e798 3166 struct intel_iommu *iommu;
ea8ea460 3167 struct page *freelist;
ba395927 3168
73676832 3169 if (iommu_no_mapping(dev))
f76aec76 3170 return;
2c2e2c38 3171
1525a29a 3172 domain = find_domain(dev);
ba395927
KA		 3173 BUG_ON(!domain);
3174
8c11e798
WH		 3175 iommu = domain_get_iommu(domain);
3176
ba395927 3177 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 3178 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
3179 (unsigned long long)dev_addr))
ba395927 3180 return;
ba395927 3181
d794dc9b
DW		 3182 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3183 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 3184
d794dc9b 3185 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
207e3592 3186 dev_name(dev), start_pfn, last_pfn);
ba395927 3187
ea8ea460 3188 freelist = domain_unmap(domain, start_pfn, last_pfn);
d794dc9b 3189
5e0d2a6f 3190 if (intel_iommu_strict) {
03d6a246 3191 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
ea8ea460 3192 last_pfn - start_pfn + 1, !freelist, 0);
5e0d2a6f 3193 /* free iova */
3194 __free_iova(&domain->iovad, iova);
ea8ea460 3195 dma_free_pagelist(freelist);
5e0d2a6f 3196 } else {
ea8ea460 3197 add_unmap(domain, iova, freelist);
5e0d2a6f 3198 /*
3199 * queue up the release of the unmap to save the 1/6th of the
3200 * cpu used up by the iotlb flush operation...
3201 */
5e0d2a6f 3202 }
ba395927
KA		 3203}
3204
5040a918 3205static void *intel_alloc_coherent(struct device *dev, size_t size,
baa676fc
AP		 3206 dma_addr_t *dma_handle, gfp_t flags,
3207 struct dma_attrs *attrs)
ba395927 3208{
36746436 3209 struct page *page = NULL;
ba395927
KA		 3210 int order;
3211
5b6985ce 3212 size = PAGE_ALIGN(size);
ba395927 3213 order = get_order(size);
e8bb910d 3214
5040a918 3215 if (!iommu_no_mapping(dev))
e8bb910d 3216 flags &= ~(GFP_DMA | GFP_DMA32);
5040a918
DW		 3217 else if (dev->coherent_dma_mask < dma_get_required_mask(dev)) {
3218 if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
e8bb910d
AW		 3219 flags |= GFP_DMA;
3220 else
3221 flags |= GFP_DMA32;
3222 }
ba395927 3223
36746436
AM		 3224 if (flags & __GFP_WAIT) {
3225 unsigned int count = size >> PAGE_SHIFT;
3226
3227 page = dma_alloc_from_contiguous(dev, count, order);
3228 if (page && iommu_no_mapping(dev) &&
3229 page_to_phys(page) + size > dev->coherent_dma_mask) {
3230 dma_release_from_contiguous(dev, page, count);
3231 page = NULL;
3232 }
3233 }
3234
3235 if (!page)
3236 page = alloc_pages(flags, order);
3237 if (!page)
ba395927 3238 return NULL;
36746436 3239 memset(page_address(page), 0, size);
ba395927 3240
36746436 3241 *dma_handle = __intel_map_single(dev, page_to_phys(page), size,
bb9e6d65 3242 DMA_BIDIRECTIONAL,
5040a918 3243 dev->coherent_dma_mask);
ba395927 3244 if (*dma_handle)
36746436
AM		 3245 return page_address(page);
3246 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3247 __free_pages(page, order);
3248
ba395927
KA		 3249 return NULL;
3250}
3251
5040a918 3252static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
baa676fc 3253 dma_addr_t dma_handle, struct dma_attrs *attrs)
ba395927
KA		 3254{
3255 int order;
36746436 3256 struct page *page = virt_to_page(vaddr);
ba395927 3257
5b6985ce 3258 size = PAGE_ALIGN(size);
ba395927
KA		 3259 order = get_order(size);
3260
5040a918 3261 intel_unmap_page(dev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
36746436
AM		 3262 if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
3263 __free_pages(page, order);
ba395927
KA		 3264}
3265
5040a918 3266static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
d7ab5c46
FT		 3267 int nelems, enum dma_data_direction dir,
3268 struct dma_attrs *attrs)
ba395927 3269{
ba395927 3270 struct dmar_domain *domain;
d794dc9b 3271 unsigned long start_pfn, last_pfn;
f76aec76 3272 struct iova *iova;
8c11e798 3273 struct intel_iommu *iommu;
ea8ea460 3274 struct page *freelist;
ba395927 3275
5040a918 3276 if (iommu_no_mapping(dev))
ba395927
KA		 3277 return;
3278
5040a918 3279 domain = find_domain(dev);
8c11e798
WH		 3280 BUG_ON(!domain);
3281
3282 iommu = domain_get_iommu(domain);
ba395927 3283
c03ab37c 3284 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
85b98276
DW		 3285 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3286 (unsigned long long)sglist[0].dma_address))
f76aec76 3287 return;
f76aec76 3288
d794dc9b
DW		 3289 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3290 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
f76aec76 3291
ea8ea460 3292 freelist = domain_unmap(domain, start_pfn, last_pfn);
f76aec76 3293
acea0018
DW		 3294 if (intel_iommu_strict) {
3295 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
ea8ea460 3296 last_pfn - start_pfn + 1, !freelist, 0);
acea0018
DW		 3297 /* free iova */
3298 __free_iova(&domain->iovad, iova);
ea8ea460 3299 dma_free_pagelist(freelist);
acea0018 3300 } else {
ea8ea460 3301 add_unmap(domain, iova, freelist);
acea0018
DW		 3302 /*
3303 * queue up the release of the unmap to save the 1/6th of the
3304 * cpu used up by the iotlb flush operation...
3305 */
3306 }
ba395927
KA		 3307}
3308
ba395927 3309static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 3310 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 3311{
3312 int i;
c03ab37c 3313 struct scatterlist *sg;
ba395927 3314
c03ab37c 3315 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 3316 BUG_ON(!sg_page(sg));
4cf2e75d 3317 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 3318 sg->dma_length = sg->length;
ba395927
KA		 3319 }
3320 return nelems;
3321}
3322
5040a918 3323static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
d7ab5c46 3324 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 3325{
ba395927 3326 int i;
ba395927 3327 struct dmar_domain *domain;
f76aec76
KA		 3328 size_t size = 0;
3329 int prot = 0;
f76aec76
KA		 3330 struct iova *iova = NULL;
3331 int ret;
c03ab37c 3332 struct scatterlist *sg;
b536d24d 3333 unsigned long start_vpfn;
8c11e798 3334 struct intel_iommu *iommu;
ba395927
KA		 3335
3336 BUG_ON(dir == DMA_NONE);
5040a918
DW		 3337 if (iommu_no_mapping(dev))
3338 return intel_nontranslate_map_sg(dev, sglist, nelems, dir);
ba395927 3339
5040a918 3340 domain = get_valid_domain_for_dev(dev);
f76aec76
KA		 3341 if (!domain)
3342 return 0;
3343
8c11e798
WH		 3344 iommu = domain_get_iommu(domain);
3345
b536d24d 3346 for_each_sg(sglist, sg, nelems, i)
88cb6a74 3347 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 3348
5040a918
DW		 3349 iova = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
3350 *dev->dma_mask);
f76aec76 3351 if (!iova) {
c03ab37c 3352 sglist->dma_length = 0;
f76aec76
KA		 3353 return 0;
3354 }
3355
3356 /*
3357 * Check if DMAR supports zero-length reads on write only
3358 * mappings..
3359 */
3360 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 3361 !cap_zlr(iommu->cap))
f76aec76
KA		 3362 prot |= DMA_PTE_READ;
3363 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3364 prot |= DMA_PTE_WRITE;
3365
b536d24d 3366 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 3367
f532959b 3368 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495
DW		 3369 if (unlikely(ret)) {
3370 /* clear the page */
3371 dma_pte_clear_range(domain, start_vpfn,
3372 start_vpfn + size - 1);
3373 /* free page tables */
3374 dma_pte_free_pagetable(domain, start_vpfn,
3375 start_vpfn + size - 1);
3376 /* free iova */
3377 __free_iova(&domain->iovad, iova);
3378 return 0;
ba395927
KA		 3379 }
3380
1f0ef2aa
DW		 3381 /* it's a non-present to present mapping. Only flush if caching mode */
3382 if (cap_caching_mode(iommu->cap))
ea8ea460 3383 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 0, 1);
1f0ef2aa 3384 else
8c11e798 3385 iommu_flush_write_buffer(iommu);
1f0ef2aa 3386
ba395927
KA		 3387 return nelems;
3388}
3389
dfb805e8
FT		 3390static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3391{
3392 return !dma_addr;
3393}
3394
160c1d8e 3395struct dma_map_ops intel_dma_ops = {
baa676fc
AP		 3396 .alloc = intel_alloc_coherent,
3397 .free = intel_free_coherent,
ba395927
KA		 3398 .map_sg = intel_map_sg,
3399 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 3400 .map_page = intel_map_page,
3401 .unmap_page = intel_unmap_page,
dfb805e8 3402 .mapping_error = intel_mapping_error,
ba395927
KA		 3403};
3404
3405static inline int iommu_domain_cache_init(void)
3406{
3407 int ret = 0;
3408
3409 iommu_domain_cache = kmem_cache_create("iommu_domain",
3410 sizeof(struct dmar_domain),
3411 0,
3412 SLAB_HWCACHE_ALIGN,
3413
3414 NULL);
3415 if (!iommu_domain_cache) {
3416 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3417 ret = -ENOMEM;
3418 }
3419
3420 return ret;
3421}
3422
3423static inline int iommu_devinfo_cache_init(void)
3424{
3425 int ret = 0;
3426
3427 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3428 sizeof(struct device_domain_info),
3429 0,
3430 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3431 NULL);
3432 if (!iommu_devinfo_cache) {
3433 printk(KERN_ERR "Couldn't create devinfo cache\n");
3434 ret = -ENOMEM;
3435 }
3436
3437 return ret;
3438}
3439
3440static inline int iommu_iova_cache_init(void)
3441{
3442 int ret = 0;
3443
3444 iommu_iova_cache = kmem_cache_create("iommu_iova",
3445 sizeof(struct iova),
3446 0,
3447 SLAB_HWCACHE_ALIGN,
ba395927
KA		 3448 NULL);
3449 if (!iommu_iova_cache) {
3450 printk(KERN_ERR "Couldn't create iova cache\n");
3451 ret = -ENOMEM;
3452 }
3453
3454 return ret;
3455}
3456
3457static int __init iommu_init_mempool(void)
3458{
3459 int ret;
3460 ret = iommu_iova_cache_init();
3461 if (ret)
3462 return ret;
3463
3464 ret = iommu_domain_cache_init();
3465 if (ret)
3466 goto domain_error;
3467
3468 ret = iommu_devinfo_cache_init();
3469 if (!ret)
3470 return ret;
3471
3472 kmem_cache_destroy(iommu_domain_cache);
3473domain_error:
3474 kmem_cache_destroy(iommu_iova_cache);
3475
3476 return -ENOMEM;
3477}
3478
3479static void __init iommu_exit_mempool(void)
3480{
3481 kmem_cache_destroy(iommu_devinfo_cache);
3482 kmem_cache_destroy(iommu_domain_cache);
3483 kmem_cache_destroy(iommu_iova_cache);
3484
3485}
3486
556ab45f
DW		 3487static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3488{
3489 struct dmar_drhd_unit *drhd;
3490 u32 vtbar;
3491 int rc;
3492
3493 /* We know that this device on this chipset has its own IOMMU.
3494 * If we find it under a different IOMMU, then the BIOS is lying
3495 * to us. Hope that the IOMMU for this device is actually
3496 * disabled, and it needs no translation...
3497 */
3498 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3499 if (rc) {
3500 /* "can't" happen */
3501 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3502 return;
3503 }
3504 vtbar &= 0xffff0000;
3505
3506 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3507 drhd = dmar_find_matched_drhd_unit(pdev);
3508 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3509 TAINT_FIRMWARE_WORKAROUND,
3510 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3511 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3512}
3513DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3514
ba395927
KA		 3515static void __init init_no_remapping_devices(void)
3516{
3517 struct dmar_drhd_unit *drhd;
832bd858 3518 struct device *dev;
b683b230 3519 int i;
ba395927
KA		 3520
3521 for_each_drhd_unit(drhd) {
3522 if (!drhd->include_all) {
b683b230
JL		 3523 for_each_active_dev_scope(drhd->devices,
3524 drhd->devices_cnt, i, dev)
3525 break;
832bd858 3526 /* ignore DMAR unit if no devices exist */
ba395927
KA		 3527 if (i == drhd->devices_cnt)
3528 drhd->ignored = 1;
3529 }
3530 }
3531
7c919779 3532 for_each_active_drhd_unit(drhd) {
7c919779 3533 if (drhd->include_all)
ba395927
KA		 3534 continue;
3535
b683b230
JL		 3536 for_each_active_dev_scope(drhd->devices,
3537 drhd->devices_cnt, i, dev)
832bd858 3538 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
ba395927 3539 break;
ba395927
KA		 3540 if (i < drhd->devices_cnt)
3541 continue;
3542
c0771df8
DW		 3543 /* This IOMMU has *only* gfx devices. Either bypass it or
3544 set the gfx_mapped flag, as appropriate */
3545 if (dmar_map_gfx) {
3546 intel_iommu_gfx_mapped = 1;
3547 } else {
3548 drhd->ignored = 1;
b683b230
JL		 3549 for_each_active_dev_scope(drhd->devices,
3550 drhd->devices_cnt, i, dev)
832bd858 3551 dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3552 }
3553 }
3554}
3555
f59c7b69
FY		 3556#ifdef CONFIG_SUSPEND
3557static int init_iommu_hw(void)
3558{
3559 struct dmar_drhd_unit *drhd;
3560 struct intel_iommu *iommu = NULL;
3561
3562 for_each_active_iommu(iommu, drhd)
3563 if (iommu->qi)
3564 dmar_reenable_qi(iommu);
3565
b779260b
JC		 3566 for_each_iommu(iommu, drhd) {
3567 if (drhd->ignored) {
3568 /*
3569 * we always have to disable PMRs or DMA may fail on
3570 * this device
3571 */
3572 if (force_on)
3573 iommu_disable_protect_mem_regions(iommu);
3574 continue;
3575 }
3576
f59c7b69
FY		 3577 iommu_flush_write_buffer(iommu);
3578
3579 iommu_set_root_entry(iommu);
3580
3581 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3582 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3583 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3584 DMA_TLB_GLOBAL_FLUSH);
b779260b
JC		 3585 if (iommu_enable_translation(iommu))
3586 return 1;
b94996c9 3587 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3588 }
3589
3590 return 0;
3591}
3592
3593static void iommu_flush_all(void)
3594{
3595 struct dmar_drhd_unit *drhd;
3596 struct intel_iommu *iommu;
3597
3598 for_each_active_iommu(iommu, drhd) {
3599 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3600 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3601 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3602 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3603 }
3604}
3605
134fac3f 3606static int iommu_suspend(void)
f59c7b69
FY		 3607{
3608 struct dmar_drhd_unit *drhd;
3609 struct intel_iommu *iommu = NULL;
3610 unsigned long flag;
3611
3612 for_each_active_iommu(iommu, drhd) {
3613 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3614 GFP_ATOMIC);
3615 if (!iommu->iommu_state)
3616 goto nomem;
3617 }
3618
3619 iommu_flush_all();
3620
3621 for_each_active_iommu(iommu, drhd) {
3622 iommu_disable_translation(iommu);
3623
1f5b3c3f 3624 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3625
3626 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3627 readl(iommu->reg + DMAR_FECTL_REG);
3628 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3629 readl(iommu->reg + DMAR_FEDATA_REG);
3630 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3631 readl(iommu->reg + DMAR_FEADDR_REG);
3632 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3633 readl(iommu->reg + DMAR_FEUADDR_REG);
3634
1f5b3c3f 3635 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3636 }
3637 return 0;
3638
3639nomem:
3640 for_each_active_iommu(iommu, drhd)
3641 kfree(iommu->iommu_state);
3642
3643 return -ENOMEM;
3644}
3645
134fac3f 3646static void iommu_resume(void)
f59c7b69
FY		 3647{
3648 struct dmar_drhd_unit *drhd;
3649 struct intel_iommu *iommu = NULL;
3650 unsigned long flag;
3651
3652 if (init_iommu_hw()) {
b779260b
JC		 3653 if (force_on)
3654 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3655 else
3656 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
134fac3f 3657 return;
f59c7b69
FY		 3658 }
3659
3660 for_each_active_iommu(iommu, drhd) {
3661
1f5b3c3f 3662 raw_spin_lock_irqsave(&iommu->register_lock, flag);
f59c7b69
FY		 3663
3664 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3665 iommu->reg + DMAR_FECTL_REG);
3666 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3667 iommu->reg + DMAR_FEDATA_REG);
3668 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3669 iommu->reg + DMAR_FEADDR_REG);
3670 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3671 iommu->reg + DMAR_FEUADDR_REG);
3672
1f5b3c3f 3673 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
f59c7b69
FY		 3674 }
3675
3676 for_each_active_iommu(iommu, drhd)
3677 kfree(iommu->iommu_state);
f59c7b69
FY		 3678}
3679
134fac3f 3680static struct syscore_ops iommu_syscore_ops = {
f59c7b69
FY		 3681 .resume = iommu_resume,
3682 .suspend = iommu_suspend,
3683};
3684
134fac3f 3685static void __init init_iommu_pm_ops(void)
f59c7b69 3686{
134fac3f 3687 register_syscore_ops(&iommu_syscore_ops);
f59c7b69
FY		 3688}
3689
3690#else
99592ba4 3691static inline void init_iommu_pm_ops(void) {}
f59c7b69
FY		 3692#endif /* CONFIG_PM */
3693
318fe7df
SS		 3694
3695int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3696{
3697 struct acpi_dmar_reserved_memory *rmrr;
3698 struct dmar_rmrr_unit *rmrru;
3699
3700 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3701 if (!rmrru)
3702 return -ENOMEM;
3703
3704 rmrru->hdr = header;
3705 rmrr = (struct acpi_dmar_reserved_memory *)header;
3706 rmrru->base_address = rmrr->base_address;
3707 rmrru->end_address = rmrr->end_address;
2e455289
JL		 3708 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3709 ((void *)rmrr) + rmrr->header.length,
3710 &rmrru->devices_cnt);
3711 if (rmrru->devices_cnt && rmrru->devices == NULL) {
3712 kfree(rmrru);
3713 return -ENOMEM;
3714 }
318fe7df 3715
2e455289 3716 list_add(&rmrru->list, &dmar_rmrr_units);
318fe7df 3717
2e455289 3718 return 0;
318fe7df
SS		 3719}
3720
318fe7df
SS		 3721int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3722{
3723 struct acpi_dmar_atsr *atsr;
3724 struct dmar_atsr_unit *atsru;
3725
3726 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3727 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3728 if (!atsru)
3729 return -ENOMEM;
3730
3731 atsru->hdr = hdr;
3732 atsru->include_all = atsr->flags & 0x1;
2e455289
JL		 3733 if (!atsru->include_all) {
3734 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3735 (void *)atsr + atsr->header.length,
3736 &atsru->devices_cnt);
3737 if (atsru->devices_cnt && atsru->devices == NULL) {
3738 kfree(atsru);
3739 return -ENOMEM;
3740 }
3741 }
318fe7df 3742
0e242612 3743 list_add_rcu(&atsru->list, &dmar_atsr_units);
318fe7df
SS		 3744
3745 return 0;
3746}
3747
9bdc531e
JL		 3748static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3749{
3750 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3751 kfree(atsru);
3752}
3753
3754static void intel_iommu_free_dmars(void)
3755{
3756 struct dmar_rmrr_unit *rmrru, *rmrr_n;
3757 struct dmar_atsr_unit *atsru, *atsr_n;
3758
3759 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3760 list_del(&rmrru->list);
3761 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3762 kfree(rmrru);
318fe7df
SS		 3763 }
3764
9bdc531e
JL		 3765 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3766 list_del(&atsru->list);
3767 intel_iommu_free_atsr(atsru);
3768 }
318fe7df
SS		 3769}
3770
3771int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3772{
b683b230 3773 int i, ret = 1;
318fe7df 3774 struct pci_bus *bus;
832bd858
DW		 3775 struct pci_dev *bridge = NULL;
3776 struct device *tmp;
318fe7df
SS		 3777 struct acpi_dmar_atsr *atsr;
3778 struct dmar_atsr_unit *atsru;
3779
3780 dev = pci_physfn(dev);
318fe7df 3781 for (bus = dev->bus; bus; bus = bus->parent) {
b5f82ddf 3782 bridge = bus->self;
318fe7df 3783 if (!bridge || !pci_is_pcie(bridge) ||
62f87c0e 3784 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
318fe7df 3785 return 0;
b5f82ddf 3786 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
318fe7df 3787 break;
318fe7df 3788 }
b5f82ddf
JL		 3789 if (!bridge)
3790 return 0;
318fe7df 3791
0e242612 3792 rcu_read_lock();
b5f82ddf
JL		 3793 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3794 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3795 if (atsr->segment != pci_domain_nr(dev->bus))
3796 continue;
3797
b683b230 3798 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
832bd858 3799 if (tmp == &bridge->dev)
b683b230 3800 goto out;
b5f82ddf
JL		 3801
3802 if (atsru->include_all)
b683b230 3803 goto out;
b5f82ddf 3804 }
b683b230
JL		 3805 ret = 0;
3806out:
0e242612 3807 rcu_read_unlock();
318fe7df 3808
b683b230 3809 return ret;
318fe7df
SS		 3810}
3811
59ce0515
JL		 3812int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3813{
3814 int ret = 0;
3815 struct dmar_rmrr_unit *rmrru;
3816 struct dmar_atsr_unit *atsru;
3817 struct acpi_dmar_atsr *atsr;
3818 struct acpi_dmar_reserved_memory *rmrr;
3819
3820 if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING)
3821 return 0;
3822
3823 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3824 rmrr = container_of(rmrru->hdr,
3825 struct acpi_dmar_reserved_memory, header);
3826 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3827 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3828 ((void *)rmrr) + rmrr->header.length,
3829 rmrr->segment, rmrru->devices,
3830 rmrru->devices_cnt);
27e24950 3831 if(ret < 0)
59ce0515
JL		 3832 return ret;
3833 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
27e24950
JL		 3834 dmar_remove_dev_scope(info, rmrr->segment,
3835 rmrru->devices, rmrru->devices_cnt);
59ce0515
JL		 3836 }
3837 }
3838
3839 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3840 if (atsru->include_all)
3841 continue;
3842
3843 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3844 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3845 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3846 (void *)atsr + atsr->header.length,
3847 atsr->segment, atsru->devices,
3848 atsru->devices_cnt);
3849 if (ret > 0)
3850 break;
3851 else if(ret < 0)
3852 return ret;
3853 } else if (info->event == BUS_NOTIFY_DEL_DEVICE) {
3854 if (dmar_remove_dev_scope(info, atsr->segment,
3855 atsru->devices, atsru->devices_cnt))
3856 break;
3857 }
3858 }
3859
3860 return 0;
3861}
3862
99dcaded
FY		 3863/*
3864 * Here we only respond to action of unbound device from driver.
3865 *
3866 * Added device is not attached to its DMAR domain here yet. That will happen
3867 * when mapping the device to iova.
3868 */
3869static int device_notifier(struct notifier_block *nb,
3870 unsigned long action, void *data)
3871{
3872 struct device *dev = data;
99dcaded
FY		 3873 struct dmar_domain *domain;
3874
3d89194a 3875 if (iommu_dummy(dev))
44cd613c
DW		 3876 return 0;
3877
7e7dfab7
JL		 3878 if (action != BUS_NOTIFY_UNBOUND_DRIVER &&
3879 action != BUS_NOTIFY_DEL_DEVICE)
3880 return 0;
3881
1525a29a 3882 domain = find_domain(dev);
99dcaded
FY		 3883 if (!domain)
3884 return 0;
3885
3a5670e8 3886 down_read(&dmar_global_lock);
bf9c9eda 3887 domain_remove_one_dev_info(domain, dev);
ab8dfe25 3888 if (!domain_type_is_vm_or_si(domain) && list_empty(&domain->devices))
7e7dfab7 3889 domain_exit(domain);
3a5670e8 3890 up_read(&dmar_global_lock);
a97590e5 3891
99dcaded
FY		 3892 return 0;
3893}
3894
3895static struct notifier_block device_nb = {
3896 .notifier_call = device_notifier,
3897};
3898
75f05569
JL		 3899static int intel_iommu_memory_notifier(struct notifier_block *nb,
3900 unsigned long val, void *v)
3901{
3902 struct memory_notify *mhp = v;
3903 unsigned long long start, end;
3904 unsigned long start_vpfn, last_vpfn;
3905
3906 switch (val) {
3907 case MEM_GOING_ONLINE:
3908 start = mhp->start_pfn << PAGE_SHIFT;
3909 end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1;
3910 if (iommu_domain_identity_map(si_domain, start, end)) {
3911 pr_warn("dmar: failed to build identity map for [%llx-%llx]\n",
3912 start, end);
3913 return NOTIFY_BAD;
3914 }
3915 break;
3916
3917 case MEM_OFFLINE:
3918 case MEM_CANCEL_ONLINE:
3919 start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
3920 last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1);
3921 while (start_vpfn <= last_vpfn) {
3922 struct iova *iova;
3923 struct dmar_drhd_unit *drhd;
3924 struct intel_iommu *iommu;
ea8ea460 3925 struct page *freelist;
75f05569
JL		 3926
3927 iova = find_iova(&si_domain->iovad, start_vpfn);
3928 if (iova == NULL) {
3929 pr_debug("dmar: failed get IOVA for PFN %lx\n",
3930 start_vpfn);
3931 break;
3932 }
3933
3934 iova = split_and_remove_iova(&si_domain->iovad, iova,
3935 start_vpfn, last_vpfn);
3936 if (iova == NULL) {
3937 pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n",
3938 start_vpfn, last_vpfn);
3939 return NOTIFY_BAD;
3940 }
3941
ea8ea460
DW		 3942 freelist = domain_unmap(si_domain, iova->pfn_lo,
3943 iova->pfn_hi);
3944
75f05569
JL		 3945 rcu_read_lock();
3946 for_each_active_iommu(iommu, drhd)
3947 iommu_flush_iotlb_psi(iommu, si_domain->id,
3948 iova->pfn_lo,
ea8ea460
DW		 3949 iova->pfn_hi - iova->pfn_lo + 1,
3950 !freelist, 0);
75f05569 3951 rcu_read_unlock();
ea8ea460 3952 dma_free_pagelist(freelist);
75f05569
JL		 3953
3954 start_vpfn = iova->pfn_hi + 1;
3955 free_iova_mem(iova);
3956 }
3957 break;
3958 }
3959
3960 return NOTIFY_OK;
3961}
3962
3963static struct notifier_block intel_iommu_memory_nb = {
3964 .notifier_call = intel_iommu_memory_notifier,
3965 .priority = 0
3966};
3967
a5459cfe
AW		 3968
3969static ssize_t intel_iommu_show_version(struct device *dev,
3970 struct device_attribute *attr,
3971 char *buf)
3972{
3973 struct intel_iommu *iommu = dev_get_drvdata(dev);
3974 u32 ver = readl(iommu->reg + DMAR_VER_REG);
3975 return sprintf(buf, "%d:%d\n",
3976 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
3977}
3978static DEVICE_ATTR(version, S_IRUGO, intel_iommu_show_version, NULL);
3979
3980static ssize_t intel_iommu_show_address(struct device *dev,
3981 struct device_attribute *attr,
3982 char *buf)
3983{
3984 struct intel_iommu *iommu = dev_get_drvdata(dev);
3985 return sprintf(buf, "%llx\n", iommu->reg_phys);
3986}
3987static DEVICE_ATTR(address, S_IRUGO, intel_iommu_show_address, NULL);
3988
3989static ssize_t intel_iommu_show_cap(struct device *dev,
3990 struct device_attribute *attr,
3991 char *buf)
3992{
3993 struct intel_iommu *iommu = dev_get_drvdata(dev);
3994 return sprintf(buf, "%llx\n", iommu->cap);
3995}
3996static DEVICE_ATTR(cap, S_IRUGO, intel_iommu_show_cap, NULL);
3997
3998static ssize_t intel_iommu_show_ecap(struct device *dev,
3999 struct device_attribute *attr,
4000 char *buf)
4001{
4002 struct intel_iommu *iommu = dev_get_drvdata(dev);
4003 return sprintf(buf, "%llx\n", iommu->ecap);
4004}
4005static DEVICE_ATTR(ecap, S_IRUGO, intel_iommu_show_ecap, NULL);
4006
4007static struct attribute *intel_iommu_attrs[] = {
4008 &dev_attr_version.attr,
4009 &dev_attr_address.attr,
4010 &dev_attr_cap.attr,
4011 &dev_attr_ecap.attr,
4012 NULL,
4013};
4014
4015static struct attribute_group intel_iommu_group = {
4016 .name = "intel-iommu",
4017 .attrs = intel_iommu_attrs,
4018};
4019
4020const struct attribute_group *intel_iommu_groups[] = {
4021 &intel_iommu_group,
4022 NULL,
4023};
4024
ba395927
KA		 4025int __init intel_iommu_init(void)
4026{
9bdc531e 4027 int ret = -ENODEV;
3a93c841 4028 struct dmar_drhd_unit *drhd;
7c919779 4029 struct intel_iommu *iommu;
ba395927 4030
a59b50e9
JC		 4031 /* VT-d is required for a TXT/tboot launch, so enforce that */
4032 force_on = tboot_force_iommu();
4033
3a5670e8
JL		 4034 if (iommu_init_mempool()) {
4035 if (force_on)
4036 panic("tboot: Failed to initialize iommu memory\n");
4037 return -ENOMEM;
4038 }
4039
4040 down_write(&dmar_global_lock);
a59b50e9
JC		 4041 if (dmar_table_init()) {
4042 if (force_on)
4043 panic("tboot: Failed to initialize DMAR table\n");
9bdc531e 4044 goto out_free_dmar;
a59b50e9 4045 }
ba395927 4046
3a93c841
TI		 4047 /*
4048 * Disable translation if already enabled prior to OS handover.
4049 */
7c919779 4050 for_each_active_iommu(iommu, drhd)
3a93c841
TI		 4051 if (iommu->gcmd & DMA_GCMD_TE)
4052 iommu_disable_translation(iommu);
3a93c841 4053
c2c7286a 4054 if (dmar_dev_scope_init() < 0) {
a59b50e9
JC		 4055 if (force_on)
4056 panic("tboot: Failed to initialize DMAR device scope\n");
9bdc531e 4057 goto out_free_dmar;
a59b50e9 4058 }
1886e8a9 4059
75f1cdf1 4060 if (no_iommu || dmar_disabled)
9bdc531e 4061 goto out_free_dmar;
2ae21010 4062
318fe7df
SS		 4063 if (list_empty(&dmar_rmrr_units))
4064 printk(KERN_INFO "DMAR: No RMRR found\n");
4065
4066 if (list_empty(&dmar_atsr_units))
4067 printk(KERN_INFO "DMAR: No ATSR found\n");
4068
51a63e67
JC		 4069 if (dmar_init_reserved_ranges()) {
4070 if (force_on)
4071 panic("tboot: Failed to reserve iommu ranges\n");
3a5670e8 4072 goto out_free_reserved_range;
51a63e67 4073 }
ba395927
KA		 4074
4075 init_no_remapping_devices();
4076
b779260b 4077 ret = init_dmars();
ba395927 4078 if (ret) {
a59b50e9
JC		 4079 if (force_on)
4080 panic("tboot: Failed to initialize DMARs\n");
ba395927 4081 printk(KERN_ERR "IOMMU: dmar init failed\n");
9bdc531e 4082 goto out_free_reserved_range;
ba395927 4083 }
3a5670e8 4084 up_write(&dmar_global_lock);
ba395927
KA		 4085 printk(KERN_INFO
4086 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
4087
5e0d2a6f 4088 init_timer(&unmap_timer);
75f1cdf1
FT		 4089#ifdef CONFIG_SWIOTLB
4090 swiotlb = 0;
4091#endif
19943b0e 4092 dma_ops = &intel_dma_ops;
4ed0d3e6 4093
134fac3f 4094 init_iommu_pm_ops();
a8bcbb0d 4095
a5459cfe
AW		 4096 for_each_active_iommu(iommu, drhd)
4097 iommu->iommu_dev = iommu_device_create(NULL, iommu,
4098 intel_iommu_groups,
4099 iommu->name);
4100
4236d97d 4101 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
99dcaded 4102 bus_register_notifier(&pci_bus_type, &device_nb);
75f05569
JL		 4103 if (si_domain && !hw_pass_through)
4104 register_memory_notifier(&intel_iommu_memory_nb);
99dcaded 4105
8bc1f85c
ED		 4106 intel_iommu_enabled = 1;
4107
ba395927 4108 return 0;
9bdc531e
JL		 4109
4110out_free_reserved_range:
4111 put_iova_domain(&reserved_iova_list);
9bdc531e
JL		 4112out_free_dmar:
4113 intel_iommu_free_dmars();
3a5670e8
JL		 4114 up_write(&dmar_global_lock);
4115 iommu_exit_mempool();
9bdc531e 4116 return ret;
ba395927 4117}
e820482c 4118
579305f7
AW		 4119static int iommu_detach_dev_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4120{
4121 struct intel_iommu *iommu = opaque;
4122
4123 iommu_detach_dev(iommu, PCI_BUS_NUM(alias), alias & 0xff);
4124 return 0;
4125}
4126
4127/*
4128 * NB - intel-iommu lacks any sort of reference counting for the users of
4129 * dependent devices. If multiple endpoints have intersecting dependent
4130 * devices, unbinding the driver from any one of them will possibly leave
4131 * the others unable to operate.
4132 */
3199aa6b 4133static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
0bcb3e28 4134 struct device *dev)
3199aa6b 4135{
0bcb3e28 4136 if (!iommu || !dev || !dev_is_pci(dev))
3199aa6b
HW		 4137 return;
4138
579305f7 4139 pci_for_each_dma_alias(to_pci_dev(dev), &iommu_detach_dev_cb, iommu);
3199aa6b
HW		 4140}
4141
2c2e2c38 4142static void domain_remove_one_dev_info(struct dmar_domain *domain,
bf9c9eda 4143 struct device *dev)
c7151a8d 4144{
bca2b916 4145 struct device_domain_info *info, *tmp;
c7151a8d
WH		 4146 struct intel_iommu *iommu;
4147 unsigned long flags;
4148 int found = 0;
156baca8 4149 u8 bus, devfn;
c7151a8d 4150
bf9c9eda 4151 iommu = device_to_iommu(dev, &bus, &devfn);
c7151a8d
WH		 4152 if (!iommu)
4153 return;
4154
4155 spin_lock_irqsave(&device_domain_lock, flags);
bca2b916 4156 list_for_each_entry_safe(info, tmp, &domain->devices, link) {
bf9c9eda
DW		 4157 if (info->iommu == iommu && info->bus == bus &&
4158 info->devfn == devfn) {
109b9b04 4159 unlink_domain_info(info);
c7151a8d
WH		 4160 spin_unlock_irqrestore(&device_domain_lock, flags);
4161
93a23a72 4162 iommu_disable_dev_iotlb(info);
c7151a8d 4163 iommu_detach_dev(iommu, info->bus, info->devfn);
bf9c9eda 4164 iommu_detach_dependent_devices(iommu, dev);
c7151a8d
WH		 4165 free_devinfo_mem(info);
4166
4167 spin_lock_irqsave(&device_domain_lock, flags);
4168
4169 if (found)
4170 break;
4171 else
4172 continue;
4173 }
4174
4175 /* if there is no other devices under the same iommu
4176 * owned by this domain, clear this iommu in iommu_bmp
4177 * update iommu count and coherency
4178 */
8bbc4410 4179 if (info->iommu == iommu)
c7151a8d
WH		 4180 found = 1;
4181 }
4182
3e7abe25
RD		 4183 spin_unlock_irqrestore(&device_domain_lock, flags);
4184
c7151a8d 4185 if (found == 0) {
fb170fb4
JL		 4186 domain_detach_iommu(domain, iommu);
4187 if (!domain_type_is_vm_or_si(domain))
4188 iommu_detach_domain(domain, iommu);
c7151a8d 4189 }
c7151a8d
WH		 4190}
4191
2c2e2c38 4192static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 4193{
4194 int adjust_width;
4195
4196 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
5e98c4b1
WH		 4197 domain_reserve_special_ranges(domain);
4198
4199 /* calculate AGAW */
4200 domain->gaw = guest_width;
4201 adjust_width = guestwidth_to_adjustwidth(guest_width);
4202 domain->agaw = width_to_agaw(adjust_width);
4203
5e98c4b1 4204 domain->iommu_coherency = 0;
c5b15255 4205 domain->iommu_snooping = 0;
6dd9a7c7 4206 domain->iommu_superpage = 0;
fe40f1e0 4207 domain->max_addr = 0;
5e98c4b1
WH		 4208
4209 /* always allocate the top pgd */
4c923d47 4210 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 4211 if (!domain->pgd)
4212 return -ENOMEM;
4213 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4214 return 0;
4215}
4216
5d450806 4217static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 4218{
5d450806 4219 struct dmar_domain *dmar_domain;
38717946 4220
ab8dfe25 4221 dmar_domain = alloc_domain(DOMAIN_FLAG_VIRTUAL_MACHINE);
5d450806 4222 if (!dmar_domain) {
38717946 4223 printk(KERN_ERR
5d450806
JR		 4224 "intel_iommu_domain_init: dmar_domain == NULL\n");
4225 return -ENOMEM;
38717946 4226 }
2c2e2c38 4227 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 4228 printk(KERN_ERR
5d450806 4229 "intel_iommu_domain_init() failed\n");
92d03cc8 4230 domain_exit(dmar_domain);
5d450806 4231 return -ENOMEM;
38717946 4232 }
8140a95d 4233 domain_update_iommu_cap(dmar_domain);
5d450806 4234 domain->priv = dmar_domain;
faa3d6f5 4235
8a0e715b
JR		 4236 domain->geometry.aperture_start = 0;
4237 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4238 domain->geometry.force_aperture = true;
4239
5d450806 4240 return 0;
38717946 4241}
38717946 4242
5d450806 4243static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 4244{
5d450806
JR		 4245 struct dmar_domain *dmar_domain = domain->priv;
4246
4247 domain->priv = NULL;
92d03cc8 4248 domain_exit(dmar_domain);
38717946 4249}
38717946 4250
4c5478c9
JR		 4251static int intel_iommu_attach_device(struct iommu_domain *domain,
4252 struct device *dev)
38717946 4253{
4c5478c9 4254 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0
WH		 4255 struct intel_iommu *iommu;
4256 int addr_width;
156baca8 4257 u8 bus, devfn;
faa3d6f5 4258
7207d8f9
DW		 4259 /* normally dev is not mapped */
4260 if (unlikely(domain_context_mapped(dev))) {
faa3d6f5
WH		 4261 struct dmar_domain *old_domain;
4262
1525a29a 4263 old_domain = find_domain(dev);
faa3d6f5 4264 if (old_domain) {
ab8dfe25 4265 if (domain_type_is_vm_or_si(dmar_domain))
bf9c9eda 4266 domain_remove_one_dev_info(old_domain, dev);
faa3d6f5
WH		 4267 else
4268 domain_remove_dev_info(old_domain);
4269 }
4270 }
4271
156baca8 4272 iommu = device_to_iommu(dev, &bus, &devfn);
fe40f1e0
WH		 4273 if (!iommu)
4274 return -ENODEV;
4275
4276 /* check if this iommu agaw is sufficient for max mapped address */
4277 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 4278 if (addr_width > cap_mgaw(iommu->cap))
4279 addr_width = cap_mgaw(iommu->cap);
4280
4281 if (dmar_domain->max_addr > (1LL << addr_width)) {
4282 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4283 "sufficient for the mapped address (%llx)\n",
a99c47a2 4284 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 4285 return -EFAULT;
4286 }
a99c47a2
TL		 4287 dmar_domain->gaw = addr_width;
4288
4289 /*
4290 * Knock out extra levels of page tables if necessary
4291 */
4292 while (iommu->agaw < dmar_domain->agaw) {
4293 struct dma_pte *pte;
4294
4295 pte = dmar_domain->pgd;
4296 if (dma_pte_present(pte)) {
25cbff16
SY		 4297 dmar_domain->pgd = (struct dma_pte *)
4298 phys_to_virt(dma_pte_addr(pte));
7a661013 4299 free_pgtable_page(pte);
a99c47a2
TL		 4300 }
4301 dmar_domain->agaw--;
4302 }
fe40f1e0 4303
5913c9bf 4304 return domain_add_dev_info(dmar_domain, dev, CONTEXT_TT_MULTI_LEVEL);
38717946 4305}
38717946 4306
4c5478c9
JR		 4307static void intel_iommu_detach_device(struct iommu_domain *domain,
4308 struct device *dev)
38717946 4309{
4c5478c9 4310 struct dmar_domain *dmar_domain = domain->priv;
4c5478c9 4311
bf9c9eda 4312 domain_remove_one_dev_info(dmar_domain, dev);
faa3d6f5 4313}
c7151a8d 4314
b146a1c9
JR		 4315static int intel_iommu_map(struct iommu_domain *domain,
4316 unsigned long iova, phys_addr_t hpa,
5009065d 4317 size_t size, int iommu_prot)
faa3d6f5 4318{
dde57a21 4319 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0 4320 u64 max_addr;
dde57a21 4321 int prot = 0;
faa3d6f5 4322 int ret;
fe40f1e0 4323
dde57a21
JR		 4324 if (iommu_prot & IOMMU_READ)
4325 prot |= DMA_PTE_READ;
4326 if (iommu_prot & IOMMU_WRITE)
4327 prot |= DMA_PTE_WRITE;
9cf06697
SY		 4328 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4329 prot |= DMA_PTE_SNP;
dde57a21 4330
163cc52c 4331 max_addr = iova + size;
dde57a21 4332 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 4333 u64 end;
4334
4335 /* check if minimum agaw is sufficient for mapped address */
8954da1f 4336 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 4337 if (end < max_addr) {
8954da1f 4338 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 4339 "sufficient for the mapped address (%llx)\n",
8954da1f 4340 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 4341 return -EFAULT;
4342 }
dde57a21 4343 dmar_domain->max_addr = max_addr;
fe40f1e0 4344 }
ad051221
DW		 4345 /* Round up size to next multiple of PAGE_SIZE, if it and
4346 the low bits of hpa would take us onto the next page */
88cb6a74 4347 size = aligned_nrpages(hpa, size);
ad051221
DW		 4348 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4349 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 4350 return ret;
38717946 4351}
38717946 4352
5009065d 4353static size_t intel_iommu_unmap(struct iommu_domain *domain,
ea8ea460 4354 unsigned long iova, size_t size)
38717946 4355{
dde57a21 4356 struct dmar_domain *dmar_domain = domain->priv;
ea8ea460
DW		 4357 struct page *freelist = NULL;
4358 struct intel_iommu *iommu;
4359 unsigned long start_pfn, last_pfn;
4360 unsigned int npages;
4361 int iommu_id, num, ndomains, level = 0;
5cf0a76f
DW		 4362
4363 /* Cope with horrid API which requires us to unmap more than the
4364 size argument if it happens to be a large-page mapping. */
4365 if (!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level))
4366 BUG();
4367
4368 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4369 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4b99d352 4370
ea8ea460
DW		 4371 start_pfn = iova >> VTD_PAGE_SHIFT;
4372 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4373
4374 freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);
4375
4376 npages = last_pfn - start_pfn + 1;
4377
4378 for_each_set_bit(iommu_id, dmar_domain->iommu_bmp, g_num_of_iommus) {
4379 iommu = g_iommus[iommu_id];
4380
4381 /*
4382 * find bit position of dmar_domain
4383 */
4384 ndomains = cap_ndoms(iommu->cap);
4385 for_each_set_bit(num, iommu->domain_ids, ndomains) {
4386 if (iommu->domains[num] == dmar_domain)
4387 iommu_flush_iotlb_psi(iommu, num, start_pfn,
4388 npages, !freelist, 0);
4389 }
4390
4391 }
4392
4393 dma_free_pagelist(freelist);
fe40f1e0 4394
163cc52c
DW		 4395 if (dmar_domain->max_addr == iova + size)
4396 dmar_domain->max_addr = iova;
b146a1c9 4397
5cf0a76f 4398 return size;
38717946 4399}
38717946 4400
d14d6577 4401static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
bb5547ac 4402 dma_addr_t iova)
38717946 4403{
d14d6577 4404 struct dmar_domain *dmar_domain = domain->priv;
38717946 4405 struct dma_pte *pte;
5cf0a76f 4406 int level = 0;
faa3d6f5 4407 u64 phys = 0;
38717946 4408
5cf0a76f 4409 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
38717946 4410 if (pte)
faa3d6f5 4411 phys = dma_pte_addr(pte);
38717946 4412
faa3d6f5 4413 return phys;
38717946 4414}
a8bcbb0d 4415
dbb9fd86
SY		 4416static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4417 unsigned long cap)
4418{
4419 struct dmar_domain *dmar_domain = domain->priv;
4420
4421 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4422 return dmar_domain->iommu_snooping;
323f99cb 4423 if (cap == IOMMU_CAP_INTR_REMAP)
95a02e97 4424 return irq_remapping_enabled;
dbb9fd86
SY		 4425
4426 return 0;
4427}
4428
abdfdde2
AW		 4429static int intel_iommu_add_device(struct device *dev)
4430{
a5459cfe 4431 struct intel_iommu *iommu;
abdfdde2 4432 struct iommu_group *group;
156baca8 4433 u8 bus, devfn;
70ae6f0d 4434
a5459cfe
AW		 4435 iommu = device_to_iommu(dev, &bus, &devfn);
4436 if (!iommu)
70ae6f0d
AW		 4437 return -ENODEV;
4438
a5459cfe 4439 iommu_device_link(iommu->iommu_dev, dev);
a4ff1fc2 4440
e17f9ff4 4441 group = iommu_group_get_for_dev(dev);
783f157b 4442
e17f9ff4
AW		 4443 if (IS_ERR(group))
4444 return PTR_ERR(group);
bcb71abe 4445
abdfdde2 4446 iommu_group_put(group);
e17f9ff4 4447 return 0;
abdfdde2 4448}
70ae6f0d 4449
abdfdde2
AW		 4450static void intel_iommu_remove_device(struct device *dev)
4451{
a5459cfe
AW		 4452 struct intel_iommu *iommu;
4453 u8 bus, devfn;
4454
4455 iommu = device_to_iommu(dev, &bus, &devfn);
4456 if (!iommu)
4457 return;
4458
abdfdde2 4459 iommu_group_remove_device(dev);
a5459cfe
AW		 4460
4461 iommu_device_unlink(iommu->iommu_dev, dev);
70ae6f0d
AW		 4462}
4463
b22f6434 4464static const struct iommu_ops intel_iommu_ops = {
a8bcbb0d
JR		 4465 .domain_init = intel_iommu_domain_init,
4466 .domain_destroy = intel_iommu_domain_destroy,
4467 .attach_dev = intel_iommu_attach_device,
4468 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 4469 .map = intel_iommu_map,
4470 .unmap = intel_iommu_unmap,
a8bcbb0d 4471 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 4472 .domain_has_cap = intel_iommu_domain_has_cap,
abdfdde2
AW		 4473 .add_device = intel_iommu_add_device,
4474 .remove_device = intel_iommu_remove_device,
6d1c56a9 4475 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
a8bcbb0d 4476};
9af88143 4477
9452618e
DV		 4478static void quirk_iommu_g4x_gfx(struct pci_dev *dev)
4479{
4480 /* G4x/GM45 integrated gfx dmar support is totally busted. */
4481 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4482 dmar_map_gfx = 0;
4483}
4484
4485DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_g4x_gfx);
4486DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_g4x_gfx);
4487DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_g4x_gfx);
4488DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_g4x_gfx);
4489DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_g4x_gfx);
4490DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_g4x_gfx);
4491DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_g4x_gfx);
4492
d34d6517 4493static void quirk_iommu_rwbf(struct pci_dev *dev)
9af88143
DW		 4494{
4495 /*
4496 * Mobile 4 Series Chipset neglects to set RWBF capability,
210561ff 4497 * but needs it. Same seems to hold for the desktop versions.
9af88143
DW		 4498 */
4499 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4500 rwbf_quirk = 1;
4501}
4502
4503DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
210561ff
DV		 4504DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4505DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4506DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4507DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4508DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4509DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
e0fc7e0b 4510
eecfd57f
AJ		 4511#define GGC 0x52
4512#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4513#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4514#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4515#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4516#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4517#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4518#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4519#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4520
d34d6517 4521static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
9eecabcb
DW		 4522{
4523 unsigned short ggc;
4524
eecfd57f 4525 if (pci_read_config_word(dev, GGC, &ggc))
9eecabcb
DW		 4526 return;
4527
eecfd57f 4528 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
9eecabcb
DW		 4529 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4530 dmar_map_gfx = 0;
6fbcfb3e
DW		 4531 } else if (dmar_map_gfx) {
4532 /* we have to ensure the gfx device is idle before we flush */
4533 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4534 intel_iommu_strict = 1;
4535 }
9eecabcb
DW		 4536}
4537DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4538DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4539DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4540DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4541
e0fc7e0b
DW		 4542/* On Tylersburg chipsets, some BIOSes have been known to enable the
4543 ISOCH DMAR unit for the Azalia sound device, but not give it any
4544 TLB entries, which causes it to deadlock. Check for that. We do
4545 this in a function called from init_dmars(), instead of in a PCI
4546 quirk, because we don't want to print the obnoxious "BIOS broken"
4547 message if VT-d is actually disabled.
4548*/
4549static void __init check_tylersburg_isoch(void)
4550{
4551 struct pci_dev *pdev;
4552 uint32_t vtisochctrl;
4553
4554 /* If there's no Azalia in the system anyway, forget it. */
4555 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4556 if (!pdev)
4557 return;
4558 pci_dev_put(pdev);
4559
4560 /* System Management Registers. Might be hidden, in which case
4561 we can't do the sanity check. But that's OK, because the
4562 known-broken BIOSes _don't_ actually hide it, so far. */
4563 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4564 if (!pdev)
4565 return;
4566
4567 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4568 pci_dev_put(pdev);
4569 return;
4570 }
4571
4572 pci_dev_put(pdev);
4573
4574 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4575 if (vtisochctrl & 1)
4576 return;
4577
4578 /* Drop all bits other than the number of TLB entries */
4579 vtisochctrl &= 0x1c;
4580
4581 /* If we have the recommended number of TLB entries (16), fine. */
4582 if (vtisochctrl == 0x10)
4583 return;
4584
4585 /* Zero TLB entries? You get to ride the short bus to school. */
4586 if (!vtisochctrl) {
4587 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4588 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4589 dmi_get_system_info(DMI_BIOS_VENDOR),
4590 dmi_get_system_info(DMI_BIOS_VERSION),
4591 dmi_get_system_info(DMI_PRODUCT_VERSION));
4592 iommu_identity_mapping |= IDENTMAP_AZALIA;
4593 return;
4594 }
4595
4596 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4597 vtisochctrl);
4598}
ubuntu-zesty-kernel mirror