]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - drivers/pci/intel-iommu.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
intel-iommu: Clean up intel_map_sg(), remove domain_page_mapping()
[mirror_ubuntu-jammy-kernel.git] / drivers / pci / intel-iommu.c
CommitLineData
ba395927
KA		 1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
98bcef56 17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
5b6985ce 21 * Author: Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 22 */
23
24#include <linux/init.h>
25#include <linux/bitmap.h>
5e0d2a6f 26#include <linux/debugfs.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
5e0d2a6f 35#include <linux/timer.h>
38717946 36#include <linux/iova.h>
5d450806 37#include <linux/iommu.h>
38717946 38#include <linux/intel-iommu.h>
f59c7b69 39#include <linux/sysdev.h>
ba395927 40#include <asm/cacheflush.h>
46a7fa27 41#include <asm/iommu.h>
ba395927
KA		 42#include "pci.h"
43
5b6985ce
FY		 44#define ROOT_SIZE VTD_PAGE_SIZE
45#define CONTEXT_SIZE VTD_PAGE_SIZE
46
ba395927
KA		 47#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
48#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
49
50#define IOAPIC_RANGE_START (0xfee00000)
51#define IOAPIC_RANGE_END (0xfeefffff)
52#define IOVA_START_ADDR (0x1000)
53
54#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
55
4ed0d3e6
FY		 56#define MAX_AGAW_WIDTH 64
57
ba395927 58#define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
595badf5 59#define DOMAIN_MAX_PFN(gaw) ((((u64)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
ba395927 60
f27be03b 61#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 62#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 63#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 64
fd18de50
DW		 65#ifndef PHYSICAL_PAGE_MASK
66#define PHYSICAL_PAGE_MASK PAGE_MASK
67#endif
68
dd4e8319
DW		 69/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
70 are never going to work. */
71static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
72{
73 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
74}
75
76static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
77{
78 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
79}
80static inline unsigned long page_to_dma_pfn(struct page *pg)
81{
82 return mm_to_dma_pfn(page_to_pfn(pg));
83}
84static inline unsigned long virt_to_dma_pfn(void *p)
85{
86 return page_to_dma_pfn(virt_to_page(p));
87}
88
d9630fe9
WH		 89/* global iommu list, set NULL for ignored DMAR units */
90static struct intel_iommu **g_iommus;
91
9af88143
DW		 92static int rwbf_quirk;
93
46b08e1a
MM		 94/*
95 * 0: Present
96 * 1-11: Reserved
97 * 12-63: Context Ptr (12 - (haw-1))
98 * 64-127: Reserved
99 */
100struct root_entry {
101 u64 val;
102 u64 rsvd1;
103};
104#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
105static inline bool root_present(struct root_entry *root)
106{
107 return (root->val & 1);
108}
109static inline void set_root_present(struct root_entry *root)
110{
111 root->val |= 1;
112}
113static inline void set_root_value(struct root_entry *root, unsigned long value)
114{
115 root->val |= value & VTD_PAGE_MASK;
116}
117
118static inline struct context_entry *
119get_context_addr_from_root(struct root_entry *root)
120{
121 return (struct context_entry *)
122 (root_present(root)?phys_to_virt(
123 root->val & VTD_PAGE_MASK) :
124 NULL);
125}
126
7a8fc25e
MM		 127/*
128 * low 64 bits:
129 * 0: present
130 * 1: fault processing disable
131 * 2-3: translation type
132 * 12-63: address space root
133 * high 64 bits:
134 * 0-2: address width
135 * 3-6: aval
136 * 8-23: domain id
137 */
138struct context_entry {
139 u64 lo;
140 u64 hi;
141};
c07e7d21
MM		 142
143static inline bool context_present(struct context_entry *context)
144{
145 return (context->lo & 1);
146}
147static inline void context_set_present(struct context_entry *context)
148{
149 context->lo |= 1;
150}
151
152static inline void context_set_fault_enable(struct context_entry *context)
153{
154 context->lo &= (((u64)-1) << 2) | 1;
155}
156
c07e7d21
MM		 157static inline void context_set_translation_type(struct context_entry *context,
158 unsigned long value)
159{
160 context->lo &= (((u64)-1) << 4) | 3;
161 context->lo |= (value & 3) << 2;
162}
163
164static inline void context_set_address_root(struct context_entry *context,
165 unsigned long value)
166{
167 context->lo |= value & VTD_PAGE_MASK;
168}
169
170static inline void context_set_address_width(struct context_entry *context,
171 unsigned long value)
172{
173 context->hi |= value & 7;
174}
175
176static inline void context_set_domain_id(struct context_entry *context,
177 unsigned long value)
178{
179 context->hi |= (value & ((1 << 16) - 1)) << 8;
180}
181
182static inline void context_clear_entry(struct context_entry *context)
183{
184 context->lo = 0;
185 context->hi = 0;
186}
7a8fc25e 187
622ba12a
MM		 188/*
189 * 0: readable
190 * 1: writable
191 * 2-6: reserved
192 * 7: super page
9cf06697
SY		 193 * 8-10: available
194 * 11: snoop behavior
622ba12a
MM		 195 * 12-63: Host physcial address
196 */
197struct dma_pte {
198 u64 val;
199};
622ba12a 200
19c239ce
MM		 201static inline void dma_clear_pte(struct dma_pte *pte)
202{
203 pte->val = 0;
204}
205
206static inline void dma_set_pte_readable(struct dma_pte *pte)
207{
208 pte->val |= DMA_PTE_READ;
209}
210
211static inline void dma_set_pte_writable(struct dma_pte *pte)
212{
213 pte->val |= DMA_PTE_WRITE;
214}
215
9cf06697
SY		 216static inline void dma_set_pte_snp(struct dma_pte *pte)
217{
218 pte->val |= DMA_PTE_SNP;
219}
220
19c239ce
MM		 221static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
222{
223 pte->val = (pte->val & ~3) | (prot & 3);
224}
225
226static inline u64 dma_pte_addr(struct dma_pte *pte)
227{
228 return (pte->val & VTD_PAGE_MASK);
229}
230
dd4e8319 231static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
19c239ce 232{
dd4e8319 233 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
19c239ce
MM		 234}
235
236static inline bool dma_pte_present(struct dma_pte *pte)
237{
238 return (pte->val & 3) != 0;
239}
622ba12a 240
2c2e2c38
FY		 241/*
242 * This domain is a statically identity mapping domain.
243 * 1. This domain creats a static 1:1 mapping to all usable memory.
244 * 2. It maps to each iommu if successful.
245 * 3. Each iommu mapps to this domain if successful.
246 */
247struct dmar_domain *si_domain;
248
3b5410e7 249/* devices under the same p2p bridge are owned in one domain */
cdc7b837 250#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
3b5410e7 251
1ce28feb
WH		 252/* domain represents a virtual machine, more than one devices
253 * across iommus may be owned in one domain, e.g. kvm guest.
254 */
255#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
256
2c2e2c38
FY		 257/* si_domain contains mulitple devices */
258#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
259
99126f7c
MM		 260struct dmar_domain {
261 int id; /* domain id */
8c11e798 262 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
99126f7c
MM		 263
264 struct list_head devices; /* all devices' list */
265 struct iova_domain iovad; /* iova's that belong to this domain */
266
267 struct dma_pte *pgd; /* virtual address */
268 spinlock_t mapping_lock; /* page table lock */
269 int gaw; /* max guest address width */
270
271 /* adjusted guest address width, 0 is level 2 30-bit */
272 int agaw;
273
3b5410e7 274 int flags; /* flags to find out type of domain */
8e604097
WH		 275
276 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 277 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d
WH		 278 int iommu_count; /* reference count of iommu */
279 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 280 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 281};
282
a647dacb
MM		 283/* PCI domain-device relationship */
284struct device_domain_info {
285 struct list_head link; /* link to domain siblings */
286 struct list_head global; /* link to global list */
276dbf99
DW		 287 int segment; /* PCI domain */
288 u8 bus; /* PCI bus number */
a647dacb
MM		 289 u8 devfn; /* PCI devfn number */
290 struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
93a23a72 291 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 292 struct dmar_domain *domain; /* pointer to domain */
293};
294
5e0d2a6f 295static void flush_unmaps_timeout(unsigned long data);
296
297DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
298
80b20dd8 299#define HIGH_WATER_MARK 250
300struct deferred_flush_tables {
301 int next;
302 struct iova *iova[HIGH_WATER_MARK];
303 struct dmar_domain *domain[HIGH_WATER_MARK];
304};
305
306static struct deferred_flush_tables *deferred_flush;
307
5e0d2a6f 308/* bitmap for indexing intel_iommus */
5e0d2a6f 309static int g_num_of_iommus;
310
311static DEFINE_SPINLOCK(async_umap_flush_lock);
312static LIST_HEAD(unmaps_to_do);
313
314static int timer_on;
315static long list_size;
5e0d2a6f 316
ba395927
KA		 317static void domain_remove_dev_info(struct dmar_domain *domain);
318
0cd5c3c8
KM		 319#ifdef CONFIG_DMAR_DEFAULT_ON
320int dmar_disabled = 0;
321#else
322int dmar_disabled = 1;
323#endif /*CONFIG_DMAR_DEFAULT_ON*/
324
ba395927 325static int __initdata dmar_map_gfx = 1;
7d3b03ce 326static int dmar_forcedac;
5e0d2a6f 327static int intel_iommu_strict;
ba395927
KA		 328
329#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
330static DEFINE_SPINLOCK(device_domain_lock);
331static LIST_HEAD(device_domain_list);
332
a8bcbb0d
JR		 333static struct iommu_ops intel_iommu_ops;
334
ba395927
KA		 335static int __init intel_iommu_setup(char *str)
336{
337 if (!str)
338 return -EINVAL;
339 while (*str) {
0cd5c3c8
KM		 340 if (!strncmp(str, "on", 2)) {
341 dmar_disabled = 0;
342 printk(KERN_INFO "Intel-IOMMU: enabled\n");
343 } else if (!strncmp(str, "off", 3)) {
ba395927 344 dmar_disabled = 1;
0cd5c3c8 345 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 346 } else if (!strncmp(str, "igfx_off", 8)) {
347 dmar_map_gfx = 0;
348 printk(KERN_INFO
349 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 350 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 351 printk(KERN_INFO
7d3b03ce
KA		 352 "Intel-IOMMU: Forcing DAC for PCI devices\n");
353 dmar_forcedac = 1;
5e0d2a6f 354 } else if (!strncmp(str, "strict", 6)) {
355 printk(KERN_INFO
356 "Intel-IOMMU: disable batched IOTLB flush\n");
357 intel_iommu_strict = 1;
ba395927
KA		 358 }
359
360 str += strcspn(str, ",");
361 while (*str == ',')
362 str++;
363 }
364 return 0;
365}
366__setup("intel_iommu=", intel_iommu_setup);
367
368static struct kmem_cache *iommu_domain_cache;
369static struct kmem_cache *iommu_devinfo_cache;
370static struct kmem_cache *iommu_iova_cache;
371
eb3fa7cb
KA		 372static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
373{
374 unsigned int flags;
375 void *vaddr;
376
377 /* trying to avoid low memory issues */
378 flags = current->flags & PF_MEMALLOC;
379 current->flags |= PF_MEMALLOC;
380 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
381 current->flags &= (~PF_MEMALLOC | flags);
382 return vaddr;
383}
384
385
ba395927
KA		 386static inline void *alloc_pgtable_page(void)
387{
eb3fa7cb
KA		 388 unsigned int flags;
389 void *vaddr;
390
391 /* trying to avoid low memory issues */
392 flags = current->flags & PF_MEMALLOC;
393 current->flags |= PF_MEMALLOC;
394 vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
395 current->flags &= (~PF_MEMALLOC | flags);
396 return vaddr;
ba395927
KA		 397}
398
399static inline void free_pgtable_page(void *vaddr)
400{
401 free_page((unsigned long)vaddr);
402}
403
404static inline void *alloc_domain_mem(void)
405{
eb3fa7cb 406 return iommu_kmem_cache_alloc(iommu_domain_cache);
ba395927
KA		 407}
408
38717946 409static void free_domain_mem(void *vaddr)
ba395927
KA		 410{
411 kmem_cache_free(iommu_domain_cache, vaddr);
412}
413
414static inline void * alloc_devinfo_mem(void)
415{
eb3fa7cb 416 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
ba395927
KA		 417}
418
419static inline void free_devinfo_mem(void *vaddr)
420{
421 kmem_cache_free(iommu_devinfo_cache, vaddr);
422}
423
424struct iova *alloc_iova_mem(void)
425{
eb3fa7cb 426 return iommu_kmem_cache_alloc(iommu_iova_cache);
ba395927
KA		 427}
428
429void free_iova_mem(struct iova *iova)
430{
431 kmem_cache_free(iommu_iova_cache, iova);
432}
433
1b573683
WH		 434
435static inline int width_to_agaw(int width);
436
4ed0d3e6 437static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 438{
439 unsigned long sagaw;
440 int agaw = -1;
441
442 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 443 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 444 agaw >= 0; agaw--) {
445 if (test_bit(agaw, &sagaw))
446 break;
447 }
448
449 return agaw;
450}
451
4ed0d3e6
FY		 452/*
453 * Calculate max SAGAW for each iommu.
454 */
455int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
456{
457 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
458}
459
460/*
461 * calculate agaw for each iommu.
462 * "SAGAW" may be different across iommus, use a default agaw, and
463 * get a supported less agaw for iommus that don't support the default agaw.
464 */
465int iommu_calculate_agaw(struct intel_iommu *iommu)
466{
467 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
468}
469
2c2e2c38 470/* This functionin only returns single iommu in a domain */
8c11e798
WH		 471static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
472{
473 int iommu_id;
474
2c2e2c38 475 /* si_domain and vm domain should not get here. */
1ce28feb 476 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
2c2e2c38 477 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
1ce28feb 478
8c11e798
WH		 479 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
480 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
481 return NULL;
482
483 return g_iommus[iommu_id];
484}
485
8e604097
WH		 486static void domain_update_iommu_coherency(struct dmar_domain *domain)
487{
488 int i;
489
490 domain->iommu_coherency = 1;
491
492 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
493 for (; i < g_num_of_iommus; ) {
494 if (!ecap_coherent(g_iommus[i]->ecap)) {
495 domain->iommu_coherency = 0;
496 break;
497 }
498 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
499 }
500}
501
58c610bd
SY		 502static void domain_update_iommu_snooping(struct dmar_domain *domain)
503{
504 int i;
505
506 domain->iommu_snooping = 1;
507
508 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
509 for (; i < g_num_of_iommus; ) {
510 if (!ecap_sc_support(g_iommus[i]->ecap)) {
511 domain->iommu_snooping = 0;
512 break;
513 }
514 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
515 }
516}
517
518/* Some capabilities may be different across iommus */
519static void domain_update_iommu_cap(struct dmar_domain *domain)
520{
521 domain_update_iommu_coherency(domain);
522 domain_update_iommu_snooping(domain);
523}
524
276dbf99 525static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
c7151a8d
WH		 526{
527 struct dmar_drhd_unit *drhd = NULL;
528 int i;
529
530 for_each_drhd_unit(drhd) {
531 if (drhd->ignored)
532 continue;
276dbf99
DW		 533 if (segment != drhd->segment)
534 continue;
c7151a8d 535
924b6231 536 for (i = 0; i < drhd->devices_cnt; i++) {
288e4877
DH		 537 if (drhd->devices[i] &&
538 drhd->devices[i]->bus->number == bus &&
c7151a8d
WH		 539 drhd->devices[i]->devfn == devfn)
540 return drhd->iommu;
4958c5dc
DW		 541 if (drhd->devices[i] &&
542 drhd->devices[i]->subordinate &&
924b6231
DW		 543 drhd->devices[i]->subordinate->number <= bus &&
544 drhd->devices[i]->subordinate->subordinate >= bus)
545 return drhd->iommu;
546 }
c7151a8d
WH		 547
548 if (drhd->include_all)
549 return drhd->iommu;
550 }
551
552 return NULL;
553}
554
5331fe6f
WH		 555static void domain_flush_cache(struct dmar_domain *domain,
556 void *addr, int size)
557{
558 if (!domain->iommu_coherency)
559 clflush_cache_range(addr, size);
560}
561
ba395927
KA		 562/* Gets context entry for a given bus and devfn */
563static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
564 u8 bus, u8 devfn)
565{
566 struct root_entry *root;
567 struct context_entry *context;
568 unsigned long phy_addr;
569 unsigned long flags;
570
571 spin_lock_irqsave(&iommu->lock, flags);
572 root = &iommu->root_entry[bus];
573 context = get_context_addr_from_root(root);
574 if (!context) {
575 context = (struct context_entry *)alloc_pgtable_page();
576 if (!context) {
577 spin_unlock_irqrestore(&iommu->lock, flags);
578 return NULL;
579 }
5b6985ce 580 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 581 phy_addr = virt_to_phys((void *)context);
582 set_root_value(root, phy_addr);
583 set_root_present(root);
584 __iommu_flush_cache(iommu, root, sizeof(*root));
585 }
586 spin_unlock_irqrestore(&iommu->lock, flags);
587 return &context[devfn];
588}
589
590static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
591{
592 struct root_entry *root;
593 struct context_entry *context;
594 int ret;
595 unsigned long flags;
596
597 spin_lock_irqsave(&iommu->lock, flags);
598 root = &iommu->root_entry[bus];
599 context = get_context_addr_from_root(root);
600 if (!context) {
601 ret = 0;
602 goto out;
603 }
c07e7d21 604 ret = context_present(&context[devfn]);
ba395927
KA		 605out:
606 spin_unlock_irqrestore(&iommu->lock, flags);
607 return ret;
608}
609
610static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
611{
612 struct root_entry *root;
613 struct context_entry *context;
614 unsigned long flags;
615
616 spin_lock_irqsave(&iommu->lock, flags);
617 root = &iommu->root_entry[bus];
618 context = get_context_addr_from_root(root);
619 if (context) {
c07e7d21 620 context_clear_entry(&context[devfn]);
ba395927
KA		 621 __iommu_flush_cache(iommu, &context[devfn], \
622 sizeof(*context));
623 }
624 spin_unlock_irqrestore(&iommu->lock, flags);
625}
626
627static void free_context_table(struct intel_iommu *iommu)
628{
629 struct root_entry *root;
630 int i;
631 unsigned long flags;
632 struct context_entry *context;
633
634 spin_lock_irqsave(&iommu->lock, flags);
635 if (!iommu->root_entry) {
636 goto out;
637 }
638 for (i = 0; i < ROOT_ENTRY_NR; i++) {
639 root = &iommu->root_entry[i];
640 context = get_context_addr_from_root(root);
641 if (context)
642 free_pgtable_page(context);
643 }
644 free_pgtable_page(iommu->root_entry);
645 iommu->root_entry = NULL;
646out:
647 spin_unlock_irqrestore(&iommu->lock, flags);
648}
649
650/* page table handling */
651#define LEVEL_STRIDE (9)
652#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
653
654static inline int agaw_to_level(int agaw)
655{
656 return agaw + 2;
657}
658
659static inline int agaw_to_width(int agaw)
660{
661 return 30 + agaw * LEVEL_STRIDE;
662
663}
664
665static inline int width_to_agaw(int width)
666{
667 return (width - 30) / LEVEL_STRIDE;
668}
669
670static inline unsigned int level_to_offset_bits(int level)
671{
6660c63a 672 return (level - 1) * LEVEL_STRIDE;
ba395927
KA		 673}
674
77dfa56c 675static inline int pfn_level_offset(unsigned long pfn, int level)
ba395927 676{
6660c63a 677 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
ba395927
KA		 678}
679
6660c63a 680static inline unsigned long level_mask(int level)
ba395927 681{
6660c63a 682 return -1UL << level_to_offset_bits(level);
ba395927
KA		 683}
684
6660c63a 685static inline unsigned long level_size(int level)
ba395927 686{
6660c63a 687 return 1UL << level_to_offset_bits(level);
ba395927
KA		 688}
689
6660c63a 690static inline unsigned long align_to_level(unsigned long pfn, int level)
ba395927 691{
6660c63a 692 return (pfn + level_size(level) - 1) & level_mask(level);
ba395927
KA		 693}
694
b026fd28
DW		 695static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
696 unsigned long pfn)
ba395927 697{
b026fd28 698 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927
KA		 699 struct dma_pte *parent, *pte = NULL;
700 int level = agaw_to_level(domain->agaw);
701 int offset;
702 unsigned long flags;
703
704 BUG_ON(!domain->pgd);
b026fd28 705 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
ba395927
KA		 706 parent = domain->pgd;
707
708 spin_lock_irqsave(&domain->mapping_lock, flags);
709 while (level > 0) {
710 void *tmp_page;
711
b026fd28 712 offset = pfn_level_offset(pfn, level);
ba395927
KA		 713 pte = &parent[offset];
714 if (level == 1)
715 break;
716
19c239ce 717 if (!dma_pte_present(pte)) {
ba395927
KA		 718 tmp_page = alloc_pgtable_page();
719
720 if (!tmp_page) {
721 spin_unlock_irqrestore(&domain->mapping_lock,
722 flags);
723 return NULL;
724 }
5331fe6f 725 domain_flush_cache(domain, tmp_page, PAGE_SIZE);
dd4e8319 726 dma_set_pte_pfn(pte, virt_to_dma_pfn(tmp_page));
ba395927
KA		 727 /*
728 * high level table always sets r/w, last level page
729 * table control read/write
730 */
19c239ce
MM		 731 dma_set_pte_readable(pte);
732 dma_set_pte_writable(pte);
5331fe6f 733 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927 734 }
19c239ce 735 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 736 level--;
737 }
738
739 spin_unlock_irqrestore(&domain->mapping_lock, flags);
740 return pte;
741}
742
743/* return address's pte at specific level */
90dcfb5e
DW		 744static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
745 unsigned long pfn,
746 int level)
ba395927
KA		 747{
748 struct dma_pte *parent, *pte = NULL;
749 int total = agaw_to_level(domain->agaw);
750 int offset;
751
752 parent = domain->pgd;
753 while (level <= total) {
90dcfb5e 754 offset = pfn_level_offset(pfn, total);
ba395927
KA		 755 pte = &parent[offset];
756 if (level == total)
757 return pte;
758
19c239ce 759 if (!dma_pte_present(pte))
ba395927 760 break;
19c239ce 761 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 762 total--;
763 }
764 return NULL;
765}
766
767/* clear one page's page table */
a75f7cf9 768static void dma_pte_clear_one(struct dmar_domain *domain, unsigned long pfn)
ba395927
KA		 769{
770 struct dma_pte *pte = NULL;
771
772 /* get last level pte */
a75f7cf9 773 pte = dma_pfn_level_pte(domain, pfn, 1);
ba395927
KA		 774
775 if (pte) {
19c239ce 776 dma_clear_pte(pte);
5331fe6f 777 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927
KA		 778 }
779}
780
781/* clear last level pte, a tlb flush should be followed */
595badf5
DW		 782static void dma_pte_clear_range(struct dmar_domain *domain,
783 unsigned long start_pfn,
784 unsigned long last_pfn)
ba395927 785{
04b18e65 786 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
66eae846 787
04b18e65 788 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
595badf5 789 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
ba395927 790
04b18e65 791 /* we don't need lock here; nobody else touches the iova range */
595badf5 792 while (start_pfn <= last_pfn) {
04b18e65
DW		 793 dma_pte_clear_one(domain, start_pfn);
794 start_pfn++;
ba395927
KA		 795 }
796}
797
798/* free page table pages. last level pte should already be cleared */
799static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 800 unsigned long start_pfn,
801 unsigned long last_pfn)
ba395927 802{
6660c63a 803 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927
KA		 804 struct dma_pte *pte;
805 int total = agaw_to_level(domain->agaw);
806 int level;
6660c63a 807 unsigned long tmp;
ba395927 808
6660c63a
DW		 809 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
810 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
ba395927
KA		 811
812 /* we don't need lock here, nobody else touches the iova range */
813 level = 2;
814 while (level <= total) {
6660c63a
DW		 815 tmp = align_to_level(start_pfn, level);
816
817 /* Only clear this pte/pmd if we're asked to clear its
818 _whole_ range */
819 if (tmp + level_size(level) - 1 > last_pfn)
ba395927
KA		 820 return;
821
6660c63a
DW		 822 while (tmp <= last_pfn) {
823 pte = dma_pfn_level_pte(domain, tmp, level);
ba395927
KA		 824 if (pte) {
825 free_pgtable_page(
19c239ce
MM		 826 phys_to_virt(dma_pte_addr(pte)));
827 dma_clear_pte(pte);
5331fe6f 828 domain_flush_cache(domain, pte, sizeof(*pte));
ba395927
KA		 829 }
830 tmp += level_size(level);
831 }
832 level++;
833 }
834 /* free pgd */
d794dc9b 835 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 836 free_pgtable_page(domain->pgd);
837 domain->pgd = NULL;
838 }
839}
840
841/* iommu handling */
842static int iommu_alloc_root_entry(struct intel_iommu *iommu)
843{
844 struct root_entry *root;
845 unsigned long flags;
846
847 root = (struct root_entry *)alloc_pgtable_page();
848 if (!root)
849 return -ENOMEM;
850
5b6985ce 851 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 852
853 spin_lock_irqsave(&iommu->lock, flags);
854 iommu->root_entry = root;
855 spin_unlock_irqrestore(&iommu->lock, flags);
856
857 return 0;
858}
859
ba395927
KA		 860static void iommu_set_root_entry(struct intel_iommu *iommu)
861{
862 void *addr;
c416daa9 863 u32 sts;
ba395927
KA		 864 unsigned long flag;
865
866 addr = iommu->root_entry;
867
868 spin_lock_irqsave(&iommu->register_lock, flag);
869 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
870
c416daa9 871 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 872
873 /* Make sure hardware complete it */
874 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 875 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927
KA		 876
877 spin_unlock_irqrestore(&iommu->register_lock, flag);
878}
879
880static void iommu_flush_write_buffer(struct intel_iommu *iommu)
881{
882 u32 val;
883 unsigned long flag;
884
9af88143 885 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 886 return;
ba395927
KA		 887
888 spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 889 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 890
891 /* Make sure hardware complete it */
892 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 893 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927
KA		 894
895 spin_unlock_irqrestore(&iommu->register_lock, flag);
896}
897
898/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 899static void __iommu_flush_context(struct intel_iommu *iommu,
900 u16 did, u16 source_id, u8 function_mask,
901 u64 type)
ba395927
KA		 902{
903 u64 val = 0;
904 unsigned long flag;
905
ba395927
KA		 906 switch (type) {
907 case DMA_CCMD_GLOBAL_INVL:
908 val = DMA_CCMD_GLOBAL_INVL;
909 break;
910 case DMA_CCMD_DOMAIN_INVL:
911 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
912 break;
913 case DMA_CCMD_DEVICE_INVL:
914 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
915 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
916 break;
917 default:
918 BUG();
919 }
920 val |= DMA_CCMD_ICC;
921
922 spin_lock_irqsave(&iommu->register_lock, flag);
923 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
924
925 /* Make sure hardware complete it */
926 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
927 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
928
929 spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 930}
931
ba395927 932/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 933static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
934 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 935{
936 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
937 u64 val = 0, val_iva = 0;
938 unsigned long flag;
939
ba395927
KA		 940 switch (type) {
941 case DMA_TLB_GLOBAL_FLUSH:
942 /* global flush doesn't need set IVA_REG */
943 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
944 break;
945 case DMA_TLB_DSI_FLUSH:
946 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
947 break;
948 case DMA_TLB_PSI_FLUSH:
949 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
950 /* Note: always flush non-leaf currently */
951 val_iva = size_order | addr;
952 break;
953 default:
954 BUG();
955 }
956 /* Note: set drain read/write */
957#if 0
958 /*
959 * This is probably to be super secure.. Looks like we can
960 * ignore it without any impact.
961 */
962 if (cap_read_drain(iommu->cap))
963 val |= DMA_TLB_READ_DRAIN;
964#endif
965 if (cap_write_drain(iommu->cap))
966 val |= DMA_TLB_WRITE_DRAIN;
967
968 spin_lock_irqsave(&iommu->register_lock, flag);
969 /* Note: Only uses first TLB reg currently */
970 if (val_iva)
971 dmar_writeq(iommu->reg + tlb_offset, val_iva);
972 dmar_writeq(iommu->reg + tlb_offset + 8, val);
973
974 /* Make sure hardware complete it */
975 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
976 dmar_readq, (!(val & DMA_TLB_IVT)), val);
977
978 spin_unlock_irqrestore(&iommu->register_lock, flag);
979
980 /* check IOTLB invalidation granularity */
981 if (DMA_TLB_IAIG(val) == 0)
982 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
983 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
984 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 985 (unsigned long long)DMA_TLB_IIRG(type),
986 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 987}
988
93a23a72
YZ		 989static struct device_domain_info *iommu_support_dev_iotlb(
990 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
991{
992 int found = 0;
993 unsigned long flags;
994 struct device_domain_info *info;
995 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
996
997 if (!ecap_dev_iotlb_support(iommu->ecap))
998 return NULL;
999
1000 if (!iommu->qi)
1001 return NULL;
1002
1003 spin_lock_irqsave(&device_domain_lock, flags);
1004 list_for_each_entry(info, &domain->devices, link)
1005 if (info->bus == bus && info->devfn == devfn) {
1006 found = 1;
1007 break;
1008 }
1009 spin_unlock_irqrestore(&device_domain_lock, flags);
1010
1011 if (!found || !info->dev)
1012 return NULL;
1013
1014 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1015 return NULL;
1016
1017 if (!dmar_find_matched_atsr_unit(info->dev))
1018 return NULL;
1019
1020 info->iommu = iommu;
1021
1022 return info;
1023}
1024
1025static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1026{
93a23a72
YZ		 1027 if (!info)
1028 return;
1029
1030 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1031}
1032
1033static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1034{
1035 if (!info->dev || !pci_ats_enabled(info->dev))
1036 return;
1037
1038 pci_disable_ats(info->dev);
1039}
1040
1041static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1042 u64 addr, unsigned mask)
1043{
1044 u16 sid, qdep;
1045 unsigned long flags;
1046 struct device_domain_info *info;
1047
1048 spin_lock_irqsave(&device_domain_lock, flags);
1049 list_for_each_entry(info, &domain->devices, link) {
1050 if (!info->dev || !pci_ats_enabled(info->dev))
1051 continue;
1052
1053 sid = info->bus << 8 | info->devfn;
1054 qdep = pci_ats_queue_depth(info->dev);
1055 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1056 }
1057 spin_unlock_irqrestore(&device_domain_lock, flags);
1058}
1059
1f0ef2aa
DW		 1060static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1061 u64 addr, unsigned int pages)
ba395927 1062{
9dd2fe89 1063 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
ba395927 1064
5b6985ce 1065 BUG_ON(addr & (~VTD_PAGE_MASK));
ba395927
KA		 1066 BUG_ON(pages == 0);
1067
ba395927 1068 /*
9dd2fe89
YZ		 1069 * Fallback to domain selective flush if no PSI support or the size is
1070 * too big.
ba395927
KA		 1071 * PSI requires page size to be 2 ^ x, and the base address is naturally
1072 * aligned to the size
1073 */
9dd2fe89
YZ		 1074 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1075 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1076 DMA_TLB_DSI_FLUSH);
9dd2fe89
YZ		 1077 else
1078 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1079 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1080
1081 /*
1082 * In caching mode, domain ID 0 is reserved for non-present to present
1083 * mapping flush. Device IOTLB doesn't need to be flushed in this case.
1084 */
1085 if (!cap_caching_mode(iommu->cap) || did)
93a23a72 1086 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1087}
1088
f8bab735 1089static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1090{
1091 u32 pmen;
1092 unsigned long flags;
1093
1094 spin_lock_irqsave(&iommu->register_lock, flags);
1095 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1096 pmen &= ~DMA_PMEN_EPM;
1097 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1098
1099 /* wait for the protected region status bit to clear */
1100 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1101 readl, !(pmen & DMA_PMEN_PRS), pmen);
1102
1103 spin_unlock_irqrestore(&iommu->register_lock, flags);
1104}
1105
ba395927
KA		 1106static int iommu_enable_translation(struct intel_iommu *iommu)
1107{
1108 u32 sts;
1109 unsigned long flags;
1110
1111 spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1112 iommu->gcmd |= DMA_GCMD_TE;
1113 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1114
1115 /* Make sure hardware complete it */
1116 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1117 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1118
ba395927
KA		 1119 spin_unlock_irqrestore(&iommu->register_lock, flags);
1120 return 0;
1121}
1122
1123static int iommu_disable_translation(struct intel_iommu *iommu)
1124{
1125 u32 sts;
1126 unsigned long flag;
1127
1128 spin_lock_irqsave(&iommu->register_lock, flag);
1129 iommu->gcmd &= ~DMA_GCMD_TE;
1130 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1131
1132 /* Make sure hardware complete it */
1133 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1134 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927
KA		 1135
1136 spin_unlock_irqrestore(&iommu->register_lock, flag);
1137 return 0;
1138}
1139
3460a6d9 1140
ba395927
KA		 1141static int iommu_init_domains(struct intel_iommu *iommu)
1142{
1143 unsigned long ndomains;
1144 unsigned long nlongs;
1145
1146 ndomains = cap_ndoms(iommu->cap);
1147 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1148 nlongs = BITS_TO_LONGS(ndomains);
1149
1150 /* TBD: there might be 64K domains,
1151 * consider other allocation for future chip
1152 */
1153 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1154 if (!iommu->domain_ids) {
1155 printk(KERN_ERR "Allocating domain id array failed\n");
1156 return -ENOMEM;
1157 }
1158 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1159 GFP_KERNEL);
1160 if (!iommu->domains) {
1161 printk(KERN_ERR "Allocating domain array failed\n");
1162 kfree(iommu->domain_ids);
1163 return -ENOMEM;
1164 }
1165
e61d98d8
SS		 1166 spin_lock_init(&iommu->lock);
1167
ba395927
KA		 1168 /*
1169 * if Caching mode is set, then invalid translations are tagged
1170 * with domainid 0. Hence we need to pre-allocate it.
1171 */
1172 if (cap_caching_mode(iommu->cap))
1173 set_bit(0, iommu->domain_ids);
1174 return 0;
1175}
ba395927 1176
ba395927
KA		 1177
1178static void domain_exit(struct dmar_domain *domain);
5e98c4b1 1179static void vm_domain_exit(struct dmar_domain *domain);
e61d98d8
SS		 1180
1181void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1182{
1183 struct dmar_domain *domain;
1184 int i;
c7151a8d 1185 unsigned long flags;
ba395927 1186
ba395927
KA		 1187 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1188 for (; i < cap_ndoms(iommu->cap); ) {
1189 domain = iommu->domains[i];
1190 clear_bit(i, iommu->domain_ids);
c7151a8d
WH		 1191
1192 spin_lock_irqsave(&domain->iommu_lock, flags);
5e98c4b1
WH		 1193 if (--domain->iommu_count == 0) {
1194 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1195 vm_domain_exit(domain);
1196 else
1197 domain_exit(domain);
1198 }
c7151a8d
WH		 1199 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1200
ba395927
KA		 1201 i = find_next_bit(iommu->domain_ids,
1202 cap_ndoms(iommu->cap), i+1);
1203 }
1204
1205 if (iommu->gcmd & DMA_GCMD_TE)
1206 iommu_disable_translation(iommu);
1207
1208 if (iommu->irq) {
1209 set_irq_data(iommu->irq, NULL);
1210 /* This will mask the irq */
1211 free_irq(iommu->irq, iommu);
1212 destroy_irq(iommu->irq);
1213 }
1214
1215 kfree(iommu->domains);
1216 kfree(iommu->domain_ids);
1217
d9630fe9
WH		 1218 g_iommus[iommu->seq_id] = NULL;
1219
1220 /* if all iommus are freed, free g_iommus */
1221 for (i = 0; i < g_num_of_iommus; i++) {
1222 if (g_iommus[i])
1223 break;
1224 }
1225
1226 if (i == g_num_of_iommus)
1227 kfree(g_iommus);
1228
ba395927
KA		 1229 /* free context mapping */
1230 free_context_table(iommu);
ba395927
KA		 1231}
1232
2c2e2c38 1233static struct dmar_domain *alloc_domain(void)
ba395927 1234{
ba395927 1235 struct dmar_domain *domain;
ba395927
KA		 1236
1237 domain = alloc_domain_mem();
1238 if (!domain)
1239 return NULL;
1240
2c2e2c38
FY		 1241 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1242 domain->flags = 0;
1243
1244 return domain;
1245}
1246
1247static int iommu_attach_domain(struct dmar_domain *domain,
1248 struct intel_iommu *iommu)
1249{
1250 int num;
1251 unsigned long ndomains;
1252 unsigned long flags;
1253
ba395927
KA		 1254 ndomains = cap_ndoms(iommu->cap);
1255
1256 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1257
ba395927
KA		 1258 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1259 if (num >= ndomains) {
1260 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1261 printk(KERN_ERR "IOMMU: no free domain ids\n");
2c2e2c38 1262 return -ENOMEM;
ba395927
KA		 1263 }
1264
ba395927 1265 domain->id = num;
2c2e2c38 1266 set_bit(num, iommu->domain_ids);
8c11e798 1267 set_bit(iommu->seq_id, &domain->iommu_bmp);
ba395927
KA		 1268 iommu->domains[num] = domain;
1269 spin_unlock_irqrestore(&iommu->lock, flags);
1270
2c2e2c38 1271 return 0;
ba395927
KA		 1272}
1273
2c2e2c38
FY		 1274static void iommu_detach_domain(struct dmar_domain *domain,
1275 struct intel_iommu *iommu)
ba395927
KA		 1276{
1277 unsigned long flags;
2c2e2c38
FY		 1278 int num, ndomains;
1279 int found = 0;
ba395927 1280
8c11e798 1281 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38
FY		 1282 ndomains = cap_ndoms(iommu->cap);
1283 num = find_first_bit(iommu->domain_ids, ndomains);
1284 for (; num < ndomains; ) {
1285 if (iommu->domains[num] == domain) {
1286 found = 1;
1287 break;
1288 }
1289 num = find_next_bit(iommu->domain_ids,
1290 cap_ndoms(iommu->cap), num+1);
1291 }
1292
1293 if (found) {
1294 clear_bit(num, iommu->domain_ids);
1295 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1296 iommu->domains[num] = NULL;
1297 }
8c11e798 1298 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1299}
1300
1301static struct iova_domain reserved_iova_list;
8a443df4
MG		 1302static struct lock_class_key reserved_alloc_key;
1303static struct lock_class_key reserved_rbtree_key;
ba395927
KA		 1304
1305static void dmar_init_reserved_ranges(void)
1306{
1307 struct pci_dev *pdev = NULL;
1308 struct iova *iova;
1309 int i;
1310 u64 addr, size;
1311
f661197e 1312 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1313
8a443df4
MG		 1314 lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1315 &reserved_alloc_key);
1316 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1317 &reserved_rbtree_key);
1318
ba395927
KA		 1319 /* IOAPIC ranges shouldn't be accessed by DMA */
1320 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1321 IOVA_PFN(IOAPIC_RANGE_END));
1322 if (!iova)
1323 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1324
1325 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1326 for_each_pci_dev(pdev) {
1327 struct resource *r;
1328
1329 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1330 r = &pdev->resource[i];
1331 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1332 continue;
1333 addr = r->start;
fd18de50 1334 addr &= PHYSICAL_PAGE_MASK;
ba395927 1335 size = r->end - addr;
5b6985ce 1336 size = PAGE_ALIGN(size);
ba395927
KA		 1337 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1338 IOVA_PFN(size + addr) - 1);
1339 if (!iova)
1340 printk(KERN_ERR "Reserve iova failed\n");
1341 }
1342 }
1343
1344}
1345
1346static void domain_reserve_special_ranges(struct dmar_domain *domain)
1347{
1348 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1349}
1350
1351static inline int guestwidth_to_adjustwidth(int gaw)
1352{
1353 int agaw;
1354 int r = (gaw - 12) % 9;
1355
1356 if (r == 0)
1357 agaw = gaw;
1358 else
1359 agaw = gaw + 9 - r;
1360 if (agaw > 64)
1361 agaw = 64;
1362 return agaw;
1363}
1364
1365static int domain_init(struct dmar_domain *domain, int guest_width)
1366{
1367 struct intel_iommu *iommu;
1368 int adjust_width, agaw;
1369 unsigned long sagaw;
1370
f661197e 1371 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
ba395927 1372 spin_lock_init(&domain->mapping_lock);
c7151a8d 1373 spin_lock_init(&domain->iommu_lock);
ba395927
KA		 1374
1375 domain_reserve_special_ranges(domain);
1376
1377 /* calculate AGAW */
8c11e798 1378 iommu = domain_get_iommu(domain);
ba395927
KA		 1379 if (guest_width > cap_mgaw(iommu->cap))
1380 guest_width = cap_mgaw(iommu->cap);
1381 domain->gaw = guest_width;
1382 adjust_width = guestwidth_to_adjustwidth(guest_width);
1383 agaw = width_to_agaw(adjust_width);
1384 sagaw = cap_sagaw(iommu->cap);
1385 if (!test_bit(agaw, &sagaw)) {
1386 /* hardware doesn't support it, choose a bigger one */
1387 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1388 agaw = find_next_bit(&sagaw, 5, agaw);
1389 if (agaw >= 5)
1390 return -ENODEV;
1391 }
1392 domain->agaw = agaw;
1393 INIT_LIST_HEAD(&domain->devices);
1394
8e604097
WH		 1395 if (ecap_coherent(iommu->ecap))
1396 domain->iommu_coherency = 1;
1397 else
1398 domain->iommu_coherency = 0;
1399
58c610bd
SY		 1400 if (ecap_sc_support(iommu->ecap))
1401 domain->iommu_snooping = 1;
1402 else
1403 domain->iommu_snooping = 0;
1404
c7151a8d
WH		 1405 domain->iommu_count = 1;
1406
ba395927
KA		 1407 /* always allocate the top pgd */
1408 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1409 if (!domain->pgd)
1410 return -ENOMEM;
5b6985ce 1411 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1412 return 0;
1413}
1414
1415static void domain_exit(struct dmar_domain *domain)
1416{
2c2e2c38
FY		 1417 struct dmar_drhd_unit *drhd;
1418 struct intel_iommu *iommu;
ba395927
KA		 1419
1420 /* Domain 0 is reserved, so dont process it */
1421 if (!domain)
1422 return;
1423
1424 domain_remove_dev_info(domain);
1425 /* destroy iovas */
1426 put_iova_domain(&domain->iovad);
ba395927
KA		 1427
1428 /* clear ptes */
595badf5 1429 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927
KA		 1430
1431 /* free page tables */
d794dc9b 1432 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1433
2c2e2c38
FY		 1434 for_each_active_iommu(iommu, drhd)
1435 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1436 iommu_detach_domain(domain, iommu);
1437
ba395927
KA		 1438 free_domain_mem(domain);
1439}
1440
4ed0d3e6
FY		 1441static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1442 u8 bus, u8 devfn, int translation)
ba395927
KA		 1443{
1444 struct context_entry *context;
ba395927 1445 unsigned long flags;
5331fe6f 1446 struct intel_iommu *iommu;
ea6606b0
WH		 1447 struct dma_pte *pgd;
1448 unsigned long num;
1449 unsigned long ndomains;
1450 int id;
1451 int agaw;
93a23a72 1452 struct device_domain_info *info = NULL;
ba395927
KA		 1453
1454 pr_debug("Set context mapping for %02x:%02x.%d\n",
1455 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1456
ba395927 1457 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1458 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1459 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1460
276dbf99 1461 iommu = device_to_iommu(segment, bus, devfn);
5331fe6f
WH		 1462 if (!iommu)
1463 return -ENODEV;
1464
ba395927
KA		 1465 context = device_to_context_entry(iommu, bus, devfn);
1466 if (!context)
1467 return -ENOMEM;
1468 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1469 if (context_present(context)) {
ba395927
KA		 1470 spin_unlock_irqrestore(&iommu->lock, flags);
1471 return 0;
1472 }
1473
ea6606b0
WH		 1474 id = domain->id;
1475 pgd = domain->pgd;
1476
2c2e2c38
FY		 1477 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1478 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
ea6606b0
WH		 1479 int found = 0;
1480
1481 /* find an available domain id for this device in iommu */
1482 ndomains = cap_ndoms(iommu->cap);
1483 num = find_first_bit(iommu->domain_ids, ndomains);
1484 for (; num < ndomains; ) {
1485 if (iommu->domains[num] == domain) {
1486 id = num;
1487 found = 1;
1488 break;
1489 }
1490 num = find_next_bit(iommu->domain_ids,
1491 cap_ndoms(iommu->cap), num+1);
1492 }
1493
1494 if (found == 0) {
1495 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1496 if (num >= ndomains) {
1497 spin_unlock_irqrestore(&iommu->lock, flags);
1498 printk(KERN_ERR "IOMMU: no free domain ids\n");
1499 return -EFAULT;
1500 }
1501
1502 set_bit(num, iommu->domain_ids);
2c2e2c38 1503 set_bit(iommu->seq_id, &domain->iommu_bmp);
ea6606b0
WH		 1504 iommu->domains[num] = domain;
1505 id = num;
1506 }
1507
1508 /* Skip top levels of page tables for
1509 * iommu which has less agaw than default.
1510 */
1511 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1512 pgd = phys_to_virt(dma_pte_addr(pgd));
1513 if (!dma_pte_present(pgd)) {
1514 spin_unlock_irqrestore(&iommu->lock, flags);
1515 return -ENOMEM;
1516 }
1517 }
1518 }
1519
1520 context_set_domain_id(context, id);
4ed0d3e6 1521
93a23a72
YZ		 1522 if (translation != CONTEXT_TT_PASS_THROUGH) {
1523 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1524 translation = info ? CONTEXT_TT_DEV_IOTLB :
1525 CONTEXT_TT_MULTI_LEVEL;
1526 }
4ed0d3e6
FY		 1527 /*
1528 * In pass through mode, AW must be programmed to indicate the largest
1529 * AGAW value supported by hardware. And ASR is ignored by hardware.
1530 */
93a23a72 1531 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1532 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1533 else {
1534 context_set_address_root(context, virt_to_phys(pgd));
1535 context_set_address_width(context, iommu->agaw);
1536 }
4ed0d3e6
FY		 1537
1538 context_set_translation_type(context, translation);
c07e7d21
MM		 1539 context_set_fault_enable(context);
1540 context_set_present(context);
5331fe6f 1541 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1542
4c25a2c1
DW		 1543 /*
1544 * It's a non-present to present mapping. If hardware doesn't cache
1545 * non-present entry we only need to flush the write-buffer. If the
1546 * _does_ cache non-present entries, then it does so in the special
1547 * domain #0, which we have to flush:
1548 */
1549 if (cap_caching_mode(iommu->cap)) {
1550 iommu->flush.flush_context(iommu, 0,
1551 (((u16)bus) << 8) | devfn,
1552 DMA_CCMD_MASK_NOBIT,
1553 DMA_CCMD_DEVICE_INVL);
1f0ef2aa 1554 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1555 } else {
ba395927 1556 iommu_flush_write_buffer(iommu);
4c25a2c1 1557 }
93a23a72 1558 iommu_enable_dev_iotlb(info);
ba395927 1559 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d
WH		 1560
1561 spin_lock_irqsave(&domain->iommu_lock, flags);
1562 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1563 domain->iommu_count++;
58c610bd 1564 domain_update_iommu_cap(domain);
c7151a8d
WH		 1565 }
1566 spin_unlock_irqrestore(&domain->iommu_lock, flags);
ba395927
KA		 1567 return 0;
1568}
1569
1570static int
4ed0d3e6
FY		 1571domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1572 int translation)
ba395927
KA		 1573{
1574 int ret;
1575 struct pci_dev *tmp, *parent;
1576
276dbf99 1577 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
4ed0d3e6
FY		 1578 pdev->bus->number, pdev->devfn,
1579 translation);
ba395927
KA		 1580 if (ret)
1581 return ret;
1582
1583 /* dependent device mapping */
1584 tmp = pci_find_upstream_pcie_bridge(pdev);
1585 if (!tmp)
1586 return 0;
1587 /* Secondary interface's bus number and devfn 0 */
1588 parent = pdev->bus->self;
1589 while (parent != tmp) {
276dbf99
DW		 1590 ret = domain_context_mapping_one(domain,
1591 pci_domain_nr(parent->bus),
1592 parent->bus->number,
4ed0d3e6 1593 parent->devfn, translation);
ba395927
KA		 1594 if (ret)
1595 return ret;
1596 parent = parent->bus->self;
1597 }
1598 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1599 return domain_context_mapping_one(domain,
276dbf99 1600 pci_domain_nr(tmp->subordinate),
4ed0d3e6
FY		 1601 tmp->subordinate->number, 0,
1602 translation);
ba395927
KA		 1603 else /* this is a legacy PCI bridge */
1604 return domain_context_mapping_one(domain,
276dbf99
DW		 1605 pci_domain_nr(tmp->bus),
1606 tmp->bus->number,
4ed0d3e6
FY		 1607 tmp->devfn,
1608 translation);
ba395927
KA		 1609}
1610
5331fe6f 1611static int domain_context_mapped(struct pci_dev *pdev)
ba395927
KA		 1612{
1613 int ret;
1614 struct pci_dev *tmp, *parent;
5331fe6f
WH		 1615 struct intel_iommu *iommu;
1616
276dbf99
DW		 1617 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1618 pdev->devfn);
5331fe6f
WH		 1619 if (!iommu)
1620 return -ENODEV;
ba395927 1621
276dbf99 1622 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
ba395927
KA		 1623 if (!ret)
1624 return ret;
1625 /* dependent device mapping */
1626 tmp = pci_find_upstream_pcie_bridge(pdev);
1627 if (!tmp)
1628 return ret;
1629 /* Secondary interface's bus number and devfn 0 */
1630 parent = pdev->bus->self;
1631 while (parent != tmp) {
8c11e798 1632 ret = device_context_mapped(iommu, parent->bus->number,
276dbf99 1633 parent->devfn);
ba395927
KA		 1634 if (!ret)
1635 return ret;
1636 parent = parent->bus->self;
1637 }
1638 if (tmp->is_pcie)
276dbf99
DW		 1639 return device_context_mapped(iommu, tmp->subordinate->number,
1640 0);
ba395927 1641 else
276dbf99
DW		 1642 return device_context_mapped(iommu, tmp->bus->number,
1643 tmp->devfn);
ba395927
KA		 1644}
1645
61df7443
DW		 1646static int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1647 unsigned long phys_pfn, unsigned long nr_pages,
1648 int prot)
ba395927 1649{
ba395927 1650 struct dma_pte *pte;
1c5a46ed 1651 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
5b6985ce 1652
61df7443 1653 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
ba395927
KA		 1654
1655 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1656 return -EINVAL;
1c5a46ed 1657
61df7443
DW		 1658 while (nr_pages--) {
1659 pte = pfn_to_dma_pte(domain, iov_pfn);
ba395927
KA		 1660 if (!pte)
1661 return -ENOMEM;
1662 /* We don't need lock here, nobody else
1663 * touches the iova range
1664 */
19c239ce 1665 BUG_ON(dma_pte_addr(pte));
61df7443 1666 dma_set_pte_pfn(pte, phys_pfn);
19c239ce 1667 dma_set_pte_prot(pte, prot);
9cf06697
SY		 1668 if (prot & DMA_PTE_SNP)
1669 dma_set_pte_snp(pte);
5331fe6f 1670 domain_flush_cache(domain, pte, sizeof(*pte));
61df7443
DW		 1671 iov_pfn++;
1672 phys_pfn++;
ba395927
KA		 1673 }
1674 return 0;
1675}
1676
c7151a8d 1677static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 1678{
c7151a8d
WH		 1679 if (!iommu)
1680 return;
8c11e798
WH		 1681
1682 clear_context_table(iommu, bus, devfn);
1683 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 1684 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 1685 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 1686}
1687
1688static void domain_remove_dev_info(struct dmar_domain *domain)
1689{
1690 struct device_domain_info *info;
1691 unsigned long flags;
c7151a8d 1692 struct intel_iommu *iommu;
ba395927
KA		 1693
1694 spin_lock_irqsave(&device_domain_lock, flags);
1695 while (!list_empty(&domain->devices)) {
1696 info = list_entry(domain->devices.next,
1697 struct device_domain_info, link);
1698 list_del(&info->link);
1699 list_del(&info->global);
1700 if (info->dev)
358dd8ac 1701 info->dev->dev.archdata.iommu = NULL;
ba395927
KA		 1702 spin_unlock_irqrestore(&device_domain_lock, flags);
1703
93a23a72 1704 iommu_disable_dev_iotlb(info);
276dbf99 1705 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 1706 iommu_detach_dev(iommu, info->bus, info->devfn);
ba395927
KA		 1707 free_devinfo_mem(info);
1708
1709 spin_lock_irqsave(&device_domain_lock, flags);
1710 }
1711 spin_unlock_irqrestore(&device_domain_lock, flags);
1712}
1713
1714/*
1715 * find_domain
358dd8ac 1716 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
ba395927 1717 */
38717946 1718static struct dmar_domain *
ba395927
KA		 1719find_domain(struct pci_dev *pdev)
1720{
1721 struct device_domain_info *info;
1722
1723 /* No lock here, assumes no domain exit in normal case */
358dd8ac 1724 info = pdev->dev.archdata.iommu;
ba395927
KA		 1725 if (info)
1726 return info->domain;
1727 return NULL;
1728}
1729
ba395927
KA		 1730/* domain is initialized */
1731static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1732{
1733 struct dmar_domain *domain, *found = NULL;
1734 struct intel_iommu *iommu;
1735 struct dmar_drhd_unit *drhd;
1736 struct device_domain_info *info, *tmp;
1737 struct pci_dev *dev_tmp;
1738 unsigned long flags;
1739 int bus = 0, devfn = 0;
276dbf99 1740 int segment;
2c2e2c38 1741 int ret;
ba395927
KA		 1742
1743 domain = find_domain(pdev);
1744 if (domain)
1745 return domain;
1746
276dbf99
DW		 1747 segment = pci_domain_nr(pdev->bus);
1748
ba395927
KA		 1749 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1750 if (dev_tmp) {
1751 if (dev_tmp->is_pcie) {
1752 bus = dev_tmp->subordinate->number;
1753 devfn = 0;
1754 } else {
1755 bus = dev_tmp->bus->number;
1756 devfn = dev_tmp->devfn;
1757 }
1758 spin_lock_irqsave(&device_domain_lock, flags);
1759 list_for_each_entry(info, &device_domain_list, global) {
276dbf99
DW		 1760 if (info->segment == segment &&
1761 info->bus == bus && info->devfn == devfn) {
ba395927
KA		 1762 found = info->domain;
1763 break;
1764 }
1765 }
1766 spin_unlock_irqrestore(&device_domain_lock, flags);
1767 /* pcie-pci bridge already has a domain, uses it */
1768 if (found) {
1769 domain = found;
1770 goto found_domain;
1771 }
1772 }
1773
2c2e2c38
FY		 1774 domain = alloc_domain();
1775 if (!domain)
1776 goto error;
1777
ba395927
KA		 1778 /* Allocate new domain for the device */
1779 drhd = dmar_find_matched_drhd_unit(pdev);
1780 if (!drhd) {
1781 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1782 pci_name(pdev));
1783 return NULL;
1784 }
1785 iommu = drhd->iommu;
1786
2c2e2c38
FY		 1787 ret = iommu_attach_domain(domain, iommu);
1788 if (ret) {
1789 domain_exit(domain);
ba395927 1790 goto error;
2c2e2c38 1791 }
ba395927
KA		 1792
1793 if (domain_init(domain, gaw)) {
1794 domain_exit(domain);
1795 goto error;
1796 }
1797
1798 /* register pcie-to-pci device */
1799 if (dev_tmp) {
1800 info = alloc_devinfo_mem();
1801 if (!info) {
1802 domain_exit(domain);
1803 goto error;
1804 }
276dbf99 1805 info->segment = segment;
ba395927
KA		 1806 info->bus = bus;
1807 info->devfn = devfn;
1808 info->dev = NULL;
1809 info->domain = domain;
1810 /* This domain is shared by devices under p2p bridge */
3b5410e7 1811 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
ba395927
KA		 1812
1813 /* pcie-to-pci bridge already has a domain, uses it */
1814 found = NULL;
1815 spin_lock_irqsave(&device_domain_lock, flags);
1816 list_for_each_entry(tmp, &device_domain_list, global) {
276dbf99
DW		 1817 if (tmp->segment == segment &&
1818 tmp->bus == bus && tmp->devfn == devfn) {
ba395927
KA		 1819 found = tmp->domain;
1820 break;
1821 }
1822 }
1823 if (found) {
1824 free_devinfo_mem(info);
1825 domain_exit(domain);
1826 domain = found;
1827 } else {
1828 list_add(&info->link, &domain->devices);
1829 list_add(&info->global, &device_domain_list);
1830 }
1831 spin_unlock_irqrestore(&device_domain_lock, flags);
1832 }
1833
1834found_domain:
1835 info = alloc_devinfo_mem();
1836 if (!info)
1837 goto error;
276dbf99 1838 info->segment = segment;
ba395927
KA		 1839 info->bus = pdev->bus->number;
1840 info->devfn = pdev->devfn;
1841 info->dev = pdev;
1842 info->domain = domain;
1843 spin_lock_irqsave(&device_domain_lock, flags);
1844 /* somebody is fast */
1845 found = find_domain(pdev);
1846 if (found != NULL) {
1847 spin_unlock_irqrestore(&device_domain_lock, flags);
1848 if (found != domain) {
1849 domain_exit(domain);
1850 domain = found;
1851 }
1852 free_devinfo_mem(info);
1853 return domain;
1854 }
1855 list_add(&info->link, &domain->devices);
1856 list_add(&info->global, &device_domain_list);
358dd8ac 1857 pdev->dev.archdata.iommu = info;
ba395927
KA		 1858 spin_unlock_irqrestore(&device_domain_lock, flags);
1859 return domain;
1860error:
1861 /* recheck it here, maybe others set it */
1862 return find_domain(pdev);
1863}
1864
2c2e2c38
FY		 1865static int iommu_identity_mapping;
1866
b213203e
DW		 1867static int iommu_domain_identity_map(struct dmar_domain *domain,
1868 unsigned long long start,
1869 unsigned long long end)
ba395927 1870{
ba395927 1871 unsigned long size;
5b6985ce 1872 unsigned long long base;
ba395927
KA		 1873
1874 /* The address might not be aligned */
5b6985ce 1875 base = start & PAGE_MASK;
ba395927 1876 size = end - base;
5b6985ce 1877 size = PAGE_ALIGN(size);
ba395927
KA		 1878 if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1879 IOVA_PFN(base + size) - 1)) {
1880 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 1881 return -ENOMEM;
ba395927
KA		 1882 }
1883
b213203e
DW		 1884 pr_debug("Mapping reserved region %lx@%llx for domain %d\n",
1885 size, base, domain->id);
ba395927
KA		 1886 /*
1887 * RMRR range might have overlap with physical memory range,
1888 * clear it first
1889 */
595badf5
DW		 1890 dma_pte_clear_range(domain, base >> VTD_PAGE_SHIFT,
1891 (base + size - 1) >> VTD_PAGE_SHIFT);
ba395927 1892
61df7443
DW		 1893 return domain_pfn_mapping(domain, base >> VTD_PAGE_SHIFT,
1894 base >> VTD_PAGE_SHIFT,
1895 size >> VTD_PAGE_SHIFT,
1896 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 1897}
1898
1899static int iommu_prepare_identity_map(struct pci_dev *pdev,
1900 unsigned long long start,
1901 unsigned long long end)
1902{
1903 struct dmar_domain *domain;
1904 int ret;
1905
1906 printk(KERN_INFO
1907 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1908 pci_name(pdev), start, end);
1909
c7ab48d2 1910 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 1911 if (!domain)
1912 return -ENOMEM;
1913
1914 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 1915 if (ret)
1916 goto error;
1917
1918 /* context entry init */
4ed0d3e6 1919 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 1920 if (ret)
1921 goto error;
1922
1923 return 0;
1924
1925 error:
ba395927
KA		 1926 domain_exit(domain);
1927 return ret;
ba395927
KA		 1928}
1929
1930static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1931 struct pci_dev *pdev)
1932{
358dd8ac 1933 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927
KA		 1934 return 0;
1935 return iommu_prepare_identity_map(pdev, rmrr->base_address,
1936 rmrr->end_address + 1);
1937}
1938
49a0429e
KA		 1939#ifdef CONFIG_DMAR_FLOPPY_WA
1940static inline void iommu_prepare_isa(void)
1941{
1942 struct pci_dev *pdev;
1943 int ret;
1944
1945 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1946 if (!pdev)
1947 return;
1948
c7ab48d2 1949 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
49a0429e
KA		 1950 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1951
1952 if (ret)
c7ab48d2
DW		 1953 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
1954 "floppy might not work\n");
49a0429e
KA		 1955
1956}
1957#else
1958static inline void iommu_prepare_isa(void)
1959{
1960 return;
1961}
1962#endif /* !CONFIG_DMAR_FLPY_WA */
1963
4ed0d3e6
FY		 1964/* Initialize each context entry as pass through.*/
1965static int __init init_context_pass_through(void)
1966{
1967 struct pci_dev *pdev = NULL;
1968 struct dmar_domain *domain;
1969 int ret;
1970
1971 for_each_pci_dev(pdev) {
1972 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1973 ret = domain_context_mapping(domain, pdev,
1974 CONTEXT_TT_PASS_THROUGH);
1975 if (ret)
1976 return ret;
1977 }
1978 return 0;
1979}
1980
2c2e2c38 1981static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2
DW		 1982
1983static int __init si_domain_work_fn(unsigned long start_pfn,
1984 unsigned long end_pfn, void *datax)
1985{
1986 int *ret = datax;
1987
1988 *ret = iommu_domain_identity_map(si_domain,
1989 (uint64_t)start_pfn << PAGE_SHIFT,
1990 (uint64_t)end_pfn << PAGE_SHIFT);
1991 return *ret;
1992
1993}
1994
2c2e2c38
FY		 1995static int si_domain_init(void)
1996{
1997 struct dmar_drhd_unit *drhd;
1998 struct intel_iommu *iommu;
c7ab48d2 1999 int nid, ret = 0;
2c2e2c38
FY		 2000
2001 si_domain = alloc_domain();
2002 if (!si_domain)
2003 return -EFAULT;
2004
c7ab48d2 2005 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2c2e2c38
FY		 2006
2007 for_each_active_iommu(iommu, drhd) {
2008 ret = iommu_attach_domain(si_domain, iommu);
2009 if (ret) {
2010 domain_exit(si_domain);
2011 return -EFAULT;
2012 }
2013 }
2014
2015 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2016 domain_exit(si_domain);
2017 return -EFAULT;
2018 }
2019
2020 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2021
c7ab48d2
DW		 2022 for_each_online_node(nid) {
2023 work_with_active_regions(nid, si_domain_work_fn, &ret);
2024 if (ret)
2025 return ret;
2026 }
2027
2c2e2c38
FY		 2028 return 0;
2029}
2030
2031static void domain_remove_one_dev_info(struct dmar_domain *domain,
2032 struct pci_dev *pdev);
2033static int identity_mapping(struct pci_dev *pdev)
2034{
2035 struct device_domain_info *info;
2036
2037 if (likely(!iommu_identity_mapping))
2038 return 0;
2039
2040
2041 list_for_each_entry(info, &si_domain->devices, link)
2042 if (info->dev == pdev)
2043 return 1;
2044 return 0;
2045}
2046
2047static int domain_add_dev_info(struct dmar_domain *domain,
2048 struct pci_dev *pdev)
2049{
2050 struct device_domain_info *info;
2051 unsigned long flags;
2052
2053 info = alloc_devinfo_mem();
2054 if (!info)
2055 return -ENOMEM;
2056
2057 info->segment = pci_domain_nr(pdev->bus);
2058 info->bus = pdev->bus->number;
2059 info->devfn = pdev->devfn;
2060 info->dev = pdev;
2061 info->domain = domain;
2062
2063 spin_lock_irqsave(&device_domain_lock, flags);
2064 list_add(&info->link, &domain->devices);
2065 list_add(&info->global, &device_domain_list);
2066 pdev->dev.archdata.iommu = info;
2067 spin_unlock_irqrestore(&device_domain_lock, flags);
2068
2069 return 0;
2070}
2071
2072static int iommu_prepare_static_identity_mapping(void)
2073{
2c2e2c38
FY		 2074 struct pci_dev *pdev = NULL;
2075 int ret;
2076
2077 ret = si_domain_init();
2078 if (ret)
2079 return -EFAULT;
2080
2c2e2c38 2081 for_each_pci_dev(pdev) {
c7ab48d2
DW		 2082 printk(KERN_INFO "IOMMU: identity mapping for device %s\n",
2083 pci_name(pdev));
2084
2085 ret = domain_context_mapping(si_domain, pdev,
2086 CONTEXT_TT_MULTI_LEVEL);
2087 if (ret)
2088 return ret;
2c2e2c38
FY		 2089 ret = domain_add_dev_info(si_domain, pdev);
2090 if (ret)
2091 return ret;
2092 }
2093
2094 return 0;
2095}
2096
2097int __init init_dmars(void)
ba395927
KA		 2098{
2099 struct dmar_drhd_unit *drhd;
2100 struct dmar_rmrr_unit *rmrr;
2101 struct pci_dev *pdev;
2102 struct intel_iommu *iommu;
9d783ba0 2103 int i, ret;
4ed0d3e6 2104 int pass_through = 1;
ba395927 2105
2c2e2c38
FY		 2106 /*
2107 * In case pass through can not be enabled, iommu tries to use identity
2108 * mapping.
2109 */
2110 if (iommu_pass_through)
2111 iommu_identity_mapping = 1;
2112
ba395927
KA		 2113 /*
2114 * for each drhd
2115 * allocate root
2116 * initialize and program root entry to not present
2117 * endfor
2118 */
2119 for_each_drhd_unit(drhd) {
5e0d2a6f 2120 g_num_of_iommus++;
2121 /*
2122 * lock not needed as this is only incremented in the single
2123 * threaded kernel __init code path all other access are read
2124 * only
2125 */
2126 }
2127
d9630fe9
WH		 2128 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2129 GFP_KERNEL);
2130 if (!g_iommus) {
2131 printk(KERN_ERR "Allocating global iommu array failed\n");
2132 ret = -ENOMEM;
2133 goto error;
2134 }
2135
80b20dd8 2136 deferred_flush = kzalloc(g_num_of_iommus *
2137 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2138 if (!deferred_flush) {
d9630fe9 2139 kfree(g_iommus);
5e0d2a6f 2140 ret = -ENOMEM;
2141 goto error;
2142 }
2143
5e0d2a6f 2144 for_each_drhd_unit(drhd) {
2145 if (drhd->ignored)
2146 continue;
1886e8a9
SS		 2147
2148 iommu = drhd->iommu;
d9630fe9 2149 g_iommus[iommu->seq_id] = iommu;
ba395927 2150
e61d98d8
SS		 2151 ret = iommu_init_domains(iommu);
2152 if (ret)
2153 goto error;
2154
ba395927
KA		 2155 /*
2156 * TBD:
2157 * we could share the same root & context tables
2158 * amoung all IOMMU's. Need to Split it later.
2159 */
2160 ret = iommu_alloc_root_entry(iommu);
2161 if (ret) {
2162 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2163 goto error;
2164 }
4ed0d3e6
FY		 2165 if (!ecap_pass_through(iommu->ecap))
2166 pass_through = 0;
ba395927 2167 }
4ed0d3e6
FY		 2168 if (iommu_pass_through)
2169 if (!pass_through) {
2170 printk(KERN_INFO
2171 "Pass Through is not supported by hardware.\n");
2172 iommu_pass_through = 0;
2173 }
ba395927 2174
1531a6a6
SS		 2175 /*
2176 * Start from the sane iommu hardware state.
2177 */
a77b67d4
YS		 2178 for_each_drhd_unit(drhd) {
2179 if (drhd->ignored)
2180 continue;
2181
2182 iommu = drhd->iommu;
1531a6a6
SS		 2183
2184 /*
2185 * If the queued invalidation is already initialized by us
2186 * (for example, while enabling interrupt-remapping) then
2187 * we got the things already rolling from a sane state.
2188 */
2189 if (iommu->qi)
2190 continue;
2191
2192 /*
2193 * Clear any previous faults.
2194 */
2195 dmar_fault(-1, iommu);
2196 /*
2197 * Disable queued invalidation if supported and already enabled
2198 * before OS handover.
2199 */
2200 dmar_disable_qi(iommu);
2201 }
2202
2203 for_each_drhd_unit(drhd) {
2204 if (drhd->ignored)
2205 continue;
2206
2207 iommu = drhd->iommu;
2208
a77b67d4
YS		 2209 if (dmar_enable_qi(iommu)) {
2210 /*
2211 * Queued Invalidate not enabled, use Register Based
2212 * Invalidate
2213 */
2214 iommu->flush.flush_context = __iommu_flush_context;
2215 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2216 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
b4e0f9eb
FT		 2217 "invalidation\n",
2218 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2219 } else {
2220 iommu->flush.flush_context = qi_flush_context;
2221 iommu->flush.flush_iotlb = qi_flush_iotlb;
2222 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
b4e0f9eb
FT		 2223 "invalidation\n",
2224 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2225 }
2226 }
2227
ba395927 2228 /*
4ed0d3e6
FY		 2229 * If pass through is set and enabled, context entries of all pci
2230 * devices are intialized by pass through translation type.
ba395927 2231 */
4ed0d3e6
FY		 2232 if (iommu_pass_through) {
2233 ret = init_context_pass_through();
2234 if (ret) {
2235 printk(KERN_ERR "IOMMU: Pass through init failed.\n");
2236 iommu_pass_through = 0;
ba395927
KA		 2237 }
2238 }
2239
ba395927 2240 /*
4ed0d3e6 2241 * If pass through is not set or not enabled, setup context entries for
2c2e2c38
FY		 2242 * identity mappings for rmrr, gfx, and isa and may fall back to static
2243 * identity mapping if iommu_identity_mapping is set.
ba395927 2244 */
4ed0d3e6 2245 if (!iommu_pass_through) {
2c2e2c38
FY		 2246 if (iommu_identity_mapping)
2247 iommu_prepare_static_identity_mapping();
4ed0d3e6
FY		 2248 /*
2249 * For each rmrr
2250 * for each dev attached to rmrr
2251 * do
2252 * locate drhd for dev, alloc domain for dev
2253 * allocate free domain
2254 * allocate page table entries for rmrr
2255 * if context not allocated for bus
2256 * allocate and init context
2257 * set present in root table for this bus
2258 * init context with domain, translation etc
2259 * endfor
2260 * endfor
2261 */
2c2e2c38 2262 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
4ed0d3e6
FY		 2263 for_each_rmrr_units(rmrr) {
2264 for (i = 0; i < rmrr->devices_cnt; i++) {
2265 pdev = rmrr->devices[i];
2266 /*
2267 * some BIOS lists non-exist devices in DMAR
2268 * table.
2269 */
2270 if (!pdev)
2271 continue;
2272 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2273 if (ret)
2274 printk(KERN_ERR
ba395927 2275 "IOMMU: mapping reserved region failed\n");
4ed0d3e6 2276 }
ba395927 2277 }
ba395927 2278
4ed0d3e6
FY		 2279 iommu_prepare_isa();
2280 }
49a0429e 2281
ba395927
KA		 2282 /*
2283 * for each drhd
2284 * enable fault log
2285 * global invalidate context cache
2286 * global invalidate iotlb
2287 * enable translation
2288 */
2289 for_each_drhd_unit(drhd) {
2290 if (drhd->ignored)
2291 continue;
2292 iommu = drhd->iommu;
ba395927
KA		 2293
2294 iommu_flush_write_buffer(iommu);
2295
3460a6d9
KA		 2296 ret = dmar_set_interrupt(iommu);
2297 if (ret)
2298 goto error;
2299
ba395927
KA		 2300 iommu_set_root_entry(iommu);
2301
4c25a2c1 2302 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2303 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
f8bab735 2304 iommu_disable_protect_mem_regions(iommu);
2305
ba395927
KA		 2306 ret = iommu_enable_translation(iommu);
2307 if (ret)
2308 goto error;
2309 }
2310
2311 return 0;
2312error:
2313 for_each_drhd_unit(drhd) {
2314 if (drhd->ignored)
2315 continue;
2316 iommu = drhd->iommu;
2317 free_iommu(iommu);
2318 }
d9630fe9 2319 kfree(g_iommus);
ba395927
KA		 2320 return ret;
2321}
2322
2323static inline u64 aligned_size(u64 host_addr, size_t size)
2324{
2325 u64 addr;
5b6985ce
FY		 2326 addr = (host_addr & (~PAGE_MASK)) + size;
2327 return PAGE_ALIGN(addr);
ba395927
KA		 2328}
2329
2330struct iova *
f76aec76 2331iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
ba395927 2332{
ba395927
KA		 2333 struct iova *piova;
2334
2335 /* Make sure it's in range */
ba395927 2336 end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
f76aec76 2337 if (!size || (IOVA_START_ADDR + size > end))
ba395927
KA		 2338 return NULL;
2339
2340 piova = alloc_iova(&domain->iovad,
5b6985ce 2341 size >> PAGE_SHIFT, IOVA_PFN(end), 1);
ba395927
KA		 2342 return piova;
2343}
2344
f76aec76
KA		 2345static struct iova *
2346__intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
bb9e6d65 2347 size_t size, u64 dma_mask)
ba395927 2348{
ba395927 2349 struct pci_dev *pdev = to_pci_dev(dev);
ba395927 2350 struct iova *iova = NULL;
ba395927 2351
284901a9 2352 if (dma_mask <= DMA_BIT_MASK(32) || dmar_forcedac)
bb9e6d65
FT		 2353 iova = iommu_alloc_iova(domain, size, dma_mask);
2354 else {
ba395927
KA		 2355 /*
2356 * First try to allocate an io virtual address in
284901a9 2357 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2358 * from higher range
ba395927 2359 */
284901a9 2360 iova = iommu_alloc_iova(domain, size, DMA_BIT_MASK(32));
ba395927 2361 if (!iova)
bb9e6d65 2362 iova = iommu_alloc_iova(domain, size, dma_mask);
ba395927
KA		 2363 }
2364
2365 if (!iova) {
2366 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
f76aec76
KA		 2367 return NULL;
2368 }
2369
2370 return iova;
2371}
2372
2373static struct dmar_domain *
2374get_valid_domain_for_dev(struct pci_dev *pdev)
2375{
2376 struct dmar_domain *domain;
2377 int ret;
2378
2379 domain = get_domain_for_dev(pdev,
2380 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2381 if (!domain) {
2382 printk(KERN_ERR
2383 "Allocating domain for %s failed", pci_name(pdev));
4fe05bbc 2384 return NULL;
ba395927
KA		 2385 }
2386
2387 /* make sure context mapping is ok */
5331fe6f 2388 if (unlikely(!domain_context_mapped(pdev))) {
4ed0d3e6
FY		 2389 ret = domain_context_mapping(domain, pdev,
2390 CONTEXT_TT_MULTI_LEVEL);
f76aec76
KA		 2391 if (ret) {
2392 printk(KERN_ERR
2393 "Domain context map for %s failed",
2394 pci_name(pdev));
4fe05bbc 2395 return NULL;
f76aec76 2396 }
ba395927
KA		 2397 }
2398
f76aec76
KA		 2399 return domain;
2400}
2401
2c2e2c38
FY		 2402static int iommu_dummy(struct pci_dev *pdev)
2403{
2404 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2405}
2406
2407/* Check if the pdev needs to go through non-identity map and unmap process.*/
2408static int iommu_no_mapping(struct pci_dev *pdev)
2409{
2410 int found;
2411
2412 if (!iommu_identity_mapping)
2413 return iommu_dummy(pdev);
2414
2415 found = identity_mapping(pdev);
2416 if (found) {
2417 if (pdev->dma_mask > DMA_BIT_MASK(32))
2418 return 1;
2419 else {
2420 /*
2421 * 32 bit DMA is removed from si_domain and fall back
2422 * to non-identity mapping.
2423 */
2424 domain_remove_one_dev_info(si_domain, pdev);
2425 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2426 pci_name(pdev));
2427 return 0;
2428 }
2429 } else {
2430 /*
2431 * In case of a detached 64 bit DMA device from vm, the device
2432 * is put into si_domain for identity mapping.
2433 */
2434 if (pdev->dma_mask > DMA_BIT_MASK(32)) {
2435 int ret;
2436 ret = domain_add_dev_info(si_domain, pdev);
2437 if (!ret) {
2438 printk(KERN_INFO "64bit %s uses identity mapping\n",
2439 pci_name(pdev));
2440 return 1;
2441 }
2442 }
2443 }
2444
2445 return iommu_dummy(pdev);
2446}
2447
bb9e6d65
FT		 2448static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2449 size_t size, int dir, u64 dma_mask)
f76aec76
KA		 2450{
2451 struct pci_dev *pdev = to_pci_dev(hwdev);
f76aec76 2452 struct dmar_domain *domain;
5b6985ce 2453 phys_addr_t start_paddr;
f76aec76
KA		 2454 struct iova *iova;
2455 int prot = 0;
6865f0d1 2456 int ret;
8c11e798 2457 struct intel_iommu *iommu;
f76aec76
KA		 2458
2459 BUG_ON(dir == DMA_NONE);
2c2e2c38
FY		 2460
2461 if (iommu_no_mapping(pdev))
6865f0d1 2462 return paddr;
f76aec76
KA		 2463
2464 domain = get_valid_domain_for_dev(pdev);
2465 if (!domain)
2466 return 0;
2467
8c11e798 2468 iommu = domain_get_iommu(domain);
0ab36de2 2469 size = aligned_size(paddr, size) >> VTD_PAGE_SHIFT;
f76aec76 2470
0ab36de2 2471 iova = __intel_alloc_iova(hwdev, domain, size << VTD_PAGE_SHIFT, pdev->dma_mask);
f76aec76
KA		 2472 if (!iova)
2473 goto error;
2474
ba395927
KA		 2475 /*
2476 * Check if DMAR supports zero-length reads on write only
2477 * mappings..
2478 */
2479 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2480 !cap_zlr(iommu->cap))
ba395927
KA		 2481 prot |= DMA_PTE_READ;
2482 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2483 prot |= DMA_PTE_WRITE;
2484 /*
6865f0d1 2485 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 2486 * page. Note: if two part of one page are separately mapped, we
6865f0d1 2487 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 2488 * is not a big problem
2489 */
0ab36de2
DW		 2490 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2491 paddr >> VTD_PAGE_SHIFT, size, prot);
ba395927
KA		 2492 if (ret)
2493 goto error;
2494
0ab36de2
DW		 2495 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2496
1f0ef2aa
DW		 2497 /* it's a non-present to present mapping. Only flush if caching mode */
2498 if (cap_caching_mode(iommu->cap))
0ab36de2 2499 iommu_flush_iotlb_psi(iommu, 0, start_paddr, size);
1f0ef2aa 2500 else
8c11e798 2501 iommu_flush_write_buffer(iommu);
f76aec76 2502
0ab36de2 2503 return start_paddr + (paddr & (~PAGE_MASK));
ba395927 2504
ba395927 2505error:
f76aec76
KA		 2506 if (iova)
2507 __free_iova(&domain->iovad, iova);
4cf2e75d 2508 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5b6985ce 2509 pci_name(pdev), size, (unsigned long long)paddr, dir);
ba395927
KA		 2510 return 0;
2511}
2512
ffbbef5c
FT		 2513static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2514 unsigned long offset, size_t size,
2515 enum dma_data_direction dir,
2516 struct dma_attrs *attrs)
bb9e6d65 2517{
ffbbef5c
FT		 2518 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2519 dir, to_pci_dev(dev)->dma_mask);
bb9e6d65
FT		 2520}
2521
5e0d2a6f 2522static void flush_unmaps(void)
2523{
80b20dd8 2524 int i, j;
5e0d2a6f 2525
5e0d2a6f 2526 timer_on = 0;
2527
2528 /* just flush them all */
2529 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 2530 struct intel_iommu *iommu = g_iommus[i];
2531 if (!iommu)
2532 continue;
c42d9f32 2533
9dd2fe89
YZ		 2534 if (!deferred_flush[i].next)
2535 continue;
2536
2537 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 2538 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 2539 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 2540 unsigned long mask;
2541 struct iova *iova = deferred_flush[i].iova[j];
2542
2543 mask = (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT;
2544 mask = ilog2(mask >> VTD_PAGE_SHIFT);
2545 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2546 iova->pfn_lo << PAGE_SHIFT, mask);
2547 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
80b20dd8 2548 }
9dd2fe89 2549 deferred_flush[i].next = 0;
5e0d2a6f 2550 }
2551
5e0d2a6f 2552 list_size = 0;
5e0d2a6f 2553}
2554
2555static void flush_unmaps_timeout(unsigned long data)
2556{
80b20dd8 2557 unsigned long flags;
2558
2559 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 2560 flush_unmaps();
80b20dd8 2561 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 2562}
2563
2564static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2565{
2566 unsigned long flags;
80b20dd8 2567 int next, iommu_id;
8c11e798 2568 struct intel_iommu *iommu;
5e0d2a6f 2569
2570 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 2571 if (list_size == HIGH_WATER_MARK)
2572 flush_unmaps();
2573
8c11e798
WH		 2574 iommu = domain_get_iommu(dom);
2575 iommu_id = iommu->seq_id;
c42d9f32 2576
80b20dd8 2577 next = deferred_flush[iommu_id].next;
2578 deferred_flush[iommu_id].domain[next] = dom;
2579 deferred_flush[iommu_id].iova[next] = iova;
2580 deferred_flush[iommu_id].next++;
5e0d2a6f 2581
2582 if (!timer_on) {
2583 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2584 timer_on = 1;
2585 }
2586 list_size++;
2587 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2588}
2589
ffbbef5c
FT		 2590static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2591 size_t size, enum dma_data_direction dir,
2592 struct dma_attrs *attrs)
ba395927 2593{
ba395927 2594 struct pci_dev *pdev = to_pci_dev(dev);
f76aec76 2595 struct dmar_domain *domain;
d794dc9b 2596 unsigned long start_pfn, last_pfn;
ba395927 2597 struct iova *iova;
8c11e798 2598 struct intel_iommu *iommu;
ba395927 2599
2c2e2c38 2600 if (iommu_no_mapping(pdev))
f76aec76 2601 return;
2c2e2c38 2602
ba395927
KA		 2603 domain = find_domain(pdev);
2604 BUG_ON(!domain);
2605
8c11e798
WH		 2606 iommu = domain_get_iommu(domain);
2607
ba395927 2608 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
f76aec76 2609 if (!iova)
ba395927 2610 return;
ba395927 2611
d794dc9b
DW		 2612 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2613 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 2614
d794dc9b
DW		 2615 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2616 pci_name(pdev), start_pfn, last_pfn);
ba395927 2617
f76aec76 2618 /* clear the whole page */
d794dc9b
DW		 2619 dma_pte_clear_range(domain, start_pfn, last_pfn);
2620
f76aec76 2621 /* free page tables */
d794dc9b
DW		 2622 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2623
5e0d2a6f 2624 if (intel_iommu_strict) {
d794dc9b
DW		 2625 iommu_flush_iotlb_psi(iommu, domain->id,
2626 start_pfn << VTD_PAGE_SHIFT,
2627 last_pfn - start_pfn + 1);
5e0d2a6f 2628 /* free iova */
2629 __free_iova(&domain->iovad, iova);
2630 } else {
2631 add_unmap(domain, iova);
2632 /*
2633 * queue up the release of the unmap to save the 1/6th of the
2634 * cpu used up by the iotlb flush operation...
2635 */
5e0d2a6f 2636 }
ba395927
KA		 2637}
2638
d7ab5c46
FT		 2639static void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
2640 int dir)
ffbbef5c
FT		 2641{
2642 intel_unmap_page(dev, dev_addr, size, dir, NULL);
2643}
2644
d7ab5c46
FT		 2645static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2646 dma_addr_t *dma_handle, gfp_t flags)
ba395927
KA		 2647{
2648 void *vaddr;
2649 int order;
2650
5b6985ce 2651 size = PAGE_ALIGN(size);
ba395927
KA		 2652 order = get_order(size);
2653 flags &= ~(GFP_DMA | GFP_DMA32);
2654
2655 vaddr = (void *)__get_free_pages(flags, order);
2656 if (!vaddr)
2657 return NULL;
2658 memset(vaddr, 0, size);
2659
bb9e6d65
FT		 2660 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2661 DMA_BIDIRECTIONAL,
2662 hwdev->coherent_dma_mask);
ba395927
KA		 2663 if (*dma_handle)
2664 return vaddr;
2665 free_pages((unsigned long)vaddr, order);
2666 return NULL;
2667}
2668
d7ab5c46
FT		 2669static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2670 dma_addr_t dma_handle)
ba395927
KA		 2671{
2672 int order;
2673
5b6985ce 2674 size = PAGE_ALIGN(size);
ba395927
KA		 2675 order = get_order(size);
2676
2677 intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
2678 free_pages((unsigned long)vaddr, order);
2679}
2680
d7ab5c46
FT		 2681static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2682 int nelems, enum dma_data_direction dir,
2683 struct dma_attrs *attrs)
ba395927 2684{
ba395927
KA		 2685 struct pci_dev *pdev = to_pci_dev(hwdev);
2686 struct dmar_domain *domain;
d794dc9b 2687 unsigned long start_pfn, last_pfn;
f76aec76 2688 struct iova *iova;
8c11e798 2689 struct intel_iommu *iommu;
ba395927 2690
2c2e2c38 2691 if (iommu_no_mapping(pdev))
ba395927
KA		 2692 return;
2693
2694 domain = find_domain(pdev);
8c11e798
WH		 2695 BUG_ON(!domain);
2696
2697 iommu = domain_get_iommu(domain);
ba395927 2698
c03ab37c 2699 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
f76aec76
KA		 2700 if (!iova)
2701 return;
f76aec76 2702
d794dc9b
DW		 2703 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2704 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
f76aec76
KA		 2705
2706 /* clear the whole page */
d794dc9b
DW		 2707 dma_pte_clear_range(domain, start_pfn, last_pfn);
2708
f76aec76 2709 /* free page tables */
d794dc9b 2710 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
f76aec76 2711
d794dc9b
DW		 2712 iommu_flush_iotlb_psi(iommu, domain->id,
2713 start_pfn << VTD_PAGE_SHIFT,
2714 (last_pfn - start_pfn + 1));
f76aec76
KA		 2715
2716 /* free iova */
2717 __free_iova(&domain->iovad, iova);
ba395927
KA		 2718}
2719
ba395927 2720static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 2721 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 2722{
2723 int i;
c03ab37c 2724 struct scatterlist *sg;
ba395927 2725
c03ab37c 2726 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 2727 BUG_ON(!sg_page(sg));
4cf2e75d 2728 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 2729 sg->dma_length = sg->length;
ba395927
KA		 2730 }
2731 return nelems;
2732}
2733
d7ab5c46
FT		 2734static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2735 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 2736{
ba395927 2737 int i;
ba395927
KA		 2738 struct pci_dev *pdev = to_pci_dev(hwdev);
2739 struct dmar_domain *domain;
f76aec76
KA		 2740 size_t size = 0;
2741 int prot = 0;
b536d24d 2742 size_t offset_pfn = 0;
f76aec76
KA		 2743 struct iova *iova = NULL;
2744 int ret;
c03ab37c 2745 struct scatterlist *sg;
b536d24d 2746 unsigned long start_vpfn;
8c11e798 2747 struct intel_iommu *iommu;
ba395927
KA		 2748
2749 BUG_ON(dir == DMA_NONE);
2c2e2c38 2750 if (iommu_no_mapping(pdev))
c03ab37c 2751 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
ba395927 2752
f76aec76
KA		 2753 domain = get_valid_domain_for_dev(pdev);
2754 if (!domain)
2755 return 0;
2756
8c11e798
WH		 2757 iommu = domain_get_iommu(domain);
2758
b536d24d
DW		 2759 for_each_sg(sglist, sg, nelems, i)
2760 size += aligned_size(sg->offset, sg->length);
f76aec76 2761
bb9e6d65 2762 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
f76aec76 2763 if (!iova) {
c03ab37c 2764 sglist->dma_length = 0;
f76aec76
KA		 2765 return 0;
2766 }
2767
2768 /*
2769 * Check if DMAR supports zero-length reads on write only
2770 * mappings..
2771 */
2772 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2773 !cap_zlr(iommu->cap))
f76aec76
KA		 2774 prot |= DMA_PTE_READ;
2775 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2776 prot |= DMA_PTE_WRITE;
2777
b536d24d
DW		 2778 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
2779 offset_pfn = 0;
c03ab37c 2780 for_each_sg(sglist, sg, nelems, i) {
b536d24d
DW		 2781 int nr_pages = aligned_size(sg->offset, sg->length) >> VTD_PAGE_SHIFT;
2782 ret = domain_pfn_mapping(domain, start_vpfn + offset_pfn,
2783 page_to_dma_pfn(sg_page(sg)),
2784 nr_pages, prot);
f76aec76
KA		 2785 if (ret) {
2786 /* clear the page */
b536d24d
DW		 2787 dma_pte_clear_range(domain, start_vpfn,
2788 start_vpfn + offset_pfn);
f76aec76 2789 /* free page tables */
b536d24d
DW		 2790 dma_pte_free_pagetable(domain, start_vpfn,
2791 start_vpfn + offset_pfn);
f76aec76
KA		 2792 /* free iova */
2793 __free_iova(&domain->iovad, iova);
ba395927
KA		 2794 return 0;
2795 }
b536d24d
DW		 2796 sg->dma_address = ((dma_addr_t)(start_vpfn + offset_pfn)
2797 << VTD_PAGE_SHIFT) + sg->offset;
ba395927 2798 sg->dma_length = sg->length;
b536d24d 2799 offset_pfn += nr_pages;
ba395927
KA		 2800 }
2801
1f0ef2aa
DW		 2802 /* it's a non-present to present mapping. Only flush if caching mode */
2803 if (cap_caching_mode(iommu->cap))
b536d24d
DW		 2804 iommu_flush_iotlb_psi(iommu, 0, start_vpfn << VTD_PAGE_SHIFT,
2805 offset_pfn);
1f0ef2aa 2806 else
8c11e798 2807 iommu_flush_write_buffer(iommu);
1f0ef2aa 2808
ba395927
KA		 2809 return nelems;
2810}
2811
dfb805e8
FT		 2812static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2813{
2814 return !dma_addr;
2815}
2816
160c1d8e 2817struct dma_map_ops intel_dma_ops = {
ba395927
KA		 2818 .alloc_coherent = intel_alloc_coherent,
2819 .free_coherent = intel_free_coherent,
ba395927
KA		 2820 .map_sg = intel_map_sg,
2821 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 2822 .map_page = intel_map_page,
2823 .unmap_page = intel_unmap_page,
dfb805e8 2824 .mapping_error = intel_mapping_error,
ba395927
KA		 2825};
2826
2827static inline int iommu_domain_cache_init(void)
2828{
2829 int ret = 0;
2830
2831 iommu_domain_cache = kmem_cache_create("iommu_domain",
2832 sizeof(struct dmar_domain),
2833 0,
2834 SLAB_HWCACHE_ALIGN,
2835
2836 NULL);
2837 if (!iommu_domain_cache) {
2838 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2839 ret = -ENOMEM;
2840 }
2841
2842 return ret;
2843}
2844
2845static inline int iommu_devinfo_cache_init(void)
2846{
2847 int ret = 0;
2848
2849 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2850 sizeof(struct device_domain_info),
2851 0,
2852 SLAB_HWCACHE_ALIGN,
ba395927
KA		 2853 NULL);
2854 if (!iommu_devinfo_cache) {
2855 printk(KERN_ERR "Couldn't create devinfo cache\n");
2856 ret = -ENOMEM;
2857 }
2858
2859 return ret;
2860}
2861
2862static inline int iommu_iova_cache_init(void)
2863{
2864 int ret = 0;
2865
2866 iommu_iova_cache = kmem_cache_create("iommu_iova",
2867 sizeof(struct iova),
2868 0,
2869 SLAB_HWCACHE_ALIGN,
ba395927
KA		 2870 NULL);
2871 if (!iommu_iova_cache) {
2872 printk(KERN_ERR "Couldn't create iova cache\n");
2873 ret = -ENOMEM;
2874 }
2875
2876 return ret;
2877}
2878
2879static int __init iommu_init_mempool(void)
2880{
2881 int ret;
2882 ret = iommu_iova_cache_init();
2883 if (ret)
2884 return ret;
2885
2886 ret = iommu_domain_cache_init();
2887 if (ret)
2888 goto domain_error;
2889
2890 ret = iommu_devinfo_cache_init();
2891 if (!ret)
2892 return ret;
2893
2894 kmem_cache_destroy(iommu_domain_cache);
2895domain_error:
2896 kmem_cache_destroy(iommu_iova_cache);
2897
2898 return -ENOMEM;
2899}
2900
2901static void __init iommu_exit_mempool(void)
2902{
2903 kmem_cache_destroy(iommu_devinfo_cache);
2904 kmem_cache_destroy(iommu_domain_cache);
2905 kmem_cache_destroy(iommu_iova_cache);
2906
2907}
2908
ba395927
KA		 2909static void __init init_no_remapping_devices(void)
2910{
2911 struct dmar_drhd_unit *drhd;
2912
2913 for_each_drhd_unit(drhd) {
2914 if (!drhd->include_all) {
2915 int i;
2916 for (i = 0; i < drhd->devices_cnt; i++)
2917 if (drhd->devices[i] != NULL)
2918 break;
2919 /* ignore DMAR unit if no pci devices exist */
2920 if (i == drhd->devices_cnt)
2921 drhd->ignored = 1;
2922 }
2923 }
2924
2925 if (dmar_map_gfx)
2926 return;
2927
2928 for_each_drhd_unit(drhd) {
2929 int i;
2930 if (drhd->ignored || drhd->include_all)
2931 continue;
2932
2933 for (i = 0; i < drhd->devices_cnt; i++)
2934 if (drhd->devices[i] &&
2935 !IS_GFX_DEVICE(drhd->devices[i]))
2936 break;
2937
2938 if (i < drhd->devices_cnt)
2939 continue;
2940
2941 /* bypass IOMMU if it is just for gfx devices */
2942 drhd->ignored = 1;
2943 for (i = 0; i < drhd->devices_cnt; i++) {
2944 if (!drhd->devices[i])
2945 continue;
358dd8ac 2946 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 2947 }
2948 }
2949}
2950
f59c7b69
FY		 2951#ifdef CONFIG_SUSPEND
2952static int init_iommu_hw(void)
2953{
2954 struct dmar_drhd_unit *drhd;
2955 struct intel_iommu *iommu = NULL;
2956
2957 for_each_active_iommu(iommu, drhd)
2958 if (iommu->qi)
2959 dmar_reenable_qi(iommu);
2960
2961 for_each_active_iommu(iommu, drhd) {
2962 iommu_flush_write_buffer(iommu);
2963
2964 iommu_set_root_entry(iommu);
2965
2966 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 2967 DMA_CCMD_GLOBAL_INVL);
f59c7b69 2968 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 2969 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 2970 iommu_disable_protect_mem_regions(iommu);
2971 iommu_enable_translation(iommu);
2972 }
2973
2974 return 0;
2975}
2976
2977static void iommu_flush_all(void)
2978{
2979 struct dmar_drhd_unit *drhd;
2980 struct intel_iommu *iommu;
2981
2982 for_each_active_iommu(iommu, drhd) {
2983 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 2984 DMA_CCMD_GLOBAL_INVL);
f59c7b69 2985 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 2986 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 2987 }
2988}
2989
2990static int iommu_suspend(struct sys_device *dev, pm_message_t state)
2991{
2992 struct dmar_drhd_unit *drhd;
2993 struct intel_iommu *iommu = NULL;
2994 unsigned long flag;
2995
2996 for_each_active_iommu(iommu, drhd) {
2997 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
2998 GFP_ATOMIC);
2999 if (!iommu->iommu_state)
3000 goto nomem;
3001 }
3002
3003 iommu_flush_all();
3004
3005 for_each_active_iommu(iommu, drhd) {
3006 iommu_disable_translation(iommu);
3007
3008 spin_lock_irqsave(&iommu->register_lock, flag);
3009
3010 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3011 readl(iommu->reg + DMAR_FECTL_REG);
3012 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3013 readl(iommu->reg + DMAR_FEDATA_REG);
3014 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3015 readl(iommu->reg + DMAR_FEADDR_REG);
3016 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3017 readl(iommu->reg + DMAR_FEUADDR_REG);
3018
3019 spin_unlock_irqrestore(&iommu->register_lock, flag);
3020 }
3021 return 0;
3022
3023nomem:
3024 for_each_active_iommu(iommu, drhd)
3025 kfree(iommu->iommu_state);
3026
3027 return -ENOMEM;
3028}
3029
3030static int iommu_resume(struct sys_device *dev)
3031{
3032 struct dmar_drhd_unit *drhd;
3033 struct intel_iommu *iommu = NULL;
3034 unsigned long flag;
3035
3036 if (init_iommu_hw()) {
3037 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3038 return -EIO;
3039 }
3040
3041 for_each_active_iommu(iommu, drhd) {
3042
3043 spin_lock_irqsave(&iommu->register_lock, flag);
3044
3045 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3046 iommu->reg + DMAR_FECTL_REG);
3047 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3048 iommu->reg + DMAR_FEDATA_REG);
3049 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3050 iommu->reg + DMAR_FEADDR_REG);
3051 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3052 iommu->reg + DMAR_FEUADDR_REG);
3053
3054 spin_unlock_irqrestore(&iommu->register_lock, flag);
3055 }
3056
3057 for_each_active_iommu(iommu, drhd)
3058 kfree(iommu->iommu_state);
3059
3060 return 0;
3061}
3062
3063static struct sysdev_class iommu_sysclass = {
3064 .name = "iommu",
3065 .resume = iommu_resume,
3066 .suspend = iommu_suspend,
3067};
3068
3069static struct sys_device device_iommu = {
3070 .cls = &iommu_sysclass,
3071};
3072
3073static int __init init_iommu_sysfs(void)
3074{
3075 int error;
3076
3077 error = sysdev_class_register(&iommu_sysclass);
3078 if (error)
3079 return error;
3080
3081 error = sysdev_register(&device_iommu);
3082 if (error)
3083 sysdev_class_unregister(&iommu_sysclass);
3084
3085 return error;
3086}
3087
3088#else
3089static int __init init_iommu_sysfs(void)
3090{
3091 return 0;
3092}
3093#endif /* CONFIG_PM */
3094
ba395927
KA		 3095int __init intel_iommu_init(void)
3096{
3097 int ret = 0;
3098
ba395927
KA		 3099 if (dmar_table_init())
3100 return -ENODEV;
3101
1886e8a9
SS		 3102 if (dmar_dev_scope_init())
3103 return -ENODEV;
3104
2ae21010
SS		 3105 /*
3106 * Check the need for DMA-remapping initialization now.
3107 * Above initialization will also be used by Interrupt-remapping.
3108 */
4ed0d3e6 3109 if (no_iommu || (swiotlb && !iommu_pass_through) || dmar_disabled)
2ae21010
SS		 3110 return -ENODEV;
3111
ba395927
KA		 3112 iommu_init_mempool();
3113 dmar_init_reserved_ranges();
3114
3115 init_no_remapping_devices();
3116
3117 ret = init_dmars();
3118 if (ret) {
3119 printk(KERN_ERR "IOMMU: dmar init failed\n");
3120 put_iova_domain(&reserved_iova_list);
3121 iommu_exit_mempool();
3122 return ret;
3123 }
3124 printk(KERN_INFO
3125 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3126
5e0d2a6f 3127 init_timer(&unmap_timer);
ba395927 3128 force_iommu = 1;
4ed0d3e6
FY		 3129
3130 if (!iommu_pass_through) {
3131 printk(KERN_INFO
3132 "Multi-level page-table translation for DMAR.\n");
3133 dma_ops = &intel_dma_ops;
3134 } else
3135 printk(KERN_INFO
3136 "DMAR: Pass through translation for DMAR.\n");
3137
f59c7b69 3138 init_iommu_sysfs();
a8bcbb0d
JR		 3139
3140 register_iommu(&intel_iommu_ops);
3141
ba395927
KA		 3142 return 0;
3143}
e820482c 3144
3199aa6b
HW		 3145static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3146 struct pci_dev *pdev)
3147{
3148 struct pci_dev *tmp, *parent;
3149
3150 if (!iommu || !pdev)
3151 return;
3152
3153 /* dependent device detach */
3154 tmp = pci_find_upstream_pcie_bridge(pdev);
3155 /* Secondary interface's bus number and devfn 0 */
3156 if (tmp) {
3157 parent = pdev->bus->self;
3158 while (parent != tmp) {
3159 iommu_detach_dev(iommu, parent->bus->number,
276dbf99 3160 parent->devfn);
3199aa6b
HW		 3161 parent = parent->bus->self;
3162 }
3163 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
3164 iommu_detach_dev(iommu,
3165 tmp->subordinate->number, 0);
3166 else /* this is a legacy PCI bridge */
276dbf99
DW		 3167 iommu_detach_dev(iommu, tmp->bus->number,
3168 tmp->devfn);
3199aa6b
HW		 3169 }
3170}
3171
2c2e2c38 3172static void domain_remove_one_dev_info(struct dmar_domain *domain,
c7151a8d
WH		 3173 struct pci_dev *pdev)
3174{
3175 struct device_domain_info *info;
3176 struct intel_iommu *iommu;
3177 unsigned long flags;
3178 int found = 0;
3179 struct list_head *entry, *tmp;
3180
276dbf99
DW		 3181 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3182 pdev->devfn);
c7151a8d
WH		 3183 if (!iommu)
3184 return;
3185
3186 spin_lock_irqsave(&device_domain_lock, flags);
3187 list_for_each_safe(entry, tmp, &domain->devices) {
3188 info = list_entry(entry, struct device_domain_info, link);
276dbf99 3189 /* No need to compare PCI domain; it has to be the same */
c7151a8d
WH		 3190 if (info->bus == pdev->bus->number &&
3191 info->devfn == pdev->devfn) {
3192 list_del(&info->link);
3193 list_del(&info->global);
3194 if (info->dev)
3195 info->dev->dev.archdata.iommu = NULL;
3196 spin_unlock_irqrestore(&device_domain_lock, flags);
3197
93a23a72 3198 iommu_disable_dev_iotlb(info);
c7151a8d 3199 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3200 iommu_detach_dependent_devices(iommu, pdev);
c7151a8d
WH		 3201 free_devinfo_mem(info);
3202
3203 spin_lock_irqsave(&device_domain_lock, flags);
3204
3205 if (found)
3206 break;
3207 else
3208 continue;
3209 }
3210
3211 /* if there is no other devices under the same iommu
3212 * owned by this domain, clear this iommu in iommu_bmp
3213 * update iommu count and coherency
3214 */
276dbf99
DW		 3215 if (iommu == device_to_iommu(info->segment, info->bus,
3216 info->devfn))
c7151a8d
WH		 3217 found = 1;
3218 }
3219
3220 if (found == 0) {
3221 unsigned long tmp_flags;
3222 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3223 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3224 domain->iommu_count--;
58c610bd 3225 domain_update_iommu_cap(domain);
c7151a8d
WH		 3226 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3227 }
3228
3229 spin_unlock_irqrestore(&device_domain_lock, flags);
3230}
3231
3232static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3233{
3234 struct device_domain_info *info;
3235 struct intel_iommu *iommu;
3236 unsigned long flags1, flags2;
3237
3238 spin_lock_irqsave(&device_domain_lock, flags1);
3239 while (!list_empty(&domain->devices)) {
3240 info = list_entry(domain->devices.next,
3241 struct device_domain_info, link);
3242 list_del(&info->link);
3243 list_del(&info->global);
3244 if (info->dev)
3245 info->dev->dev.archdata.iommu = NULL;
3246
3247 spin_unlock_irqrestore(&device_domain_lock, flags1);
3248
93a23a72 3249 iommu_disable_dev_iotlb(info);
276dbf99 3250 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 3251 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3252 iommu_detach_dependent_devices(iommu, info->dev);
c7151a8d
WH		 3253
3254 /* clear this iommu in iommu_bmp, update iommu count
58c610bd 3255 * and capabilities
c7151a8d
WH		 3256 */
3257 spin_lock_irqsave(&domain->iommu_lock, flags2);
3258 if (test_and_clear_bit(iommu->seq_id,
3259 &domain->iommu_bmp)) {
3260 domain->iommu_count--;
58c610bd 3261 domain_update_iommu_cap(domain);
c7151a8d
WH		 3262 }
3263 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3264
3265 free_devinfo_mem(info);
3266 spin_lock_irqsave(&device_domain_lock, flags1);
3267 }
3268 spin_unlock_irqrestore(&device_domain_lock, flags1);
3269}
3270
5e98c4b1
WH		 3271/* domain id for virtual machine, it won't be set in context */
3272static unsigned long vm_domid;
3273
fe40f1e0
WH		 3274static int vm_domain_min_agaw(struct dmar_domain *domain)
3275{
3276 int i;
3277 int min_agaw = domain->agaw;
3278
3279 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
3280 for (; i < g_num_of_iommus; ) {
3281 if (min_agaw > g_iommus[i]->agaw)
3282 min_agaw = g_iommus[i]->agaw;
3283
3284 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
3285 }
3286
3287 return min_agaw;
3288}
3289
5e98c4b1
WH		 3290static struct dmar_domain *iommu_alloc_vm_domain(void)
3291{
3292 struct dmar_domain *domain;
3293
3294 domain = alloc_domain_mem();
3295 if (!domain)
3296 return NULL;
3297
3298 domain->id = vm_domid++;
3299 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3300 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3301
3302 return domain;
3303}
3304
2c2e2c38 3305static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 3306{
3307 int adjust_width;
3308
3309 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3310 spin_lock_init(&domain->mapping_lock);
3311 spin_lock_init(&domain->iommu_lock);
3312
3313 domain_reserve_special_ranges(domain);
3314
3315 /* calculate AGAW */
3316 domain->gaw = guest_width;
3317 adjust_width = guestwidth_to_adjustwidth(guest_width);
3318 domain->agaw = width_to_agaw(adjust_width);
3319
3320 INIT_LIST_HEAD(&domain->devices);
3321
3322 domain->iommu_count = 0;
3323 domain->iommu_coherency = 0;
fe40f1e0 3324 domain->max_addr = 0;
5e98c4b1
WH		 3325
3326 /* always allocate the top pgd */
3327 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
3328 if (!domain->pgd)
3329 return -ENOMEM;
3330 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3331 return 0;
3332}
3333
3334static void iommu_free_vm_domain(struct dmar_domain *domain)
3335{
3336 unsigned long flags;
3337 struct dmar_drhd_unit *drhd;
3338 struct intel_iommu *iommu;
3339 unsigned long i;
3340 unsigned long ndomains;
3341
3342 for_each_drhd_unit(drhd) {
3343 if (drhd->ignored)
3344 continue;
3345 iommu = drhd->iommu;
3346
3347 ndomains = cap_ndoms(iommu->cap);
3348 i = find_first_bit(iommu->domain_ids, ndomains);
3349 for (; i < ndomains; ) {
3350 if (iommu->domains[i] == domain) {
3351 spin_lock_irqsave(&iommu->lock, flags);
3352 clear_bit(i, iommu->domain_ids);
3353 iommu->domains[i] = NULL;
3354 spin_unlock_irqrestore(&iommu->lock, flags);
3355 break;
3356 }
3357 i = find_next_bit(iommu->domain_ids, ndomains, i+1);
3358 }
3359 }
3360}
3361
3362static void vm_domain_exit(struct dmar_domain *domain)
3363{
5e98c4b1
WH		 3364 /* Domain 0 is reserved, so dont process it */
3365 if (!domain)
3366 return;
3367
3368 vm_domain_remove_all_dev_info(domain);
3369 /* destroy iovas */
3370 put_iova_domain(&domain->iovad);
5e98c4b1
WH		 3371
3372 /* clear ptes */
595badf5 3373 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3374
3375 /* free page tables */
d794dc9b 3376 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3377
3378 iommu_free_vm_domain(domain);
3379 free_domain_mem(domain);
3380}
3381
5d450806 3382static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 3383{
5d450806 3384 struct dmar_domain *dmar_domain;
38717946 3385
5d450806
JR		 3386 dmar_domain = iommu_alloc_vm_domain();
3387 if (!dmar_domain) {
38717946 3388 printk(KERN_ERR
5d450806
JR		 3389 "intel_iommu_domain_init: dmar_domain == NULL\n");
3390 return -ENOMEM;
38717946 3391 }
2c2e2c38 3392 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 3393 printk(KERN_ERR
5d450806
JR		 3394 "intel_iommu_domain_init() failed\n");
3395 vm_domain_exit(dmar_domain);
3396 return -ENOMEM;
38717946 3397 }
5d450806 3398 domain->priv = dmar_domain;
faa3d6f5 3399
5d450806 3400 return 0;
38717946 3401}
38717946 3402
5d450806 3403static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 3404{
5d450806
JR		 3405 struct dmar_domain *dmar_domain = domain->priv;
3406
3407 domain->priv = NULL;
3408 vm_domain_exit(dmar_domain);
38717946 3409}
38717946 3410
4c5478c9
JR		 3411static int intel_iommu_attach_device(struct iommu_domain *domain,
3412 struct device *dev)
38717946 3413{
4c5478c9
JR		 3414 struct dmar_domain *dmar_domain = domain->priv;
3415 struct pci_dev *pdev = to_pci_dev(dev);
fe40f1e0
WH		 3416 struct intel_iommu *iommu;
3417 int addr_width;
3418 u64 end;
faa3d6f5
WH		 3419 int ret;
3420
3421 /* normally pdev is not mapped */
3422 if (unlikely(domain_context_mapped(pdev))) {
3423 struct dmar_domain *old_domain;
3424
3425 old_domain = find_domain(pdev);
3426 if (old_domain) {
2c2e2c38
FY		 3427 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3428 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3429 domain_remove_one_dev_info(old_domain, pdev);
faa3d6f5
WH		 3430 else
3431 domain_remove_dev_info(old_domain);
3432 }
3433 }
3434
276dbf99
DW		 3435 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3436 pdev->devfn);
fe40f1e0
WH		 3437 if (!iommu)
3438 return -ENODEV;
3439
3440 /* check if this iommu agaw is sufficient for max mapped address */
3441 addr_width = agaw_to_width(iommu->agaw);
3442 end = DOMAIN_MAX_ADDR(addr_width);
3443 end = end & VTD_PAGE_MASK;
4c5478c9 3444 if (end < dmar_domain->max_addr) {
fe40f1e0
WH		 3445 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3446 "sufficient for the mapped address (%llx)\n",
4c5478c9 3447 __func__, iommu->agaw, dmar_domain->max_addr);
fe40f1e0
WH		 3448 return -EFAULT;
3449 }
3450
2c2e2c38 3451 ret = domain_add_dev_info(dmar_domain, pdev);
faa3d6f5
WH		 3452 if (ret)
3453 return ret;
3454
93a23a72 3455 ret = domain_context_mapping(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
faa3d6f5 3456 return ret;
38717946 3457}
38717946 3458
4c5478c9
JR		 3459static void intel_iommu_detach_device(struct iommu_domain *domain,
3460 struct device *dev)
38717946 3461{
4c5478c9
JR		 3462 struct dmar_domain *dmar_domain = domain->priv;
3463 struct pci_dev *pdev = to_pci_dev(dev);
3464
2c2e2c38 3465 domain_remove_one_dev_info(dmar_domain, pdev);
faa3d6f5 3466}
c7151a8d 3467
dde57a21
JR		 3468static int intel_iommu_map_range(struct iommu_domain *domain,
3469 unsigned long iova, phys_addr_t hpa,
3470 size_t size, int iommu_prot)
faa3d6f5 3471{
dde57a21 3472 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0
WH		 3473 u64 max_addr;
3474 int addr_width;
dde57a21 3475 int prot = 0;
faa3d6f5 3476 int ret;
fe40f1e0 3477
dde57a21
JR		 3478 if (iommu_prot & IOMMU_READ)
3479 prot |= DMA_PTE_READ;
3480 if (iommu_prot & IOMMU_WRITE)
3481 prot |= DMA_PTE_WRITE;
9cf06697
SY		 3482 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3483 prot |= DMA_PTE_SNP;
dde57a21 3484
163cc52c 3485 max_addr = iova + size;
dde57a21 3486 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 3487 int min_agaw;
3488 u64 end;
3489
3490 /* check if minimum agaw is sufficient for mapped address */
dde57a21 3491 min_agaw = vm_domain_min_agaw(dmar_domain);
fe40f1e0
WH		 3492 addr_width = agaw_to_width(min_agaw);
3493 end = DOMAIN_MAX_ADDR(addr_width);
3494 end = end & VTD_PAGE_MASK;
3495 if (end < max_addr) {
3496 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3497 "sufficient for the mapped address (%llx)\n",
3498 __func__, min_agaw, max_addr);
3499 return -EFAULT;
3500 }
dde57a21 3501 dmar_domain->max_addr = max_addr;
fe40f1e0 3502 }
ad051221
DW		 3503 /* Round up size to next multiple of PAGE_SIZE, if it and
3504 the low bits of hpa would take us onto the next page */
3505 size = aligned_size(hpa, size) >> VTD_PAGE_SHIFT;
3506 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3507 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 3508 return ret;
38717946 3509}
38717946 3510
dde57a21
JR		 3511static void intel_iommu_unmap_range(struct iommu_domain *domain,
3512 unsigned long iova, size_t size)
38717946 3513{
dde57a21 3514 struct dmar_domain *dmar_domain = domain->priv;
faa3d6f5 3515
163cc52c
DW		 3516 dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3517 (iova + size - 1) >> VTD_PAGE_SHIFT);
fe40f1e0 3518
163cc52c
DW		 3519 if (dmar_domain->max_addr == iova + size)
3520 dmar_domain->max_addr = iova;
38717946 3521}
38717946 3522
d14d6577
JR		 3523static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3524 unsigned long iova)
38717946 3525{
d14d6577 3526 struct dmar_domain *dmar_domain = domain->priv;
38717946 3527 struct dma_pte *pte;
faa3d6f5 3528 u64 phys = 0;
38717946 3529
b026fd28 3530 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT);
38717946 3531 if (pte)
faa3d6f5 3532 phys = dma_pte_addr(pte);
38717946 3533
faa3d6f5 3534 return phys;
38717946 3535}
a8bcbb0d 3536
dbb9fd86
SY		 3537static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3538 unsigned long cap)
3539{
3540 struct dmar_domain *dmar_domain = domain->priv;
3541
3542 if (cap == IOMMU_CAP_CACHE_COHERENCY)
3543 return dmar_domain->iommu_snooping;
3544
3545 return 0;
3546}
3547
a8bcbb0d
JR		 3548static struct iommu_ops intel_iommu_ops = {
3549 .domain_init = intel_iommu_domain_init,
3550 .domain_destroy = intel_iommu_domain_destroy,
3551 .attach_dev = intel_iommu_attach_device,
3552 .detach_dev = intel_iommu_detach_device,
3553 .map = intel_iommu_map_range,
3554 .unmap = intel_iommu_unmap_range,
3555 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 3556 .domain_has_cap = intel_iommu_domain_has_cap,
a8bcbb0d 3557};
9af88143
DW		 3558
3559static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3560{
3561 /*
3562 * Mobile 4 Series Chipset neglects to set RWBF capability,
3563 * but needs it:
3564 */
3565 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3566 rwbf_quirk = 1;
3567}
3568
3569DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
Ubuntu Jammy Linux kernel mirror