]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - drivers/iommu/amd_iommu.c
projects / mirror_ubuntu-hirsute-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
iommu/amd: Add support for IOMMUv2 domain mode
[mirror_ubuntu-hirsute-kernel.git] / drivers / iommu / amd_iommu.c
1 /*
2 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 * Leo Duran <leo.duran@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20 #include <linux/ratelimit.h>
21 #include <linux/pci.h>
22 #include <linux/pci-ats.h>
23 #include <linux/bitmap.h>
24 #include <linux/slab.h>
25 #include <linux/debugfs.h>
26 #include <linux/scatterlist.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/iommu-helper.h>
29 #include <linux/iommu.h>
30 #include <linux/delay.h>
31 #include <linux/amd-iommu.h>
32 #include <linux/notifier.h>
33 #include <linux/export.h>
34 #include <asm/msidef.h>
35 #include <asm/proto.h>
36 #include <asm/iommu.h>
37 #include <asm/gart.h>
38 #include <asm/dma.h>
39
40 #include "amd_iommu_proto.h"
41 #include "amd_iommu_types.h"
42
43 #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28))
44
45 #define LOOP_TIMEOUT 100000
46
47 static DEFINE_RWLOCK(amd_iommu_devtable_lock);
48
49 /* A list of preallocated protection domains */
50 static LIST_HEAD(iommu_pd_list);
51 static DEFINE_SPINLOCK(iommu_pd_list_lock);
52
53 /* List of all available dev_data structures */
54 static LIST_HEAD(dev_data_list);
55 static DEFINE_SPINLOCK(dev_data_list_lock);
56
57 /*
58 * Domain for untranslated devices - only allocated
59 * if iommu=pt passed on kernel cmd line.
60 */
61 static struct protection_domain *pt_domain;
62
63 static struct iommu_ops amd_iommu_ops;
64
65 static ATOMIC_NOTIFIER_HEAD(ppr_notifier);
66 int amd_iommu_max_glx_val = -1;
67
68 /*
69 * general struct to manage commands send to an IOMMU
70 */
71 struct iommu_cmd {
72 u32 data[4];
73 };
74
75 static void update_domain(struct protection_domain *domain);
76 static int __init alloc_passthrough_domain(void);
77
78 /****************************************************************************
79 *
80 * Helper functions
81 *
82 ****************************************************************************/
83
84 static struct iommu_dev_data *alloc_dev_data(u16 devid)
85 {
86 struct iommu_dev_data *dev_data;
87 unsigned long flags;
88
89 dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);
90 if (!dev_data)
91 return NULL;
92
93 dev_data->devid = devid;
94 atomic_set(&dev_data->bind, 0);
95
96 spin_lock_irqsave(&dev_data_list_lock, flags);
97 list_add_tail(&dev_data->dev_data_list, &dev_data_list);
98 spin_unlock_irqrestore(&dev_data_list_lock, flags);
99
100 return dev_data;
101 }
102
103 static void free_dev_data(struct iommu_dev_data *dev_data)
104 {
105 unsigned long flags;
106
107 spin_lock_irqsave(&dev_data_list_lock, flags);
108 list_del(&dev_data->dev_data_list);
109 spin_unlock_irqrestore(&dev_data_list_lock, flags);
110
111 kfree(dev_data);
112 }
113
114 static struct iommu_dev_data *search_dev_data(u16 devid)
115 {
116 struct iommu_dev_data *dev_data;
117 unsigned long flags;
118
119 spin_lock_irqsave(&dev_data_list_lock, flags);
120 list_for_each_entry(dev_data, &dev_data_list, dev_data_list) {
121 if (dev_data->devid == devid)
122 goto out_unlock;
123 }
124
125 dev_data = NULL;
126
127 out_unlock:
128 spin_unlock_irqrestore(&dev_data_list_lock, flags);
129
130 return dev_data;
131 }
132
133 static struct iommu_dev_data *find_dev_data(u16 devid)
134 {
135 struct iommu_dev_data *dev_data;
136
137 dev_data = search_dev_data(devid);
138
139 if (dev_data == NULL)
140 dev_data = alloc_dev_data(devid);
141
142 return dev_data;
143 }
144
145 static inline u16 get_device_id(struct device *dev)
146 {
147 struct pci_dev *pdev = to_pci_dev(dev);
148
149 return calc_devid(pdev->bus->number, pdev->devfn);
150 }
151
152 static struct iommu_dev_data *get_dev_data(struct device *dev)
153 {
154 return dev->archdata.iommu;
155 }
156
157 static bool pci_iommuv2_capable(struct pci_dev *pdev)
158 {
159 static const int caps[] = {
160 PCI_EXT_CAP_ID_ATS,
161 PCI_PRI_CAP,
162 PCI_PASID_CAP,
163 };
164 int i, pos;
165
166 for (i = 0; i < 3; ++i) {
167 pos = pci_find_ext_capability(pdev, caps[i]);
168 if (pos == 0)
169 return false;
170 }
171
172 return true;
173 }
174
175 /*
176 * In this function the list of preallocated protection domains is traversed to
177 * find the domain for a specific device
178 */
179 static struct dma_ops_domain *find_protection_domain(u16 devid)
180 {
181 struct dma_ops_domain *entry, *ret = NULL;
182 unsigned long flags;
183 u16 alias = amd_iommu_alias_table[devid];
184
185 if (list_empty(&iommu_pd_list))
186 return NULL;
187
188 spin_lock_irqsave(&iommu_pd_list_lock, flags);
189
190 list_for_each_entry(entry, &iommu_pd_list, list) {
191 if (entry->target_dev == devid ||
192 entry->target_dev == alias) {
193 ret = entry;
194 break;
195 }
196 }
197
198 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
199
200 return ret;
201 }
202
203 /*
204 * This function checks if the driver got a valid device from the caller to
205 * avoid dereferencing invalid pointers.
206 */
207 static bool check_device(struct device *dev)
208 {
209 u16 devid;
210
211 if (!dev || !dev->dma_mask)
212 return false;
213
214 /* No device or no PCI device */
215 if (dev->bus != &pci_bus_type)
216 return false;
217
218 devid = get_device_id(dev);
219
220 /* Out of our scope? */
221 if (devid > amd_iommu_last_bdf)
222 return false;
223
224 if (amd_iommu_rlookup_table[devid] == NULL)
225 return false;
226
227 return true;
228 }
229
230 static int iommu_init_device(struct device *dev)
231 {
232 struct pci_dev *pdev = to_pci_dev(dev);
233 struct iommu_dev_data *dev_data;
234 u16 alias;
235
236 if (dev->archdata.iommu)
237 return 0;
238
239 dev_data = find_dev_data(get_device_id(dev));
240 if (!dev_data)
241 return -ENOMEM;
242
243 alias = amd_iommu_alias_table[dev_data->devid];
244 if (alias != dev_data->devid) {
245 struct iommu_dev_data *alias_data;
246
247 alias_data = find_dev_data(alias);
248 if (alias_data == NULL) {
249 pr_err("AMD-Vi: Warning: Unhandled device %s\n",
250 dev_name(dev));
251 free_dev_data(dev_data);
252 return -ENOTSUPP;
253 }
254 dev_data->alias_data = alias_data;
255 }
256
257 if (pci_iommuv2_capable(pdev)) {
258 struct amd_iommu *iommu;
259
260 iommu = amd_iommu_rlookup_table[dev_data->devid];
261 dev_data->iommu_v2 = iommu->is_iommu_v2;
262 }
263
264 dev->archdata.iommu = dev_data;
265
266 return 0;
267 }
268
269 static void iommu_ignore_device(struct device *dev)
270 {
271 u16 devid, alias;
272
273 devid = get_device_id(dev);
274 alias = amd_iommu_alias_table[devid];
275
276 memset(&amd_iommu_dev_table[devid], 0, sizeof(struct dev_table_entry));
277 memset(&amd_iommu_dev_table[alias], 0, sizeof(struct dev_table_entry));
278
279 amd_iommu_rlookup_table[devid] = NULL;
280 amd_iommu_rlookup_table[alias] = NULL;
281 }
282
283 static void iommu_uninit_device(struct device *dev)
284 {
285 /*
286 * Nothing to do here - we keep dev_data around for unplugged devices
287 * and reuse it when the device is re-plugged - not doing so would
288 * introduce a ton of races.
289 */
290 }
291
292 void __init amd_iommu_uninit_devices(void)
293 {
294 struct iommu_dev_data *dev_data, *n;
295 struct pci_dev *pdev = NULL;
296
297 for_each_pci_dev(pdev) {
298
299 if (!check_device(&pdev->dev))
300 continue;
301
302 iommu_uninit_device(&pdev->dev);
303 }
304
305 /* Free all of our dev_data structures */
306 list_for_each_entry_safe(dev_data, n, &dev_data_list, dev_data_list)
307 free_dev_data(dev_data);
308 }
309
310 int __init amd_iommu_init_devices(void)
311 {
312 struct pci_dev *pdev = NULL;
313 int ret = 0;
314
315 for_each_pci_dev(pdev) {
316
317 if (!check_device(&pdev->dev))
318 continue;
319
320 ret = iommu_init_device(&pdev->dev);
321 if (ret == -ENOTSUPP)
322 iommu_ignore_device(&pdev->dev);
323 else if (ret)
324 goto out_free;
325 }
326
327 return 0;
328
329 out_free:
330
331 amd_iommu_uninit_devices();
332
333 return ret;
334 }
335 #ifdef CONFIG_AMD_IOMMU_STATS
336
337 /*
338 * Initialization code for statistics collection
339 */
340
341 DECLARE_STATS_COUNTER(compl_wait);
342 DECLARE_STATS_COUNTER(cnt_map_single);
343 DECLARE_STATS_COUNTER(cnt_unmap_single);
344 DECLARE_STATS_COUNTER(cnt_map_sg);
345 DECLARE_STATS_COUNTER(cnt_unmap_sg);
346 DECLARE_STATS_COUNTER(cnt_alloc_coherent);
347 DECLARE_STATS_COUNTER(cnt_free_coherent);
348 DECLARE_STATS_COUNTER(cross_page);
349 DECLARE_STATS_COUNTER(domain_flush_single);
350 DECLARE_STATS_COUNTER(domain_flush_all);
351 DECLARE_STATS_COUNTER(alloced_io_mem);
352 DECLARE_STATS_COUNTER(total_map_requests);
353
354 static struct dentry *stats_dir;
355 static struct dentry *de_fflush;
356
357 static void amd_iommu_stats_add(struct __iommu_counter *cnt)
358 {
359 if (stats_dir == NULL)
360 return;
361
362 cnt->dent = debugfs_create_u64(cnt->name, 0444, stats_dir,
363 &cnt->value);
364 }
365
366 static void amd_iommu_stats_init(void)
367 {
368 stats_dir = debugfs_create_dir("amd-iommu", NULL);
369 if (stats_dir == NULL)
370 return;
371
372 de_fflush = debugfs_create_bool("fullflush", 0444, stats_dir,
373 (u32 *)&amd_iommu_unmap_flush);
374
375 amd_iommu_stats_add(&compl_wait);
376 amd_iommu_stats_add(&cnt_map_single);
377 amd_iommu_stats_add(&cnt_unmap_single);
378 amd_iommu_stats_add(&cnt_map_sg);
379 amd_iommu_stats_add(&cnt_unmap_sg);
380 amd_iommu_stats_add(&cnt_alloc_coherent);
381 amd_iommu_stats_add(&cnt_free_coherent);
382 amd_iommu_stats_add(&cross_page);
383 amd_iommu_stats_add(&domain_flush_single);
384 amd_iommu_stats_add(&domain_flush_all);
385 amd_iommu_stats_add(&alloced_io_mem);
386 amd_iommu_stats_add(&total_map_requests);
387 }
388
389 #endif
390
391 /****************************************************************************
392 *
393 * Interrupt handling functions
394 *
395 ****************************************************************************/
396
397 static void dump_dte_entry(u16 devid)
398 {
399 int i;
400
401 for (i = 0; i < 4; ++i)
402 pr_err("AMD-Vi: DTE[%d]: %016llx\n", i,
403 amd_iommu_dev_table[devid].data[i]);
404 }
405
406 static void dump_command(unsigned long phys_addr)
407 {
408 struct iommu_cmd *cmd = phys_to_virt(phys_addr);
409 int i;
410
411 for (i = 0; i < 4; ++i)
412 pr_err("AMD-Vi: CMD[%d]: %08x\n", i, cmd->data[i]);
413 }
414
415 static void iommu_print_event(struct amd_iommu *iommu, void *__evt)
416 {
417 u32 *event = __evt;
418 int type = (event[1] >> EVENT_TYPE_SHIFT) & EVENT_TYPE_MASK;
419 int devid = (event[0] >> EVENT_DEVID_SHIFT) & EVENT_DEVID_MASK;
420 int domid = (event[1] >> EVENT_DOMID_SHIFT) & EVENT_DOMID_MASK;
421 int flags = (event[1] >> EVENT_FLAGS_SHIFT) & EVENT_FLAGS_MASK;
422 u64 address = (u64)(((u64)event[3]) << 32) | event[2];
423
424 printk(KERN_ERR "AMD-Vi: Event logged [");
425
426 switch (type) {
427 case EVENT_TYPE_ILL_DEV:
428 printk("ILLEGAL_DEV_TABLE_ENTRY device=%02x:%02x.%x "
429 "address=0x%016llx flags=0x%04x]\n",
430 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
431 address, flags);
432 dump_dte_entry(devid);
433 break;
434 case EVENT_TYPE_IO_FAULT:
435 printk("IO_PAGE_FAULT device=%02x:%02x.%x "
436 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
437 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
438 domid, address, flags);
439 break;
440 case EVENT_TYPE_DEV_TAB_ERR:
441 printk("DEV_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
442 "address=0x%016llx flags=0x%04x]\n",
443 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
444 address, flags);
445 break;
446 case EVENT_TYPE_PAGE_TAB_ERR:
447 printk("PAGE_TAB_HARDWARE_ERROR device=%02x:%02x.%x "
448 "domain=0x%04x address=0x%016llx flags=0x%04x]\n",
449 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
450 domid, address, flags);
451 break;
452 case EVENT_TYPE_ILL_CMD:
453 printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address);
454 dump_command(address);
455 break;
456 case EVENT_TYPE_CMD_HARD_ERR:
457 printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
458 "flags=0x%04x]\n", address, flags);
459 break;
460 case EVENT_TYPE_IOTLB_INV_TO:
461 printk("IOTLB_INV_TIMEOUT device=%02x:%02x.%x "
462 "address=0x%016llx]\n",
463 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
464 address);
465 break;
466 case EVENT_TYPE_INV_DEV_REQ:
467 printk("INVALID_DEVICE_REQUEST device=%02x:%02x.%x "
468 "address=0x%016llx flags=0x%04x]\n",
469 PCI_BUS(devid), PCI_SLOT(devid), PCI_FUNC(devid),
470 address, flags);
471 break;
472 default:
473 printk(KERN_ERR "UNKNOWN type=0x%02x]\n", type);
474 }
475 }
476
477 static void iommu_poll_events(struct amd_iommu *iommu)
478 {
479 u32 head, tail;
480 unsigned long flags;
481
482 spin_lock_irqsave(&iommu->lock, flags);
483
484 head = readl(iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
485 tail = readl(iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
486
487 while (head != tail) {
488 iommu_print_event(iommu, iommu->evt_buf + head);
489 head = (head + EVENT_ENTRY_SIZE) % iommu->evt_buf_size;
490 }
491
492 writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
493
494 spin_unlock_irqrestore(&iommu->lock, flags);
495 }
496
497 static void iommu_handle_ppr_entry(struct amd_iommu *iommu, u32 head)
498 {
499 struct amd_iommu_fault fault;
500 volatile u64 *raw;
501 int i;
502
503 raw = (u64 *)(iommu->ppr_log + head);
504
505 /*
506 * Hardware bug: Interrupt may arrive before the entry is written to
507 * memory. If this happens we need to wait for the entry to arrive.
508 */
509 for (i = 0; i < LOOP_TIMEOUT; ++i) {
510 if (PPR_REQ_TYPE(raw[0]) != 0)
511 break;
512 udelay(1);
513 }
514
515 if (PPR_REQ_TYPE(raw[0]) != PPR_REQ_FAULT) {
516 pr_err_ratelimited("AMD-Vi: Unknown PPR request received\n");
517 return;
518 }
519
520 fault.address = raw[1];
521 fault.pasid = PPR_PASID(raw[0]);
522 fault.device_id = PPR_DEVID(raw[0]);
523 fault.tag = PPR_TAG(raw[0]);
524 fault.flags = PPR_FLAGS(raw[0]);
525
526 /*
527 * To detect the hardware bug we need to clear the entry
528 * to back to zero.
529 */
530 raw[0] = raw[1] = 0;
531
532 atomic_notifier_call_chain(&ppr_notifier, 0, &fault);
533 }
534
535 static void iommu_poll_ppr_log(struct amd_iommu *iommu)
536 {
537 unsigned long flags;
538 u32 head, tail;
539
540 if (iommu->ppr_log == NULL)
541 return;
542
543 spin_lock_irqsave(&iommu->lock, flags);
544
545 head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
546 tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
547
548 while (head != tail) {
549
550 /* Handle PPR entry */
551 iommu_handle_ppr_entry(iommu, head);
552
553 /* Update and refresh ring-buffer state*/
554 head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
555 writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
556 tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
557 }
558
559 /* enable ppr interrupts again */
560 writel(MMIO_STATUS_PPR_INT_MASK, iommu->mmio_base + MMIO_STATUS_OFFSET);
561
562 spin_unlock_irqrestore(&iommu->lock, flags);
563 }
564
565 irqreturn_t amd_iommu_int_thread(int irq, void *data)
566 {
567 struct amd_iommu *iommu;
568
569 for_each_iommu(iommu) {
570 iommu_poll_events(iommu);
571 iommu_poll_ppr_log(iommu);
572 }
573
574 return IRQ_HANDLED;
575 }
576
577 irqreturn_t amd_iommu_int_handler(int irq, void *data)
578 {
579 return IRQ_WAKE_THREAD;
580 }
581
582 /****************************************************************************
583 *
584 * IOMMU command queuing functions
585 *
586 ****************************************************************************/
587
588 static int wait_on_sem(volatile u64 *sem)
589 {
590 int i = 0;
591
592 while (*sem == 0 && i < LOOP_TIMEOUT) {
593 udelay(1);
594 i += 1;
595 }
596
597 if (i == LOOP_TIMEOUT) {
598 pr_alert("AMD-Vi: Completion-Wait loop timed out\n");
599 return -EIO;
600 }
601
602 return 0;
603 }
604
605 static void copy_cmd_to_buffer(struct amd_iommu *iommu,
606 struct iommu_cmd *cmd,
607 u32 tail)
608 {
609 u8 *target;
610
611 target = iommu->cmd_buf + tail;
612 tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
613
614 /* Copy command to buffer */
615 memcpy(target, cmd, sizeof(*cmd));
616
617 /* Tell the IOMMU about it */
618 writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
619 }
620
621 static void build_completion_wait(struct iommu_cmd *cmd, u64 address)
622 {
623 WARN_ON(address & 0x7ULL);
624
625 memset(cmd, 0, sizeof(*cmd));
626 cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK;
627 cmd->data[1] = upper_32_bits(__pa(address));
628 cmd->data[2] = 1;
629 CMD_SET_TYPE(cmd, CMD_COMPL_WAIT);
630 }
631
632 static void build_inv_dte(struct iommu_cmd *cmd, u16 devid)
633 {
634 memset(cmd, 0, sizeof(*cmd));
635 cmd->data[0] = devid;
636 CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY);
637 }
638
639 static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address,
640 size_t size, u16 domid, int pde)
641 {
642 u64 pages;
643 int s;
644
645 pages = iommu_num_pages(address, size, PAGE_SIZE);
646 s = 0;
647
648 if (pages > 1) {
649 /*
650 * If we have to flush more than one page, flush all
651 * TLB entries for this domain
652 */
653 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
654 s = 1;
655 }
656
657 address &= PAGE_MASK;
658
659 memset(cmd, 0, sizeof(*cmd));
660 cmd->data[1] |= domid;
661 cmd->data[2] = lower_32_bits(address);
662 cmd->data[3] = upper_32_bits(address);
663 CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES);
664 if (s) /* size bit - we flush more than one 4kb page */
665 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
666 if (pde) /* PDE bit - we wan't flush everything not only the PTEs */
667 cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK;
668 }
669
670 static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep,
671 u64 address, size_t size)
672 {
673 u64 pages;
674 int s;
675
676 pages = iommu_num_pages(address, size, PAGE_SIZE);
677 s = 0;
678
679 if (pages > 1) {
680 /*
681 * If we have to flush more than one page, flush all
682 * TLB entries for this domain
683 */
684 address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS;
685 s = 1;
686 }
687
688 address &= PAGE_MASK;
689
690 memset(cmd, 0, sizeof(*cmd));
691 cmd->data[0] = devid;
692 cmd->data[0] |= (qdep & 0xff) << 24;
693 cmd->data[1] = devid;
694 cmd->data[2] = lower_32_bits(address);
695 cmd->data[3] = upper_32_bits(address);
696 CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES);
697 if (s)
698 cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK;
699 }
700
701 static void build_inv_all(struct iommu_cmd *cmd)
702 {
703 memset(cmd, 0, sizeof(*cmd));
704 CMD_SET_TYPE(cmd, CMD_INV_ALL);
705 }
706
707 /*
708 * Writes the command to the IOMMUs command buffer and informs the
709 * hardware about the new command.
710 */
711 static int iommu_queue_command_sync(struct amd_iommu *iommu,
712 struct iommu_cmd *cmd,
713 bool sync)
714 {
715 u32 left, tail, head, next_tail;
716 unsigned long flags;
717
718 WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED);
719
720 again:
721 spin_lock_irqsave(&iommu->lock, flags);
722
723 head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
724 tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
725 next_tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size;
726 left = (head - next_tail) % iommu->cmd_buf_size;
727
728 if (left <= 2) {
729 struct iommu_cmd sync_cmd;
730 volatile u64 sem = 0;
731 int ret;
732
733 build_completion_wait(&sync_cmd, (u64)&sem);
734 copy_cmd_to_buffer(iommu, &sync_cmd, tail);
735
736 spin_unlock_irqrestore(&iommu->lock, flags);
737
738 if ((ret = wait_on_sem(&sem)) != 0)
739 return ret;
740
741 goto again;
742 }
743
744 copy_cmd_to_buffer(iommu, cmd, tail);
745
746 /* We need to sync now to make sure all commands are processed */
747 iommu->need_sync = sync;
748
749 spin_unlock_irqrestore(&iommu->lock, flags);
750
751 return 0;
752 }
753
754 static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd)
755 {
756 return iommu_queue_command_sync(iommu, cmd, true);
757 }
758
759 /*
760 * This function queues a completion wait command into the command
761 * buffer of an IOMMU
762 */
763 static int iommu_completion_wait(struct amd_iommu *iommu)
764 {
765 struct iommu_cmd cmd;
766 volatile u64 sem = 0;
767 int ret;
768
769 if (!iommu->need_sync)
770 return 0;
771
772 build_completion_wait(&cmd, (u64)&sem);
773
774 ret = iommu_queue_command_sync(iommu, &cmd, false);
775 if (ret)
776 return ret;
777
778 return wait_on_sem(&sem);
779 }
780
781 static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid)
782 {
783 struct iommu_cmd cmd;
784
785 build_inv_dte(&cmd, devid);
786
787 return iommu_queue_command(iommu, &cmd);
788 }
789
790 static void iommu_flush_dte_all(struct amd_iommu *iommu)
791 {
792 u32 devid;
793
794 for (devid = 0; devid <= 0xffff; ++devid)
795 iommu_flush_dte(iommu, devid);
796
797 iommu_completion_wait(iommu);
798 }
799
800 /*
801 * This function uses heavy locking and may disable irqs for some time. But
802 * this is no issue because it is only called during resume.
803 */
804 static void iommu_flush_tlb_all(struct amd_iommu *iommu)
805 {
806 u32 dom_id;
807
808 for (dom_id = 0; dom_id <= 0xffff; ++dom_id) {
809 struct iommu_cmd cmd;
810 build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS,
811 dom_id, 1);
812 iommu_queue_command(iommu, &cmd);
813 }
814
815 iommu_completion_wait(iommu);
816 }
817
818 static void iommu_flush_all(struct amd_iommu *iommu)
819 {
820 struct iommu_cmd cmd;
821
822 build_inv_all(&cmd);
823
824 iommu_queue_command(iommu, &cmd);
825 iommu_completion_wait(iommu);
826 }
827
828 void iommu_flush_all_caches(struct amd_iommu *iommu)
829 {
830 if (iommu_feature(iommu, FEATURE_IA)) {
831 iommu_flush_all(iommu);
832 } else {
833 iommu_flush_dte_all(iommu);
834 iommu_flush_tlb_all(iommu);
835 }
836 }
837
838 /*
839 * Command send function for flushing on-device TLB
840 */
841 static int device_flush_iotlb(struct iommu_dev_data *dev_data,
842 u64 address, size_t size)
843 {
844 struct amd_iommu *iommu;
845 struct iommu_cmd cmd;
846 int qdep;
847
848 qdep = dev_data->ats.qdep;
849 iommu = amd_iommu_rlookup_table[dev_data->devid];
850
851 build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);
852
853 return iommu_queue_command(iommu, &cmd);
854 }
855
856 /*
857 * Command send function for invalidating a device table entry
858 */
859 static int device_flush_dte(struct iommu_dev_data *dev_data)
860 {
861 struct amd_iommu *iommu;
862 int ret;
863
864 iommu = amd_iommu_rlookup_table[dev_data->devid];
865
866 ret = iommu_flush_dte(iommu, dev_data->devid);
867 if (ret)
868 return ret;
869
870 if (dev_data->ats.enabled)
871 ret = device_flush_iotlb(dev_data, 0, ~0UL);
872
873 return ret;
874 }
875
876 /*
877 * TLB invalidation function which is called from the mapping functions.
878 * It invalidates a single PTE if the range to flush is within a single
879 * page. Otherwise it flushes the whole TLB of the IOMMU.
880 */
881 static void __domain_flush_pages(struct protection_domain *domain,
882 u64 address, size_t size, int pde)
883 {
884 struct iommu_dev_data *dev_data;
885 struct iommu_cmd cmd;
886 int ret = 0, i;
887
888 build_inv_iommu_pages(&cmd, address, size, domain->id, pde);
889
890 for (i = 0; i < amd_iommus_present; ++i) {
891 if (!domain->dev_iommu[i])
892 continue;
893
894 /*
895 * Devices of this domain are behind this IOMMU
896 * We need a TLB flush
897 */
898 ret |= iommu_queue_command(amd_iommus[i], &cmd);
899 }
900
901 list_for_each_entry(dev_data, &domain->dev_list, list) {
902
903 if (!dev_data->ats.enabled)
904 continue;
905
906 ret |= device_flush_iotlb(dev_data, address, size);
907 }
908
909 WARN_ON(ret);
910 }
911
912 static void domain_flush_pages(struct protection_domain *domain,
913 u64 address, size_t size)
914 {
915 __domain_flush_pages(domain, address, size, 0);
916 }
917
918 /* Flush the whole IO/TLB for a given protection domain */
919 static void domain_flush_tlb(struct protection_domain *domain)
920 {
921 __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0);
922 }
923
924 /* Flush the whole IO/TLB for a given protection domain - including PDE */
925 static void domain_flush_tlb_pde(struct protection_domain *domain)
926 {
927 __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1);
928 }
929
930 static void domain_flush_complete(struct protection_domain *domain)
931 {
932 int i;
933
934 for (i = 0; i < amd_iommus_present; ++i) {
935 if (!domain->dev_iommu[i])
936 continue;
937
938 /*
939 * Devices of this domain are behind this IOMMU
940 * We need to wait for completion of all commands.
941 */
942 iommu_completion_wait(amd_iommus[i]);
943 }
944 }
945
946
947 /*
948 * This function flushes the DTEs for all devices in domain
949 */
950 static void domain_flush_devices(struct protection_domain *domain)
951 {
952 struct iommu_dev_data *dev_data;
953
954 list_for_each_entry(dev_data, &domain->dev_list, list)
955 device_flush_dte(dev_data);
956 }
957
958 /****************************************************************************
959 *
960 * The functions below are used the create the page table mappings for
961 * unity mapped regions.
962 *
963 ****************************************************************************/
964
965 /*
966 * This function is used to add another level to an IO page table. Adding
967 * another level increases the size of the address space by 9 bits to a size up
968 * to 64 bits.
969 */
970 static bool increase_address_space(struct protection_domain *domain,
971 gfp_t gfp)
972 {
973 u64 *pte;
974
975 if (domain->mode == PAGE_MODE_6_LEVEL)
976 /* address space already 64 bit large */
977 return false;
978
979 pte = (void *)get_zeroed_page(gfp);
980 if (!pte)
981 return false;
982
983 *pte = PM_LEVEL_PDE(domain->mode,
984 virt_to_phys(domain->pt_root));
985 domain->pt_root = pte;
986 domain->mode += 1;
987 domain->updated = true;
988
989 return true;
990 }
991
992 static u64 *alloc_pte(struct protection_domain *domain,
993 unsigned long address,
994 unsigned long page_size,
995 u64 **pte_page,
996 gfp_t gfp)
997 {
998 int level, end_lvl;
999 u64 *pte, *page;
1000
1001 BUG_ON(!is_power_of_2(page_size));
1002
1003 while (address > PM_LEVEL_SIZE(domain->mode))
1004 increase_address_space(domain, gfp);
1005
1006 level = domain->mode - 1;
1007 pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
1008 address = PAGE_SIZE_ALIGN(address, page_size);
1009 end_lvl = PAGE_SIZE_LEVEL(page_size);
1010
1011 while (level > end_lvl) {
1012 if (!IOMMU_PTE_PRESENT(*pte)) {
1013 page = (u64 *)get_zeroed_page(gfp);
1014 if (!page)
1015 return NULL;
1016 *pte = PM_LEVEL_PDE(level, virt_to_phys(page));
1017 }
1018
1019 /* No level skipping support yet */
1020 if (PM_PTE_LEVEL(*pte) != level)
1021 return NULL;
1022
1023 level -= 1;
1024
1025 pte = IOMMU_PTE_PAGE(*pte);
1026
1027 if (pte_page && level == end_lvl)
1028 *pte_page = pte;
1029
1030 pte = &pte[PM_LEVEL_INDEX(level, address)];
1031 }
1032
1033 return pte;
1034 }
1035
1036 /*
1037 * This function checks if there is a PTE for a given dma address. If
1038 * there is one, it returns the pointer to it.
1039 */
1040 static u64 *fetch_pte(struct protection_domain *domain, unsigned long address)
1041 {
1042 int level;
1043 u64 *pte;
1044
1045 if (address > PM_LEVEL_SIZE(domain->mode))
1046 return NULL;
1047
1048 level = domain->mode - 1;
1049 pte = &domain->pt_root[PM_LEVEL_INDEX(level, address)];
1050
1051 while (level > 0) {
1052
1053 /* Not Present */
1054 if (!IOMMU_PTE_PRESENT(*pte))
1055 return NULL;
1056
1057 /* Large PTE */
1058 if (PM_PTE_LEVEL(*pte) == 0x07) {
1059 unsigned long pte_mask, __pte;
1060
1061 /*
1062 * If we have a series of large PTEs, make
1063 * sure to return a pointer to the first one.
1064 */
1065 pte_mask = PTE_PAGE_SIZE(*pte);
1066 pte_mask = ~((PAGE_SIZE_PTE_COUNT(pte_mask) << 3) - 1);
1067 __pte = ((unsigned long)pte) & pte_mask;
1068
1069 return (u64 *)__pte;
1070 }
1071
1072 /* No level skipping support yet */
1073 if (PM_PTE_LEVEL(*pte) != level)
1074 return NULL;
1075
1076 level -= 1;
1077
1078 /* Walk to the next level */
1079 pte = IOMMU_PTE_PAGE(*pte);
1080 pte = &pte[PM_LEVEL_INDEX(level, address)];
1081 }
1082
1083 return pte;
1084 }
1085
1086 /*
1087 * Generic mapping functions. It maps a physical address into a DMA
1088 * address space. It allocates the page table pages if necessary.
1089 * In the future it can be extended to a generic mapping function
1090 * supporting all features of AMD IOMMU page tables like level skipping
1091 * and full 64 bit address spaces.
1092 */
1093 static int iommu_map_page(struct protection_domain *dom,
1094 unsigned long bus_addr,
1095 unsigned long phys_addr,
1096 int prot,
1097 unsigned long page_size)
1098 {
1099 u64 __pte, *pte;
1100 int i, count;
1101
1102 if (!(prot & IOMMU_PROT_MASK))
1103 return -EINVAL;
1104
1105 bus_addr = PAGE_ALIGN(bus_addr);
1106 phys_addr = PAGE_ALIGN(phys_addr);
1107 count = PAGE_SIZE_PTE_COUNT(page_size);
1108 pte = alloc_pte(dom, bus_addr, page_size, NULL, GFP_KERNEL);
1109
1110 for (i = 0; i < count; ++i)
1111 if (IOMMU_PTE_PRESENT(pte[i]))
1112 return -EBUSY;
1113
1114 if (page_size > PAGE_SIZE) {
1115 __pte = PAGE_SIZE_PTE(phys_addr, page_size);
1116 __pte |= PM_LEVEL_ENC(7) | IOMMU_PTE_P | IOMMU_PTE_FC;
1117 } else
1118 __pte = phys_addr | IOMMU_PTE_P | IOMMU_PTE_FC;
1119
1120 if (prot & IOMMU_PROT_IR)
1121 __pte |= IOMMU_PTE_IR;
1122 if (prot & IOMMU_PROT_IW)
1123 __pte |= IOMMU_PTE_IW;
1124
1125 for (i = 0; i < count; ++i)
1126 pte[i] = __pte;
1127
1128 update_domain(dom);
1129
1130 return 0;
1131 }
1132
1133 static unsigned long iommu_unmap_page(struct protection_domain *dom,
1134 unsigned long bus_addr,
1135 unsigned long page_size)
1136 {
1137 unsigned long long unmap_size, unmapped;
1138 u64 *pte;
1139
1140 BUG_ON(!is_power_of_2(page_size));
1141
1142 unmapped = 0;
1143
1144 while (unmapped < page_size) {
1145
1146 pte = fetch_pte(dom, bus_addr);
1147
1148 if (!pte) {
1149 /*
1150 * No PTE for this address
1151 * move forward in 4kb steps
1152 */
1153 unmap_size = PAGE_SIZE;
1154 } else if (PM_PTE_LEVEL(*pte) == 0) {
1155 /* 4kb PTE found for this address */
1156 unmap_size = PAGE_SIZE;
1157 *pte = 0ULL;
1158 } else {
1159 int count, i;
1160
1161 /* Large PTE found which maps this address */
1162 unmap_size = PTE_PAGE_SIZE(*pte);
1163 count = PAGE_SIZE_PTE_COUNT(unmap_size);
1164 for (i = 0; i < count; i++)
1165 pte[i] = 0ULL;
1166 }
1167
1168 bus_addr = (bus_addr & ~(unmap_size - 1)) + unmap_size;
1169 unmapped += unmap_size;
1170 }
1171
1172 BUG_ON(!is_power_of_2(unmapped));
1173
1174 return unmapped;
1175 }
1176
1177 /*
1178 * This function checks if a specific unity mapping entry is needed for
1179 * this specific IOMMU.
1180 */
1181 static int iommu_for_unity_map(struct amd_iommu *iommu,
1182 struct unity_map_entry *entry)
1183 {
1184 u16 bdf, i;
1185
1186 for (i = entry->devid_start; i <= entry->devid_end; ++i) {
1187 bdf = amd_iommu_alias_table[i];
1188 if (amd_iommu_rlookup_table[bdf] == iommu)
1189 return 1;
1190 }
1191
1192 return 0;
1193 }
1194
1195 /*
1196 * This function actually applies the mapping to the page table of the
1197 * dma_ops domain.
1198 */
1199 static int dma_ops_unity_map(struct dma_ops_domain *dma_dom,
1200 struct unity_map_entry *e)
1201 {
1202 u64 addr;
1203 int ret;
1204
1205 for (addr = e->address_start; addr < e->address_end;
1206 addr += PAGE_SIZE) {
1207 ret = iommu_map_page(&dma_dom->domain, addr, addr, e->prot,
1208 PAGE_SIZE);
1209 if (ret)
1210 return ret;
1211 /*
1212 * if unity mapping is in aperture range mark the page
1213 * as allocated in the aperture
1214 */
1215 if (addr < dma_dom->aperture_size)
1216 __set_bit(addr >> PAGE_SHIFT,
1217 dma_dom->aperture[0]->bitmap);
1218 }
1219
1220 return 0;
1221 }
1222
1223 /*
1224 * Init the unity mappings for a specific IOMMU in the system
1225 *
1226 * Basically iterates over all unity mapping entries and applies them to
1227 * the default domain DMA of that IOMMU if necessary.
1228 */
1229 static int iommu_init_unity_mappings(struct amd_iommu *iommu)
1230 {
1231 struct unity_map_entry *entry;
1232 int ret;
1233
1234 list_for_each_entry(entry, &amd_iommu_unity_map, list) {
1235 if (!iommu_for_unity_map(iommu, entry))
1236 continue;
1237 ret = dma_ops_unity_map(iommu->default_dom, entry);
1238 if (ret)
1239 return ret;
1240 }
1241
1242 return 0;
1243 }
1244
1245 /*
1246 * Inits the unity mappings required for a specific device
1247 */
1248 static int init_unity_mappings_for_device(struct dma_ops_domain *dma_dom,
1249 u16 devid)
1250 {
1251 struct unity_map_entry *e;
1252 int ret;
1253
1254 list_for_each_entry(e, &amd_iommu_unity_map, list) {
1255 if (!(devid >= e->devid_start && devid <= e->devid_end))
1256 continue;
1257 ret = dma_ops_unity_map(dma_dom, e);
1258 if (ret)
1259 return ret;
1260 }
1261
1262 return 0;
1263 }
1264
1265 /****************************************************************************
1266 *
1267 * The next functions belong to the address allocator for the dma_ops
1268 * interface functions. They work like the allocators in the other IOMMU
1269 * drivers. Its basically a bitmap which marks the allocated pages in
1270 * the aperture. Maybe it could be enhanced in the future to a more
1271 * efficient allocator.
1272 *
1273 ****************************************************************************/
1274
1275 /*
1276 * The address allocator core functions.
1277 *
1278 * called with domain->lock held
1279 */
1280
1281 /*
1282 * Used to reserve address ranges in the aperture (e.g. for exclusion
1283 * ranges.
1284 */
1285 static void dma_ops_reserve_addresses(struct dma_ops_domain *dom,
1286 unsigned long start_page,
1287 unsigned int pages)
1288 {
1289 unsigned int i, last_page = dom->aperture_size >> PAGE_SHIFT;
1290
1291 if (start_page + pages > last_page)
1292 pages = last_page - start_page;
1293
1294 for (i = start_page; i < start_page + pages; ++i) {
1295 int index = i / APERTURE_RANGE_PAGES;
1296 int page = i % APERTURE_RANGE_PAGES;
1297 __set_bit(page, dom->aperture[index]->bitmap);
1298 }
1299 }
1300
1301 /*
1302 * This function is used to add a new aperture range to an existing
1303 * aperture in case of dma_ops domain allocation or address allocation
1304 * failure.
1305 */
1306 static int alloc_new_range(struct dma_ops_domain *dma_dom,
1307 bool populate, gfp_t gfp)
1308 {
1309 int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
1310 struct amd_iommu *iommu;
1311 unsigned long i, old_size;
1312
1313 #ifdef CONFIG_IOMMU_STRESS
1314 populate = false;
1315 #endif
1316
1317 if (index >= APERTURE_MAX_RANGES)
1318 return -ENOMEM;
1319
1320 dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
1321 if (!dma_dom->aperture[index])
1322 return -ENOMEM;
1323
1324 dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
1325 if (!dma_dom->aperture[index]->bitmap)
1326 goto out_free;
1327
1328 dma_dom->aperture[index]->offset = dma_dom->aperture_size;
1329
1330 if (populate) {
1331 unsigned long address = dma_dom->aperture_size;
1332 int i, num_ptes = APERTURE_RANGE_PAGES / 512;
1333 u64 *pte, *pte_page;
1334
1335 for (i = 0; i < num_ptes; ++i) {
1336 pte = alloc_pte(&dma_dom->domain, address, PAGE_SIZE,
1337 &pte_page, gfp);
1338 if (!pte)
1339 goto out_free;
1340
1341 dma_dom->aperture[index]->pte_pages[i] = pte_page;
1342
1343 address += APERTURE_RANGE_SIZE / 64;
1344 }
1345 }
1346
1347 old_size = dma_dom->aperture_size;
1348 dma_dom->aperture_size += APERTURE_RANGE_SIZE;
1349
1350 /* Reserve address range used for MSI messages */
1351 if (old_size < MSI_ADDR_BASE_LO &&
1352 dma_dom->aperture_size > MSI_ADDR_BASE_LO) {
1353 unsigned long spage;
1354 int pages;
1355
1356 pages = iommu_num_pages(MSI_ADDR_BASE_LO, 0x10000, PAGE_SIZE);
1357 spage = MSI_ADDR_BASE_LO >> PAGE_SHIFT;
1358
1359 dma_ops_reserve_addresses(dma_dom, spage, pages);
1360 }
1361
1362 /* Initialize the exclusion range if necessary */
1363 for_each_iommu(iommu) {
1364 if (iommu->exclusion_start &&
1365 iommu->exclusion_start >= dma_dom->aperture[index]->offset
1366 && iommu->exclusion_start < dma_dom->aperture_size) {
1367 unsigned long startpage;
1368 int pages = iommu_num_pages(iommu->exclusion_start,
1369 iommu->exclusion_length,
1370 PAGE_SIZE);
1371 startpage = iommu->exclusion_start >> PAGE_SHIFT;
1372 dma_ops_reserve_addresses(dma_dom, startpage, pages);
1373 }
1374 }
1375
1376 /*
1377 * Check for areas already mapped as present in the new aperture
1378 * range and mark those pages as reserved in the allocator. Such
1379 * mappings may already exist as a result of requested unity
1380 * mappings for devices.
1381 */
1382 for (i = dma_dom->aperture[index]->offset;
1383 i < dma_dom->aperture_size;
1384 i += PAGE_SIZE) {
1385 u64 *pte = fetch_pte(&dma_dom->domain, i);
1386 if (!pte || !IOMMU_PTE_PRESENT(*pte))
1387 continue;
1388
1389 dma_ops_reserve_addresses(dma_dom, i >> PAGE_SHIFT, 1);
1390 }
1391
1392 update_domain(&dma_dom->domain);
1393
1394 return 0;
1395
1396 out_free:
1397 update_domain(&dma_dom->domain);
1398
1399 free_page((unsigned long)dma_dom->aperture[index]->bitmap);
1400
1401 kfree(dma_dom->aperture[index]);
1402 dma_dom->aperture[index] = NULL;
1403
1404 return -ENOMEM;
1405 }
1406
1407 static unsigned long dma_ops_area_alloc(struct device *dev,
1408 struct dma_ops_domain *dom,
1409 unsigned int pages,
1410 unsigned long align_mask,
1411 u64 dma_mask,
1412 unsigned long start)
1413 {
1414 unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
1415 int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
1416 int i = start >> APERTURE_RANGE_SHIFT;
1417 unsigned long boundary_size;
1418 unsigned long address = -1;
1419 unsigned long limit;
1420
1421 next_bit >>= PAGE_SHIFT;
1422
1423 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
1424 PAGE_SIZE) >> PAGE_SHIFT;
1425
1426 for (;i < max_index; ++i) {
1427 unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
1428
1429 if (dom->aperture[i]->offset >= dma_mask)
1430 break;
1431
1432 limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
1433 dma_mask >> PAGE_SHIFT);
1434
1435 address = iommu_area_alloc(dom->aperture[i]->bitmap,
1436 limit, next_bit, pages, 0,
1437 boundary_size, align_mask);
1438 if (address != -1) {
1439 address = dom->aperture[i]->offset +
1440 (address << PAGE_SHIFT);
1441 dom->next_address = address + (pages << PAGE_SHIFT);
1442 break;
1443 }
1444
1445 next_bit = 0;
1446 }
1447
1448 return address;
1449 }
1450
1451 static unsigned long dma_ops_alloc_addresses(struct device *dev,
1452 struct dma_ops_domain *dom,
1453 unsigned int pages,
1454 unsigned long align_mask,
1455 u64 dma_mask)
1456 {
1457 unsigned long address;
1458
1459 #ifdef CONFIG_IOMMU_STRESS
1460 dom->next_address = 0;
1461 dom->need_flush = true;
1462 #endif
1463
1464 address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1465 dma_mask, dom->next_address);
1466
1467 if (address == -1) {
1468 dom->next_address = 0;
1469 address = dma_ops_area_alloc(dev, dom, pages, align_mask,
1470 dma_mask, 0);
1471 dom->need_flush = true;
1472 }
1473
1474 if (unlikely(address == -1))
1475 address = DMA_ERROR_CODE;
1476
1477 WARN_ON((address + (PAGE_SIZE*pages)) > dom->aperture_size);
1478
1479 return address;
1480 }
1481
1482 /*
1483 * The address free function.
1484 *
1485 * called with domain->lock held
1486 */
1487 static void dma_ops_free_addresses(struct dma_ops_domain *dom,
1488 unsigned long address,
1489 unsigned int pages)
1490 {
1491 unsigned i = address >> APERTURE_RANGE_SHIFT;
1492 struct aperture_range *range = dom->aperture[i];
1493
1494 BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
1495
1496 #ifdef CONFIG_IOMMU_STRESS
1497 if (i < 4)
1498 return;
1499 #endif
1500
1501 if (address >= dom->next_address)
1502 dom->need_flush = true;
1503
1504 address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
1505
1506 bitmap_clear(range->bitmap, address, pages);
1507
1508 }
1509
1510 /****************************************************************************
1511 *
1512 * The next functions belong to the domain allocation. A domain is
1513 * allocated for every IOMMU as the default domain. If device isolation
1514 * is enabled, every device get its own domain. The most important thing
1515 * about domains is the page table mapping the DMA address space they
1516 * contain.
1517 *
1518 ****************************************************************************/
1519
1520 /*
1521 * This function adds a protection domain to the global protection domain list
1522 */
1523 static void add_domain_to_list(struct protection_domain *domain)
1524 {
1525 unsigned long flags;
1526
1527 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1528 list_add(&domain->list, &amd_iommu_pd_list);
1529 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1530 }
1531
1532 /*
1533 * This function removes a protection domain to the global
1534 * protection domain list
1535 */
1536 static void del_domain_from_list(struct protection_domain *domain)
1537 {
1538 unsigned long flags;
1539
1540 spin_lock_irqsave(&amd_iommu_pd_lock, flags);
1541 list_del(&domain->list);
1542 spin_unlock_irqrestore(&amd_iommu_pd_lock, flags);
1543 }
1544
1545 static u16 domain_id_alloc(void)
1546 {
1547 unsigned long flags;
1548 int id;
1549
1550 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1551 id = find_first_zero_bit(amd_iommu_pd_alloc_bitmap, MAX_DOMAIN_ID);
1552 BUG_ON(id == 0);
1553 if (id > 0 && id < MAX_DOMAIN_ID)
1554 __set_bit(id, amd_iommu_pd_alloc_bitmap);
1555 else
1556 id = 0;
1557 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1558
1559 return id;
1560 }
1561
1562 static void domain_id_free(int id)
1563 {
1564 unsigned long flags;
1565
1566 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1567 if (id > 0 && id < MAX_DOMAIN_ID)
1568 __clear_bit(id, amd_iommu_pd_alloc_bitmap);
1569 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1570 }
1571
1572 static void free_pagetable(struct protection_domain *domain)
1573 {
1574 int i, j;
1575 u64 *p1, *p2, *p3;
1576
1577 p1 = domain->pt_root;
1578
1579 if (!p1)
1580 return;
1581
1582 for (i = 0; i < 512; ++i) {
1583 if (!IOMMU_PTE_PRESENT(p1[i]))
1584 continue;
1585
1586 p2 = IOMMU_PTE_PAGE(p1[i]);
1587 for (j = 0; j < 512; ++j) {
1588 if (!IOMMU_PTE_PRESENT(p2[j]))
1589 continue;
1590 p3 = IOMMU_PTE_PAGE(p2[j]);
1591 free_page((unsigned long)p3);
1592 }
1593
1594 free_page((unsigned long)p2);
1595 }
1596
1597 free_page((unsigned long)p1);
1598
1599 domain->pt_root = NULL;
1600 }
1601
1602 static void free_gcr3_table(struct protection_domain *domain)
1603 {
1604 free_page((unsigned long)domain->gcr3_tbl);
1605 }
1606
1607 /*
1608 * Free a domain, only used if something went wrong in the
1609 * allocation path and we need to free an already allocated page table
1610 */
1611 static void dma_ops_domain_free(struct dma_ops_domain *dom)
1612 {
1613 int i;
1614
1615 if (!dom)
1616 return;
1617
1618 del_domain_from_list(&dom->domain);
1619
1620 free_pagetable(&dom->domain);
1621
1622 for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
1623 if (!dom->aperture[i])
1624 continue;
1625 free_page((unsigned long)dom->aperture[i]->bitmap);
1626 kfree(dom->aperture[i]);
1627 }
1628
1629 kfree(dom);
1630 }
1631
1632 /*
1633 * Allocates a new protection domain usable for the dma_ops functions.
1634 * It also initializes the page table and the address allocator data
1635 * structures required for the dma_ops interface
1636 */
1637 static struct dma_ops_domain *dma_ops_domain_alloc(void)
1638 {
1639 struct dma_ops_domain *dma_dom;
1640
1641 dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
1642 if (!dma_dom)
1643 return NULL;
1644
1645 spin_lock_init(&dma_dom->domain.lock);
1646
1647 dma_dom->domain.id = domain_id_alloc();
1648 if (dma_dom->domain.id == 0)
1649 goto free_dma_dom;
1650 INIT_LIST_HEAD(&dma_dom->domain.dev_list);
1651 dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
1652 dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
1653 dma_dom->domain.flags = PD_DMA_OPS_MASK;
1654 dma_dom->domain.priv = dma_dom;
1655 if (!dma_dom->domain.pt_root)
1656 goto free_dma_dom;
1657
1658 dma_dom->need_flush = false;
1659 dma_dom->target_dev = 0xffff;
1660
1661 add_domain_to_list(&dma_dom->domain);
1662
1663 if (alloc_new_range(dma_dom, true, GFP_KERNEL))
1664 goto free_dma_dom;
1665
1666 /*
1667 * mark the first page as allocated so we never return 0 as
1668 * a valid dma-address. So we can use 0 as error value
1669 */
1670 dma_dom->aperture[0]->bitmap[0] = 1;
1671 dma_dom->next_address = 0;
1672
1673
1674 return dma_dom;
1675
1676 free_dma_dom:
1677 dma_ops_domain_free(dma_dom);
1678
1679 return NULL;
1680 }
1681
1682 /*
1683 * little helper function to check whether a given protection domain is a
1684 * dma_ops domain
1685 */
1686 static bool dma_ops_domain(struct protection_domain *domain)
1687 {
1688 return domain->flags & PD_DMA_OPS_MASK;
1689 }
1690
1691 static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats)
1692 {
1693 u64 pte_root = 0;
1694 u64 flags = 0;
1695
1696 if (domain->mode != PAGE_MODE_NONE)
1697 pte_root = virt_to_phys(domain->pt_root);
1698
1699 pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK)
1700 << DEV_ENTRY_MODE_SHIFT;
1701 pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV;
1702
1703 flags = amd_iommu_dev_table[devid].data[1];
1704
1705 if (ats)
1706 flags |= DTE_FLAG_IOTLB;
1707
1708 if (domain->flags & PD_IOMMUV2_MASK) {
1709 u64 gcr3 = __pa(domain->gcr3_tbl);
1710 u64 glx = domain->glx;
1711 u64 tmp;
1712
1713 pte_root |= DTE_FLAG_GV;
1714 pte_root |= (glx & DTE_GLX_MASK) << DTE_GLX_SHIFT;
1715
1716 /* First mask out possible old values for GCR3 table */
1717 tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B;
1718 flags &= ~tmp;
1719
1720 tmp = DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C;
1721 flags &= ~tmp;
1722
1723 /* Encode GCR3 table into DTE */
1724 tmp = DTE_GCR3_VAL_A(gcr3) << DTE_GCR3_SHIFT_A;
1725 pte_root |= tmp;
1726
1727 tmp = DTE_GCR3_VAL_B(gcr3) << DTE_GCR3_SHIFT_B;
1728 flags |= tmp;
1729
1730 tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C;
1731 flags |= tmp;
1732 }
1733
1734 flags &= ~(0xffffUL);
1735 flags |= domain->id;
1736
1737 amd_iommu_dev_table[devid].data[1] = flags;
1738 amd_iommu_dev_table[devid].data[0] = pte_root;
1739 }
1740
1741 static void clear_dte_entry(u16 devid)
1742 {
1743 /* remove entry from the device table seen by the hardware */
1744 amd_iommu_dev_table[devid].data[0] = IOMMU_PTE_P | IOMMU_PTE_TV;
1745 amd_iommu_dev_table[devid].data[1] = 0;
1746
1747 amd_iommu_apply_erratum_63(devid);
1748 }
1749
1750 static void do_attach(struct iommu_dev_data *dev_data,
1751 struct protection_domain *domain)
1752 {
1753 struct amd_iommu *iommu;
1754 bool ats;
1755
1756 iommu = amd_iommu_rlookup_table[dev_data->devid];
1757 ats = dev_data->ats.enabled;
1758
1759 /* Update data structures */
1760 dev_data->domain = domain;
1761 list_add(&dev_data->list, &domain->dev_list);
1762 set_dte_entry(dev_data->devid, domain, ats);
1763
1764 /* Do reference counting */
1765 domain->dev_iommu[iommu->index] += 1;
1766 domain->dev_cnt += 1;
1767
1768 /* Flush the DTE entry */
1769 device_flush_dte(dev_data);
1770 }
1771
1772 static void do_detach(struct iommu_dev_data *dev_data)
1773 {
1774 struct amd_iommu *iommu;
1775
1776 iommu = amd_iommu_rlookup_table[dev_data->devid];
1777
1778 /* decrease reference counters */
1779 dev_data->domain->dev_iommu[iommu->index] -= 1;
1780 dev_data->domain->dev_cnt -= 1;
1781
1782 /* Update data structures */
1783 dev_data->domain = NULL;
1784 list_del(&dev_data->list);
1785 clear_dte_entry(dev_data->devid);
1786
1787 /* Flush the DTE entry */
1788 device_flush_dte(dev_data);
1789 }
1790
1791 /*
1792 * If a device is not yet associated with a domain, this function does
1793 * assigns it visible for the hardware
1794 */
1795 static int __attach_device(struct iommu_dev_data *dev_data,
1796 struct protection_domain *domain)
1797 {
1798 int ret;
1799
1800 /* lock domain */
1801 spin_lock(&domain->lock);
1802
1803 if (dev_data->alias_data != NULL) {
1804 struct iommu_dev_data *alias_data = dev_data->alias_data;
1805
1806 /* Some sanity checks */
1807 ret = -EBUSY;
1808 if (alias_data->domain != NULL &&
1809 alias_data->domain != domain)
1810 goto out_unlock;
1811
1812 if (dev_data->domain != NULL &&
1813 dev_data->domain != domain)
1814 goto out_unlock;
1815
1816 /* Do real assignment */
1817 if (alias_data->domain == NULL)
1818 do_attach(alias_data, domain);
1819
1820 atomic_inc(&alias_data->bind);
1821 }
1822
1823 if (dev_data->domain == NULL)
1824 do_attach(dev_data, domain);
1825
1826 atomic_inc(&dev_data->bind);
1827
1828 ret = 0;
1829
1830 out_unlock:
1831
1832 /* ready */
1833 spin_unlock(&domain->lock);
1834
1835 return ret;
1836 }
1837
1838
1839 static void pdev_iommuv2_disable(struct pci_dev *pdev)
1840 {
1841 pci_disable_ats(pdev);
1842 pci_disable_pri(pdev);
1843 pci_disable_pasid(pdev);
1844 }
1845
1846 static int pdev_iommuv2_enable(struct pci_dev *pdev)
1847 {
1848 int ret;
1849
1850 /* Only allow access to user-accessible pages */
1851 ret = pci_enable_pasid(pdev, 0);
1852 if (ret)
1853 goto out_err;
1854
1855 /* First reset the PRI state of the device */
1856 ret = pci_reset_pri(pdev);
1857 if (ret)
1858 goto out_err;
1859
1860 /* FIXME: Hardcode number of outstanding requests for now */
1861 ret = pci_enable_pri(pdev, 32);
1862 if (ret)
1863 goto out_err;
1864
1865 ret = pci_enable_ats(pdev, PAGE_SHIFT);
1866 if (ret)
1867 goto out_err;
1868
1869 return 0;
1870
1871 out_err:
1872 pci_disable_pri(pdev);
1873 pci_disable_pasid(pdev);
1874
1875 return ret;
1876 }
1877
1878 /*
1879 * If a device is not yet associated with a domain, this function does
1880 * assigns it visible for the hardware
1881 */
1882 static int attach_device(struct device *dev,
1883 struct protection_domain *domain)
1884 {
1885 struct pci_dev *pdev = to_pci_dev(dev);
1886 struct iommu_dev_data *dev_data;
1887 unsigned long flags;
1888 int ret;
1889
1890 dev_data = get_dev_data(dev);
1891
1892 if (domain->flags & PD_IOMMUV2_MASK) {
1893 if (!dev_data->iommu_v2 || !dev_data->passthrough)
1894 return -EINVAL;
1895
1896 if (pdev_iommuv2_enable(pdev) != 0)
1897 return -EINVAL;
1898
1899 dev_data->ats.enabled = true;
1900 dev_data->ats.qdep = pci_ats_queue_depth(pdev);
1901 } else if (amd_iommu_iotlb_sup &&
1902 pci_enable_ats(pdev, PAGE_SHIFT) == 0) {
1903 dev_data->ats.enabled = true;
1904 dev_data->ats.qdep = pci_ats_queue_depth(pdev);
1905 }
1906
1907 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1908 ret = __attach_device(dev_data, domain);
1909 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1910
1911 /*
1912 * We might boot into a crash-kernel here. The crashed kernel
1913 * left the caches in the IOMMU dirty. So we have to flush
1914 * here to evict all dirty stuff.
1915 */
1916 domain_flush_tlb_pde(domain);
1917
1918 return ret;
1919 }
1920
1921 /*
1922 * Removes a device from a protection domain (unlocked)
1923 */
1924 static void __detach_device(struct iommu_dev_data *dev_data)
1925 {
1926 struct protection_domain *domain;
1927 unsigned long flags;
1928
1929 BUG_ON(!dev_data->domain);
1930
1931 domain = dev_data->domain;
1932
1933 spin_lock_irqsave(&domain->lock, flags);
1934
1935 if (dev_data->alias_data != NULL) {
1936 struct iommu_dev_data *alias_data = dev_data->alias_data;
1937
1938 if (atomic_dec_and_test(&alias_data->bind))
1939 do_detach(alias_data);
1940 }
1941
1942 if (atomic_dec_and_test(&dev_data->bind))
1943 do_detach(dev_data);
1944
1945 spin_unlock_irqrestore(&domain->lock, flags);
1946
1947 /*
1948 * If we run in passthrough mode the device must be assigned to the
1949 * passthrough domain if it is detached from any other domain.
1950 * Make sure we can deassign from the pt_domain itself.
1951 */
1952 if (dev_data->passthrough &&
1953 (dev_data->domain == NULL && domain != pt_domain))
1954 __attach_device(dev_data, pt_domain);
1955 }
1956
1957 /*
1958 * Removes a device from a protection domain (with devtable_lock held)
1959 */
1960 static void detach_device(struct device *dev)
1961 {
1962 struct protection_domain *domain;
1963 struct iommu_dev_data *dev_data;
1964 unsigned long flags;
1965
1966 dev_data = get_dev_data(dev);
1967 domain = dev_data->domain;
1968
1969 /* lock device table */
1970 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
1971 __detach_device(dev_data);
1972 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
1973
1974 if (domain->flags & PD_IOMMUV2_MASK)
1975 pdev_iommuv2_disable(to_pci_dev(dev));
1976 else if (dev_data->ats.enabled)
1977 pci_disable_ats(to_pci_dev(dev));
1978
1979 dev_data->ats.enabled = false;
1980 }
1981
1982 /*
1983 * Find out the protection domain structure for a given PCI device. This
1984 * will give us the pointer to the page table root for example.
1985 */
1986 static struct protection_domain *domain_for_device(struct device *dev)
1987 {
1988 struct iommu_dev_data *dev_data;
1989 struct protection_domain *dom = NULL;
1990 unsigned long flags;
1991
1992 dev_data = get_dev_data(dev);
1993
1994 if (dev_data->domain)
1995 return dev_data->domain;
1996
1997 if (dev_data->alias_data != NULL) {
1998 struct iommu_dev_data *alias_data = dev_data->alias_data;
1999
2000 read_lock_irqsave(&amd_iommu_devtable_lock, flags);
2001 if (alias_data->domain != NULL) {
2002 __attach_device(dev_data, alias_data->domain);
2003 dom = alias_data->domain;
2004 }
2005 read_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2006 }
2007
2008 return dom;
2009 }
2010
2011 static int device_change_notifier(struct notifier_block *nb,
2012 unsigned long action, void *data)
2013 {
2014 struct dma_ops_domain *dma_domain;
2015 struct protection_domain *domain;
2016 struct iommu_dev_data *dev_data;
2017 struct device *dev = data;
2018 struct amd_iommu *iommu;
2019 unsigned long flags;
2020 u16 devid;
2021
2022 if (!check_device(dev))
2023 return 0;
2024
2025 devid = get_device_id(dev);
2026 iommu = amd_iommu_rlookup_table[devid];
2027 dev_data = get_dev_data(dev);
2028
2029 switch (action) {
2030 case BUS_NOTIFY_UNBOUND_DRIVER:
2031
2032 domain = domain_for_device(dev);
2033
2034 if (!domain)
2035 goto out;
2036 if (dev_data->passthrough)
2037 break;
2038 detach_device(dev);
2039 break;
2040 case BUS_NOTIFY_ADD_DEVICE:
2041
2042 iommu_init_device(dev);
2043
2044 domain = domain_for_device(dev);
2045
2046 /* allocate a protection domain if a device is added */
2047 dma_domain = find_protection_domain(devid);
2048 if (dma_domain)
2049 goto out;
2050 dma_domain = dma_ops_domain_alloc();
2051 if (!dma_domain)
2052 goto out;
2053 dma_domain->target_dev = devid;
2054
2055 spin_lock_irqsave(&iommu_pd_list_lock, flags);
2056 list_add_tail(&dma_domain->list, &iommu_pd_list);
2057 spin_unlock_irqrestore(&iommu_pd_list_lock, flags);
2058
2059 break;
2060 case BUS_NOTIFY_DEL_DEVICE:
2061
2062 iommu_uninit_device(dev);
2063
2064 default:
2065 goto out;
2066 }
2067
2068 iommu_completion_wait(iommu);
2069
2070 out:
2071 return 0;
2072 }
2073
2074 static struct notifier_block device_nb = {
2075 .notifier_call = device_change_notifier,
2076 };
2077
2078 void amd_iommu_init_notifier(void)
2079 {
2080 bus_register_notifier(&pci_bus_type, &device_nb);
2081 }
2082
2083 /*****************************************************************************
2084 *
2085 * The next functions belong to the dma_ops mapping/unmapping code.
2086 *
2087 *****************************************************************************/
2088
2089 /*
2090 * In the dma_ops path we only have the struct device. This function
2091 * finds the corresponding IOMMU, the protection domain and the
2092 * requestor id for a given device.
2093 * If the device is not yet associated with a domain this is also done
2094 * in this function.
2095 */
2096 static struct protection_domain *get_domain(struct device *dev)
2097 {
2098 struct protection_domain *domain;
2099 struct dma_ops_domain *dma_dom;
2100 u16 devid = get_device_id(dev);
2101
2102 if (!check_device(dev))
2103 return ERR_PTR(-EINVAL);
2104
2105 domain = domain_for_device(dev);
2106 if (domain != NULL && !dma_ops_domain(domain))
2107 return ERR_PTR(-EBUSY);
2108
2109 if (domain != NULL)
2110 return domain;
2111
2112 /* Device not bount yet - bind it */
2113 dma_dom = find_protection_domain(devid);
2114 if (!dma_dom)
2115 dma_dom = amd_iommu_rlookup_table[devid]->default_dom;
2116 attach_device(dev, &dma_dom->domain);
2117 DUMP_printk("Using protection domain %d for device %s\n",
2118 dma_dom->domain.id, dev_name(dev));
2119
2120 return &dma_dom->domain;
2121 }
2122
2123 static void update_device_table(struct protection_domain *domain)
2124 {
2125 struct iommu_dev_data *dev_data;
2126
2127 list_for_each_entry(dev_data, &domain->dev_list, list)
2128 set_dte_entry(dev_data->devid, domain, dev_data->ats.enabled);
2129 }
2130
2131 static void update_domain(struct protection_domain *domain)
2132 {
2133 if (!domain->updated)
2134 return;
2135
2136 update_device_table(domain);
2137
2138 domain_flush_devices(domain);
2139 domain_flush_tlb_pde(domain);
2140
2141 domain->updated = false;
2142 }
2143
2144 /*
2145 * This function fetches the PTE for a given address in the aperture
2146 */
2147 static u64* dma_ops_get_pte(struct dma_ops_domain *dom,
2148 unsigned long address)
2149 {
2150 struct aperture_range *aperture;
2151 u64 *pte, *pte_page;
2152
2153 aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
2154 if (!aperture)
2155 return NULL;
2156
2157 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2158 if (!pte) {
2159 pte = alloc_pte(&dom->domain, address, PAGE_SIZE, &pte_page,
2160 GFP_ATOMIC);
2161 aperture->pte_pages[APERTURE_PAGE_INDEX(address)] = pte_page;
2162 } else
2163 pte += PM_LEVEL_INDEX(0, address);
2164
2165 update_domain(&dom->domain);
2166
2167 return pte;
2168 }
2169
2170 /*
2171 * This is the generic map function. It maps one 4kb page at paddr to
2172 * the given address in the DMA address space for the domain.
2173 */
2174 static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
2175 unsigned long address,
2176 phys_addr_t paddr,
2177 int direction)
2178 {
2179 u64 *pte, __pte;
2180
2181 WARN_ON(address > dom->aperture_size);
2182
2183 paddr &= PAGE_MASK;
2184
2185 pte = dma_ops_get_pte(dom, address);
2186 if (!pte)
2187 return DMA_ERROR_CODE;
2188
2189 __pte = paddr | IOMMU_PTE_P | IOMMU_PTE_FC;
2190
2191 if (direction == DMA_TO_DEVICE)
2192 __pte |= IOMMU_PTE_IR;
2193 else if (direction == DMA_FROM_DEVICE)
2194 __pte |= IOMMU_PTE_IW;
2195 else if (direction == DMA_BIDIRECTIONAL)
2196 __pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
2197
2198 WARN_ON(*pte);
2199
2200 *pte = __pte;
2201
2202 return (dma_addr_t)address;
2203 }
2204
2205 /*
2206 * The generic unmapping function for on page in the DMA address space.
2207 */
2208 static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
2209 unsigned long address)
2210 {
2211 struct aperture_range *aperture;
2212 u64 *pte;
2213
2214 if (address >= dom->aperture_size)
2215 return;
2216
2217 aperture = dom->aperture[APERTURE_RANGE_INDEX(address)];
2218 if (!aperture)
2219 return;
2220
2221 pte = aperture->pte_pages[APERTURE_PAGE_INDEX(address)];
2222 if (!pte)
2223 return;
2224
2225 pte += PM_LEVEL_INDEX(0, address);
2226
2227 WARN_ON(!*pte);
2228
2229 *pte = 0ULL;
2230 }
2231
2232 /*
2233 * This function contains common code for mapping of a physically
2234 * contiguous memory region into DMA address space. It is used by all
2235 * mapping functions provided with this IOMMU driver.
2236 * Must be called with the domain lock held.
2237 */
2238 static dma_addr_t __map_single(struct device *dev,
2239 struct dma_ops_domain *dma_dom,
2240 phys_addr_t paddr,
2241 size_t size,
2242 int dir,
2243 bool align,
2244 u64 dma_mask)
2245 {
2246 dma_addr_t offset = paddr & ~PAGE_MASK;
2247 dma_addr_t address, start, ret;
2248 unsigned int pages;
2249 unsigned long align_mask = 0;
2250 int i;
2251
2252 pages = iommu_num_pages(paddr, size, PAGE_SIZE);
2253 paddr &= PAGE_MASK;
2254
2255 INC_STATS_COUNTER(total_map_requests);
2256
2257 if (pages > 1)
2258 INC_STATS_COUNTER(cross_page);
2259
2260 if (align)
2261 align_mask = (1UL << get_order(size)) - 1;
2262
2263 retry:
2264 address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
2265 dma_mask);
2266 if (unlikely(address == DMA_ERROR_CODE)) {
2267 /*
2268 * setting next_address here will let the address
2269 * allocator only scan the new allocated range in the
2270 * first run. This is a small optimization.
2271 */
2272 dma_dom->next_address = dma_dom->aperture_size;
2273
2274 if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
2275 goto out;
2276
2277 /*
2278 * aperture was successfully enlarged by 128 MB, try
2279 * allocation again
2280 */
2281 goto retry;
2282 }
2283
2284 start = address;
2285 for (i = 0; i < pages; ++i) {
2286 ret = dma_ops_domain_map(dma_dom, start, paddr, dir);
2287 if (ret == DMA_ERROR_CODE)
2288 goto out_unmap;
2289
2290 paddr += PAGE_SIZE;
2291 start += PAGE_SIZE;
2292 }
2293 address += offset;
2294
2295 ADD_STATS_COUNTER(alloced_io_mem, size);
2296
2297 if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
2298 domain_flush_tlb(&dma_dom->domain);
2299 dma_dom->need_flush = false;
2300 } else if (unlikely(amd_iommu_np_cache))
2301 domain_flush_pages(&dma_dom->domain, address, size);
2302
2303 out:
2304 return address;
2305
2306 out_unmap:
2307
2308 for (--i; i >= 0; --i) {
2309 start -= PAGE_SIZE;
2310 dma_ops_domain_unmap(dma_dom, start);
2311 }
2312
2313 dma_ops_free_addresses(dma_dom, address, pages);
2314
2315 return DMA_ERROR_CODE;
2316 }
2317
2318 /*
2319 * Does the reverse of the __map_single function. Must be called with
2320 * the domain lock held too
2321 */
2322 static void __unmap_single(struct dma_ops_domain *dma_dom,
2323 dma_addr_t dma_addr,
2324 size_t size,
2325 int dir)
2326 {
2327 dma_addr_t flush_addr;
2328 dma_addr_t i, start;
2329 unsigned int pages;
2330
2331 if ((dma_addr == DMA_ERROR_CODE) ||
2332 (dma_addr + size > dma_dom->aperture_size))
2333 return;
2334
2335 flush_addr = dma_addr;
2336 pages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
2337 dma_addr &= PAGE_MASK;
2338 start = dma_addr;
2339
2340 for (i = 0; i < pages; ++i) {
2341 dma_ops_domain_unmap(dma_dom, start);
2342 start += PAGE_SIZE;
2343 }
2344
2345 SUB_STATS_COUNTER(alloced_io_mem, size);
2346
2347 dma_ops_free_addresses(dma_dom, dma_addr, pages);
2348
2349 if (amd_iommu_unmap_flush || dma_dom->need_flush) {
2350 domain_flush_pages(&dma_dom->domain, flush_addr, size);
2351 dma_dom->need_flush = false;
2352 }
2353 }
2354
2355 /*
2356 * The exported map_single function for dma_ops.
2357 */
2358 static dma_addr_t map_page(struct device *dev, struct page *page,
2359 unsigned long offset, size_t size,
2360 enum dma_data_direction dir,
2361 struct dma_attrs *attrs)
2362 {
2363 unsigned long flags;
2364 struct protection_domain *domain;
2365 dma_addr_t addr;
2366 u64 dma_mask;
2367 phys_addr_t paddr = page_to_phys(page) + offset;
2368
2369 INC_STATS_COUNTER(cnt_map_single);
2370
2371 domain = get_domain(dev);
2372 if (PTR_ERR(domain) == -EINVAL)
2373 return (dma_addr_t)paddr;
2374 else if (IS_ERR(domain))
2375 return DMA_ERROR_CODE;
2376
2377 dma_mask = *dev->dma_mask;
2378
2379 spin_lock_irqsave(&domain->lock, flags);
2380
2381 addr = __map_single(dev, domain->priv, paddr, size, dir, false,
2382 dma_mask);
2383 if (addr == DMA_ERROR_CODE)
2384 goto out;
2385
2386 domain_flush_complete(domain);
2387
2388 out:
2389 spin_unlock_irqrestore(&domain->lock, flags);
2390
2391 return addr;
2392 }
2393
2394 /*
2395 * The exported unmap_single function for dma_ops.
2396 */
2397 static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
2398 enum dma_data_direction dir, struct dma_attrs *attrs)
2399 {
2400 unsigned long flags;
2401 struct protection_domain *domain;
2402
2403 INC_STATS_COUNTER(cnt_unmap_single);
2404
2405 domain = get_domain(dev);
2406 if (IS_ERR(domain))
2407 return;
2408
2409 spin_lock_irqsave(&domain->lock, flags);
2410
2411 __unmap_single(domain->priv, dma_addr, size, dir);
2412
2413 domain_flush_complete(domain);
2414
2415 spin_unlock_irqrestore(&domain->lock, flags);
2416 }
2417
2418 /*
2419 * This is a special map_sg function which is used if we should map a
2420 * device which is not handled by an AMD IOMMU in the system.
2421 */
2422 static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist,
2423 int nelems, int dir)
2424 {
2425 struct scatterlist *s;
2426 int i;
2427
2428 for_each_sg(sglist, s, nelems, i) {
2429 s->dma_address = (dma_addr_t)sg_phys(s);
2430 s->dma_length = s->length;
2431 }
2432
2433 return nelems;
2434 }
2435
2436 /*
2437 * The exported map_sg function for dma_ops (handles scatter-gather
2438 * lists).
2439 */
2440 static int map_sg(struct device *dev, struct scatterlist *sglist,
2441 int nelems, enum dma_data_direction dir,
2442 struct dma_attrs *attrs)
2443 {
2444 unsigned long flags;
2445 struct protection_domain *domain;
2446 int i;
2447 struct scatterlist *s;
2448 phys_addr_t paddr;
2449 int mapped_elems = 0;
2450 u64 dma_mask;
2451
2452 INC_STATS_COUNTER(cnt_map_sg);
2453
2454 domain = get_domain(dev);
2455 if (PTR_ERR(domain) == -EINVAL)
2456 return map_sg_no_iommu(dev, sglist, nelems, dir);
2457 else if (IS_ERR(domain))
2458 return 0;
2459
2460 dma_mask = *dev->dma_mask;
2461
2462 spin_lock_irqsave(&domain->lock, flags);
2463
2464 for_each_sg(sglist, s, nelems, i) {
2465 paddr = sg_phys(s);
2466
2467 s->dma_address = __map_single(dev, domain->priv,
2468 paddr, s->length, dir, false,
2469 dma_mask);
2470
2471 if (s->dma_address) {
2472 s->dma_length = s->length;
2473 mapped_elems++;
2474 } else
2475 goto unmap;
2476 }
2477
2478 domain_flush_complete(domain);
2479
2480 out:
2481 spin_unlock_irqrestore(&domain->lock, flags);
2482
2483 return mapped_elems;
2484 unmap:
2485 for_each_sg(sglist, s, mapped_elems, i) {
2486 if (s->dma_address)
2487 __unmap_single(domain->priv, s->dma_address,
2488 s->dma_length, dir);
2489 s->dma_address = s->dma_length = 0;
2490 }
2491
2492 mapped_elems = 0;
2493
2494 goto out;
2495 }
2496
2497 /*
2498 * The exported map_sg function for dma_ops (handles scatter-gather
2499 * lists).
2500 */
2501 static void unmap_sg(struct device *dev, struct scatterlist *sglist,
2502 int nelems, enum dma_data_direction dir,
2503 struct dma_attrs *attrs)
2504 {
2505 unsigned long flags;
2506 struct protection_domain *domain;
2507 struct scatterlist *s;
2508 int i;
2509
2510 INC_STATS_COUNTER(cnt_unmap_sg);
2511
2512 domain = get_domain(dev);
2513 if (IS_ERR(domain))
2514 return;
2515
2516 spin_lock_irqsave(&domain->lock, flags);
2517
2518 for_each_sg(sglist, s, nelems, i) {
2519 __unmap_single(domain->priv, s->dma_address,
2520 s->dma_length, dir);
2521 s->dma_address = s->dma_length = 0;
2522 }
2523
2524 domain_flush_complete(domain);
2525
2526 spin_unlock_irqrestore(&domain->lock, flags);
2527 }
2528
2529 /*
2530 * The exported alloc_coherent function for dma_ops.
2531 */
2532 static void *alloc_coherent(struct device *dev, size_t size,
2533 dma_addr_t *dma_addr, gfp_t flag)
2534 {
2535 unsigned long flags;
2536 void *virt_addr;
2537 struct protection_domain *domain;
2538 phys_addr_t paddr;
2539 u64 dma_mask = dev->coherent_dma_mask;
2540
2541 INC_STATS_COUNTER(cnt_alloc_coherent);
2542
2543 domain = get_domain(dev);
2544 if (PTR_ERR(domain) == -EINVAL) {
2545 virt_addr = (void *)__get_free_pages(flag, get_order(size));
2546 *dma_addr = __pa(virt_addr);
2547 return virt_addr;
2548 } else if (IS_ERR(domain))
2549 return NULL;
2550
2551 dma_mask = dev->coherent_dma_mask;
2552 flag &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32);
2553 flag |= __GFP_ZERO;
2554
2555 virt_addr = (void *)__get_free_pages(flag, get_order(size));
2556 if (!virt_addr)
2557 return NULL;
2558
2559 paddr = virt_to_phys(virt_addr);
2560
2561 if (!dma_mask)
2562 dma_mask = *dev->dma_mask;
2563
2564 spin_lock_irqsave(&domain->lock, flags);
2565
2566 *dma_addr = __map_single(dev, domain->priv, paddr,
2567 size, DMA_BIDIRECTIONAL, true, dma_mask);
2568
2569 if (*dma_addr == DMA_ERROR_CODE) {
2570 spin_unlock_irqrestore(&domain->lock, flags);
2571 goto out_free;
2572 }
2573
2574 domain_flush_complete(domain);
2575
2576 spin_unlock_irqrestore(&domain->lock, flags);
2577
2578 return virt_addr;
2579
2580 out_free:
2581
2582 free_pages((unsigned long)virt_addr, get_order(size));
2583
2584 return NULL;
2585 }
2586
2587 /*
2588 * The exported free_coherent function for dma_ops.
2589 */
2590 static void free_coherent(struct device *dev, size_t size,
2591 void *virt_addr, dma_addr_t dma_addr)
2592 {
2593 unsigned long flags;
2594 struct protection_domain *domain;
2595
2596 INC_STATS_COUNTER(cnt_free_coherent);
2597
2598 domain = get_domain(dev);
2599 if (IS_ERR(domain))
2600 goto free_mem;
2601
2602 spin_lock_irqsave(&domain->lock, flags);
2603
2604 __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
2605
2606 domain_flush_complete(domain);
2607
2608 spin_unlock_irqrestore(&domain->lock, flags);
2609
2610 free_mem:
2611 free_pages((unsigned long)virt_addr, get_order(size));
2612 }
2613
2614 /*
2615 * This function is called by the DMA layer to find out if we can handle a
2616 * particular device. It is part of the dma_ops.
2617 */
2618 static int amd_iommu_dma_supported(struct device *dev, u64 mask)
2619 {
2620 return check_device(dev);
2621 }
2622
2623 /*
2624 * The function for pre-allocating protection domains.
2625 *
2626 * If the driver core informs the DMA layer if a driver grabs a device
2627 * we don't need to preallocate the protection domains anymore.
2628 * For now we have to.
2629 */
2630 static void prealloc_protection_domains(void)
2631 {
2632 struct iommu_dev_data *dev_data;
2633 struct dma_ops_domain *dma_dom;
2634 struct pci_dev *dev = NULL;
2635 u16 devid;
2636
2637 for_each_pci_dev(dev) {
2638
2639 /* Do we handle this device? */
2640 if (!check_device(&dev->dev))
2641 continue;
2642
2643 dev_data = get_dev_data(&dev->dev);
2644 if (!amd_iommu_force_isolation && dev_data->iommu_v2) {
2645 /* Make sure passthrough domain is allocated */
2646 alloc_passthrough_domain();
2647 dev_data->passthrough = true;
2648 attach_device(&dev->dev, pt_domain);
2649 pr_info("AMD-Vi: Using passthough domain for device %s\n",
2650 dev_name(&dev->dev));
2651 }
2652
2653 /* Is there already any domain for it? */
2654 if (domain_for_device(&dev->dev))
2655 continue;
2656
2657 devid = get_device_id(&dev->dev);
2658
2659 dma_dom = dma_ops_domain_alloc();
2660 if (!dma_dom)
2661 continue;
2662 init_unity_mappings_for_device(dma_dom, devid);
2663 dma_dom->target_dev = devid;
2664
2665 attach_device(&dev->dev, &dma_dom->domain);
2666
2667 list_add_tail(&dma_dom->list, &iommu_pd_list);
2668 }
2669 }
2670
2671 static struct dma_map_ops amd_iommu_dma_ops = {
2672 .alloc_coherent = alloc_coherent,
2673 .free_coherent = free_coherent,
2674 .map_page = map_page,
2675 .unmap_page = unmap_page,
2676 .map_sg = map_sg,
2677 .unmap_sg = unmap_sg,
2678 .dma_supported = amd_iommu_dma_supported,
2679 };
2680
2681 static unsigned device_dma_ops_init(void)
2682 {
2683 struct iommu_dev_data *dev_data;
2684 struct pci_dev *pdev = NULL;
2685 unsigned unhandled = 0;
2686
2687 for_each_pci_dev(pdev) {
2688 if (!check_device(&pdev->dev)) {
2689 unhandled += 1;
2690 continue;
2691 }
2692
2693 dev_data = get_dev_data(&pdev->dev);
2694
2695 if (!dev_data->passthrough)
2696 pdev->dev.archdata.dma_ops = &amd_iommu_dma_ops;
2697 else
2698 pdev->dev.archdata.dma_ops = &nommu_dma_ops;
2699 }
2700
2701 return unhandled;
2702 }
2703
2704 /*
2705 * The function which clues the AMD IOMMU driver into dma_ops.
2706 */
2707
2708 void __init amd_iommu_init_api(void)
2709 {
2710 bus_set_iommu(&pci_bus_type, &amd_iommu_ops);
2711 }
2712
2713 int __init amd_iommu_init_dma_ops(void)
2714 {
2715 struct amd_iommu *iommu;
2716 int ret, unhandled;
2717
2718 /*
2719 * first allocate a default protection domain for every IOMMU we
2720 * found in the system. Devices not assigned to any other
2721 * protection domain will be assigned to the default one.
2722 */
2723 for_each_iommu(iommu) {
2724 iommu->default_dom = dma_ops_domain_alloc();
2725 if (iommu->default_dom == NULL)
2726 return -ENOMEM;
2727 iommu->default_dom->domain.flags |= PD_DEFAULT_MASK;
2728 ret = iommu_init_unity_mappings(iommu);
2729 if (ret)
2730 goto free_domains;
2731 }
2732
2733 /*
2734 * Pre-allocate the protection domains for each device.
2735 */
2736 prealloc_protection_domains();
2737
2738 iommu_detected = 1;
2739 swiotlb = 0;
2740
2741 /* Make the driver finally visible to the drivers */
2742 unhandled = device_dma_ops_init();
2743 if (unhandled && max_pfn > MAX_DMA32_PFN) {
2744 /* There are unhandled devices - initialize swiotlb for them */
2745 swiotlb = 1;
2746 }
2747
2748 amd_iommu_stats_init();
2749
2750 return 0;
2751
2752 free_domains:
2753
2754 for_each_iommu(iommu) {
2755 if (iommu->default_dom)
2756 dma_ops_domain_free(iommu->default_dom);
2757 }
2758
2759 return ret;
2760 }
2761
2762 /*****************************************************************************
2763 *
2764 * The following functions belong to the exported interface of AMD IOMMU
2765 *
2766 * This interface allows access to lower level functions of the IOMMU
2767 * like protection domain handling and assignement of devices to domains
2768 * which is not possible with the dma_ops interface.
2769 *
2770 *****************************************************************************/
2771
2772 static void cleanup_domain(struct protection_domain *domain)
2773 {
2774 struct iommu_dev_data *dev_data, *next;
2775 unsigned long flags;
2776
2777 write_lock_irqsave(&amd_iommu_devtable_lock, flags);
2778
2779 list_for_each_entry_safe(dev_data, next, &domain->dev_list, list) {
2780 __detach_device(dev_data);
2781 atomic_set(&dev_data->bind, 0);
2782 }
2783
2784 write_unlock_irqrestore(&amd_iommu_devtable_lock, flags);
2785 }
2786
2787 static void protection_domain_free(struct protection_domain *domain)
2788 {
2789 if (!domain)
2790 return;
2791
2792 del_domain_from_list(domain);
2793
2794 if (domain->id)
2795 domain_id_free(domain->id);
2796
2797 kfree(domain);
2798 }
2799
2800 static struct protection_domain *protection_domain_alloc(void)
2801 {
2802 struct protection_domain *domain;
2803
2804 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
2805 if (!domain)
2806 return NULL;
2807
2808 spin_lock_init(&domain->lock);
2809 mutex_init(&domain->api_lock);
2810 domain->id = domain_id_alloc();
2811 if (!domain->id)
2812 goto out_err;
2813 INIT_LIST_HEAD(&domain->dev_list);
2814
2815 add_domain_to_list(domain);
2816
2817 return domain;
2818
2819 out_err:
2820 kfree(domain);
2821
2822 return NULL;
2823 }
2824
2825 static int __init alloc_passthrough_domain(void)
2826 {
2827 if (pt_domain != NULL)
2828 return 0;
2829
2830 /* allocate passthrough domain */
2831 pt_domain = protection_domain_alloc();
2832 if (!pt_domain)
2833 return -ENOMEM;
2834
2835 pt_domain->mode = PAGE_MODE_NONE;
2836
2837 return 0;
2838 }
2839 static int amd_iommu_domain_init(struct iommu_domain *dom)
2840 {
2841 struct protection_domain *domain;
2842
2843 domain = protection_domain_alloc();
2844 if (!domain)
2845 goto out_free;
2846
2847 domain->mode = PAGE_MODE_3_LEVEL;
2848 domain->pt_root = (void *)get_zeroed_page(GFP_KERNEL);
2849 if (!domain->pt_root)
2850 goto out_free;
2851
2852 dom->priv = domain;
2853
2854 return 0;
2855
2856 out_free:
2857 protection_domain_free(domain);
2858
2859 return -ENOMEM;
2860 }
2861
2862 static void amd_iommu_domain_destroy(struct iommu_domain *dom)
2863 {
2864 struct protection_domain *domain = dom->priv;
2865
2866 if (!domain)
2867 return;
2868
2869 if (domain->dev_cnt > 0)
2870 cleanup_domain(domain);
2871
2872 BUG_ON(domain->dev_cnt != 0);
2873
2874 if (domain->mode != PAGE_MODE_NONE)
2875 free_pagetable(domain);
2876
2877 if (domain->flags & PD_IOMMUV2_MASK)
2878 free_gcr3_table(domain);
2879
2880 protection_domain_free(domain);
2881
2882 dom->priv = NULL;
2883 }
2884
2885 static void amd_iommu_detach_device(struct iommu_domain *dom,
2886 struct device *dev)
2887 {
2888 struct iommu_dev_data *dev_data = dev->archdata.iommu;
2889 struct amd_iommu *iommu;
2890 u16 devid;
2891
2892 if (!check_device(dev))
2893 return;
2894
2895 devid = get_device_id(dev);
2896
2897 if (dev_data->domain != NULL)
2898 detach_device(dev);
2899
2900 iommu = amd_iommu_rlookup_table[devid];
2901 if (!iommu)
2902 return;
2903
2904 iommu_completion_wait(iommu);
2905 }
2906
2907 static int amd_iommu_attach_device(struct iommu_domain *dom,
2908 struct device *dev)
2909 {
2910 struct protection_domain *domain = dom->priv;
2911 struct iommu_dev_data *dev_data;
2912 struct amd_iommu *iommu;
2913 int ret;
2914
2915 if (!check_device(dev))
2916 return -EINVAL;
2917
2918 dev_data = dev->archdata.iommu;
2919
2920 iommu = amd_iommu_rlookup_table[dev_data->devid];
2921 if (!iommu)
2922 return -EINVAL;
2923
2924 if (dev_data->domain)
2925 detach_device(dev);
2926
2927 ret = attach_device(dev, domain);
2928
2929 iommu_completion_wait(iommu);
2930
2931 return ret;
2932 }
2933
2934 static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
2935 phys_addr_t paddr, int gfp_order, int iommu_prot)
2936 {
2937 unsigned long page_size = 0x1000UL << gfp_order;
2938 struct protection_domain *domain = dom->priv;
2939 int prot = 0;
2940 int ret;
2941
2942 if (domain->mode == PAGE_MODE_NONE)
2943 return -EINVAL;
2944
2945 if (iommu_prot & IOMMU_READ)
2946 prot |= IOMMU_PROT_IR;
2947 if (iommu_prot & IOMMU_WRITE)
2948 prot |= IOMMU_PROT_IW;
2949
2950 mutex_lock(&domain->api_lock);
2951 ret = iommu_map_page(domain, iova, paddr, prot, page_size);
2952 mutex_unlock(&domain->api_lock);
2953
2954 return ret;
2955 }
2956
2957 static int amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova,
2958 int gfp_order)
2959 {
2960 struct protection_domain *domain = dom->priv;
2961 unsigned long page_size, unmap_size;
2962
2963 if (domain->mode == PAGE_MODE_NONE)
2964 return -EINVAL;
2965
2966 page_size = 0x1000UL << gfp_order;
2967
2968 mutex_lock(&domain->api_lock);
2969 unmap_size = iommu_unmap_page(domain, iova, page_size);
2970 mutex_unlock(&domain->api_lock);
2971
2972 domain_flush_tlb_pde(domain);
2973
2974 return get_order(unmap_size);
2975 }
2976
2977 static phys_addr_t amd_iommu_iova_to_phys(struct iommu_domain *dom,
2978 unsigned long iova)
2979 {
2980 struct protection_domain *domain = dom->priv;
2981 unsigned long offset_mask;
2982 phys_addr_t paddr;
2983 u64 *pte, __pte;
2984
2985 if (domain->mode == PAGE_MODE_NONE)
2986 return iova;
2987
2988 pte = fetch_pte(domain, iova);
2989
2990 if (!pte || !IOMMU_PTE_PRESENT(*pte))
2991 return 0;
2992
2993 if (PM_PTE_LEVEL(*pte) == 0)
2994 offset_mask = PAGE_SIZE - 1;
2995 else
2996 offset_mask = PTE_PAGE_SIZE(*pte) - 1;
2997
2998 __pte = *pte & PM_ADDR_MASK;
2999 paddr = (__pte & ~offset_mask) | (iova & offset_mask);
3000
3001 return paddr;
3002 }
3003
3004 static int amd_iommu_domain_has_cap(struct iommu_domain *domain,
3005 unsigned long cap)
3006 {
3007 switch (cap) {
3008 case IOMMU_CAP_CACHE_COHERENCY:
3009 return 1;
3010 }
3011
3012 return 0;
3013 }
3014
3015 static struct iommu_ops amd_iommu_ops = {
3016 .domain_init = amd_iommu_domain_init,
3017 .domain_destroy = amd_iommu_domain_destroy,
3018 .attach_dev = amd_iommu_attach_device,
3019 .detach_dev = amd_iommu_detach_device,
3020 .map = amd_iommu_map,
3021 .unmap = amd_iommu_unmap,
3022 .iova_to_phys = amd_iommu_iova_to_phys,
3023 .domain_has_cap = amd_iommu_domain_has_cap,
3024 };
3025
3026 /*****************************************************************************
3027 *
3028 * The next functions do a basic initialization of IOMMU for pass through
3029 * mode
3030 *
3031 * In passthrough mode the IOMMU is initialized and enabled but not used for
3032 * DMA-API translation.
3033 *
3034 *****************************************************************************/
3035
3036 int __init amd_iommu_init_passthrough(void)
3037 {
3038 struct iommu_dev_data *dev_data;
3039 struct pci_dev *dev = NULL;
3040 struct amd_iommu *iommu;
3041 u16 devid;
3042 int ret;
3043
3044 ret = alloc_passthrough_domain();
3045 if (ret)
3046 return ret;
3047
3048 for_each_pci_dev(dev) {
3049 if (!check_device(&dev->dev))
3050 continue;
3051
3052 dev_data = get_dev_data(&dev->dev);
3053 dev_data->passthrough = true;
3054
3055 devid = get_device_id(&dev->dev);
3056
3057 iommu = amd_iommu_rlookup_table[devid];
3058 if (!iommu)
3059 continue;
3060
3061 attach_device(&dev->dev, pt_domain);
3062 }
3063
3064 pr_info("AMD-Vi: Initialized for Passthrough Mode\n");
3065
3066 return 0;
3067 }
3068
3069 /* IOMMUv2 specific functions */
3070 int amd_iommu_register_ppr_notifier(struct notifier_block *nb)
3071 {
3072 return atomic_notifier_chain_register(&ppr_notifier, nb);
3073 }
3074 EXPORT_SYMBOL(amd_iommu_register_ppr_notifier);
3075
3076 int amd_iommu_unregister_ppr_notifier(struct notifier_block *nb)
3077 {
3078 return atomic_notifier_chain_unregister(&ppr_notifier, nb);
3079 }
3080 EXPORT_SYMBOL(amd_iommu_unregister_ppr_notifier);
3081
3082 void amd_iommu_domain_direct_map(struct iommu_domain *dom)
3083 {
3084 struct protection_domain *domain = dom->priv;
3085 unsigned long flags;
3086
3087 spin_lock_irqsave(&domain->lock, flags);
3088
3089 /* Update data structure */
3090 domain->mode = PAGE_MODE_NONE;
3091 domain->updated = true;
3092
3093 /* Make changes visible to IOMMUs */
3094 update_domain(domain);
3095
3096 /* Page-table is not visible to IOMMU anymore, so free it */
3097 free_pagetable(domain);
3098
3099 spin_unlock_irqrestore(&domain->lock, flags);
3100 }
3101 EXPORT_SYMBOL(amd_iommu_domain_direct_map);
3102
3103 int amd_iommu_domain_enable_v2(struct iommu_domain *dom, int pasids)
3104 {
3105 struct protection_domain *domain = dom->priv;
3106 unsigned long flags;
3107 int levels, ret;
3108
3109 if (pasids <= 0 || pasids > (PASID_MASK + 1))
3110 return -EINVAL;
3111
3112 /* Number of GCR3 table levels required */
3113 for (levels = 0; (pasids - 1) & ~0x1ff; pasids >>= 9)
3114 levels += 1;
3115
3116 if (levels > amd_iommu_max_glx_val)
3117 return -EINVAL;
3118
3119 spin_lock_irqsave(&domain->lock, flags);
3120
3121 /*
3122 * Save us all sanity checks whether devices already in the
3123 * domain support IOMMUv2. Just force that the domain has no
3124 * devices attached when it is switched into IOMMUv2 mode.
3125 */
3126 ret = -EBUSY;
3127 if (domain->dev_cnt > 0 || domain->flags & PD_IOMMUV2_MASK)
3128 goto out;
3129
3130 ret = -ENOMEM;
3131 domain->gcr3_tbl = (void *)get_zeroed_page(GFP_ATOMIC);
3132 if (domain->gcr3_tbl == NULL)
3133 goto out;
3134
3135 domain->glx = levels;
3136 domain->flags |= PD_IOMMUV2_MASK;
3137 domain->updated = true;
3138
3139 update_domain(domain);
3140
3141 ret = 0;
3142
3143 out:
3144 spin_unlock_irqrestore(&domain->lock, flags);
3145
3146 return ret;
3147 }
3148 EXPORT_SYMBOL(amd_iommu_domain_enable_v2);
Ubuntu Hirsute Linux kernel mirror