]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - drivers/iommu/amd_iommu_init.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge remote-tracking branch 'asoc/topic/ak4104' into asoc-next
[mirror_ubuntu-artful-kernel.git] / drivers / iommu / amd_iommu_init.c
1 /*
2 * Copyright (C) 2007-2010 Advanced Micro Devices, Inc.
3 * Author: Joerg Roedel <joerg.roedel@amd.com>
4 * Leo Duran <leo.duran@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 */
19
20 #include <linux/pci.h>
21 #include <linux/acpi.h>
22 #include <linux/list.h>
23 #include <linux/slab.h>
24 #include <linux/syscore_ops.h>
25 #include <linux/interrupt.h>
26 #include <linux/msi.h>
27 #include <linux/amd-iommu.h>
28 #include <linux/export.h>
29 #include <acpi/acpi.h>
30 #include <asm/pci-direct.h>
31 #include <asm/iommu.h>
32 #include <asm/gart.h>
33 #include <asm/x86_init.h>
34 #include <asm/iommu_table.h>
35 #include <asm/io_apic.h>
36 #include <asm/irq_remapping.h>
37
38 #include "amd_iommu_proto.h"
39 #include "amd_iommu_types.h"
40 #include "irq_remapping.h"
41
42 /*
43 * definitions for the ACPI scanning code
44 */
45 #define IVRS_HEADER_LENGTH 48
46
47 #define ACPI_IVHD_TYPE 0x10
48 #define ACPI_IVMD_TYPE_ALL 0x20
49 #define ACPI_IVMD_TYPE 0x21
50 #define ACPI_IVMD_TYPE_RANGE 0x22
51
52 #define IVHD_DEV_ALL 0x01
53 #define IVHD_DEV_SELECT 0x02
54 #define IVHD_DEV_SELECT_RANGE_START 0x03
55 #define IVHD_DEV_RANGE_END 0x04
56 #define IVHD_DEV_ALIAS 0x42
57 #define IVHD_DEV_ALIAS_RANGE 0x43
58 #define IVHD_DEV_EXT_SELECT 0x46
59 #define IVHD_DEV_EXT_SELECT_RANGE 0x47
60 #define IVHD_DEV_SPECIAL 0x48
61
62 #define IVHD_SPECIAL_IOAPIC 1
63 #define IVHD_SPECIAL_HPET 2
64
65 #define IVHD_FLAG_HT_TUN_EN_MASK 0x01
66 #define IVHD_FLAG_PASSPW_EN_MASK 0x02
67 #define IVHD_FLAG_RESPASSPW_EN_MASK 0x04
68 #define IVHD_FLAG_ISOC_EN_MASK 0x08
69
70 #define IVMD_FLAG_EXCL_RANGE 0x08
71 #define IVMD_FLAG_UNITY_MAP 0x01
72
73 #define ACPI_DEVFLAG_INITPASS 0x01
74 #define ACPI_DEVFLAG_EXTINT 0x02
75 #define ACPI_DEVFLAG_NMI 0x04
76 #define ACPI_DEVFLAG_SYSMGT1 0x10
77 #define ACPI_DEVFLAG_SYSMGT2 0x20
78 #define ACPI_DEVFLAG_LINT0 0x40
79 #define ACPI_DEVFLAG_LINT1 0x80
80 #define ACPI_DEVFLAG_ATSDIS 0x10000000
81
82 /*
83 * ACPI table definitions
84 *
85 * These data structures are laid over the table to parse the important values
86 * out of it.
87 */
88
89 /*
90 * structure describing one IOMMU in the ACPI table. Typically followed by one
91 * or more ivhd_entrys.
92 */
93 struct ivhd_header {
94 u8 type;
95 u8 flags;
96 u16 length;
97 u16 devid;
98 u16 cap_ptr;
99 u64 mmio_phys;
100 u16 pci_seg;
101 u16 info;
102 u32 reserved;
103 } __attribute__((packed));
104
105 /*
106 * A device entry describing which devices a specific IOMMU translates and
107 * which requestor ids they use.
108 */
109 struct ivhd_entry {
110 u8 type;
111 u16 devid;
112 u8 flags;
113 u32 ext;
114 } __attribute__((packed));
115
116 /*
117 * An AMD IOMMU memory definition structure. It defines things like exclusion
118 * ranges for devices and regions that should be unity mapped.
119 */
120 struct ivmd_header {
121 u8 type;
122 u8 flags;
123 u16 length;
124 u16 devid;
125 u16 aux;
126 u64 resv;
127 u64 range_start;
128 u64 range_length;
129 } __attribute__((packed));
130
131 bool amd_iommu_dump;
132 bool amd_iommu_irq_remap __read_mostly;
133
134 static bool amd_iommu_detected;
135 static bool __initdata amd_iommu_disabled;
136
137 u16 amd_iommu_last_bdf; /* largest PCI device id we have
138 to handle */
139 LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings
140 we find in ACPI */
141 u32 amd_iommu_unmap_flush; /* if true, flush on every unmap */
142
143 LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the
144 system */
145
146 /* Array to assign indices to IOMMUs*/
147 struct amd_iommu *amd_iommus[MAX_IOMMUS];
148 int amd_iommus_present;
149
150 /* IOMMUs have a non-present cache? */
151 bool amd_iommu_np_cache __read_mostly;
152 bool amd_iommu_iotlb_sup __read_mostly = true;
153
154 u32 amd_iommu_max_pasids __read_mostly = ~0;
155
156 bool amd_iommu_v2_present __read_mostly;
157
158 bool amd_iommu_force_isolation __read_mostly;
159
160 /*
161 * List of protection domains - used during resume
162 */
163 LIST_HEAD(amd_iommu_pd_list);
164 spinlock_t amd_iommu_pd_lock;
165
166 /*
167 * Pointer to the device table which is shared by all AMD IOMMUs
168 * it is indexed by the PCI device id or the HT unit id and contains
169 * information about the domain the device belongs to as well as the
170 * page table root pointer.
171 */
172 struct dev_table_entry *amd_iommu_dev_table;
173
174 /*
175 * The alias table is a driver specific data structure which contains the
176 * mappings of the PCI device ids to the actual requestor ids on the IOMMU.
177 * More than one device can share the same requestor id.
178 */
179 u16 *amd_iommu_alias_table;
180
181 /*
182 * The rlookup table is used to find the IOMMU which is responsible
183 * for a specific device. It is also indexed by the PCI device id.
184 */
185 struct amd_iommu **amd_iommu_rlookup_table;
186
187 /*
188 * This table is used to find the irq remapping table for a given device id
189 * quickly.
190 */
191 struct irq_remap_table **irq_lookup_table;
192
193 /*
194 * AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap
195 * to know which ones are already in use.
196 */
197 unsigned long *amd_iommu_pd_alloc_bitmap;
198
199 static u32 dev_table_size; /* size of the device table */
200 static u32 alias_table_size; /* size of the alias table */
201 static u32 rlookup_table_size; /* size if the rlookup table */
202
203 enum iommu_init_state {
204 IOMMU_START_STATE,
205 IOMMU_IVRS_DETECTED,
206 IOMMU_ACPI_FINISHED,
207 IOMMU_ENABLED,
208 IOMMU_PCI_INIT,
209 IOMMU_INTERRUPTS_EN,
210 IOMMU_DMA_OPS,
211 IOMMU_INITIALIZED,
212 IOMMU_NOT_FOUND,
213 IOMMU_INIT_ERROR,
214 };
215
216 static enum iommu_init_state init_state = IOMMU_START_STATE;
217
218 static int amd_iommu_enable_interrupts(void);
219 static int __init iommu_go_to_state(enum iommu_init_state state);
220
221 static inline void update_last_devid(u16 devid)
222 {
223 if (devid > amd_iommu_last_bdf)
224 amd_iommu_last_bdf = devid;
225 }
226
227 static inline unsigned long tbl_size(int entry_size)
228 {
229 unsigned shift = PAGE_SHIFT +
230 get_order(((int)amd_iommu_last_bdf + 1) * entry_size);
231
232 return 1UL << shift;
233 }
234
235 /* Access to l1 and l2 indexed register spaces */
236
237 static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address)
238 {
239 u32 val;
240
241 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
242 pci_read_config_dword(iommu->dev, 0xfc, &val);
243 return val;
244 }
245
246 static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val)
247 {
248 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31));
249 pci_write_config_dword(iommu->dev, 0xfc, val);
250 pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16));
251 }
252
253 static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address)
254 {
255 u32 val;
256
257 pci_write_config_dword(iommu->dev, 0xf0, address);
258 pci_read_config_dword(iommu->dev, 0xf4, &val);
259 return val;
260 }
261
262 static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
263 {
264 pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8));
265 pci_write_config_dword(iommu->dev, 0xf4, val);
266 }
267
268 /****************************************************************************
269 *
270 * AMD IOMMU MMIO register space handling functions
271 *
272 * These functions are used to program the IOMMU device registers in
273 * MMIO space required for that driver.
274 *
275 ****************************************************************************/
276
277 /*
278 * This function set the exclusion range in the IOMMU. DMA accesses to the
279 * exclusion range are passed through untranslated
280 */
281 static void iommu_set_exclusion_range(struct amd_iommu *iommu)
282 {
283 u64 start = iommu->exclusion_start & PAGE_MASK;
284 u64 limit = (start + iommu->exclusion_length) & PAGE_MASK;
285 u64 entry;
286
287 if (!iommu->exclusion_start)
288 return;
289
290 entry = start | MMIO_EXCL_ENABLE_MASK;
291 memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
292 &entry, sizeof(entry));
293
294 entry = limit;
295 memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
296 &entry, sizeof(entry));
297 }
298
299 /* Programs the physical address of the device table into the IOMMU hardware */
300 static void iommu_set_device_table(struct amd_iommu *iommu)
301 {
302 u64 entry;
303
304 BUG_ON(iommu->mmio_base == NULL);
305
306 entry = virt_to_phys(amd_iommu_dev_table);
307 entry |= (dev_table_size >> 12) - 1;
308 memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET,
309 &entry, sizeof(entry));
310 }
311
312 /* Generic functions to enable/disable certain features of the IOMMU. */
313 static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
314 {
315 u32 ctrl;
316
317 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
318 ctrl |= (1 << bit);
319 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
320 }
321
322 static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit)
323 {
324 u32 ctrl;
325
326 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
327 ctrl &= ~(1 << bit);
328 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
329 }
330
331 static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout)
332 {
333 u32 ctrl;
334
335 ctrl = readl(iommu->mmio_base + MMIO_CONTROL_OFFSET);
336 ctrl &= ~CTRL_INV_TO_MASK;
337 ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK;
338 writel(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET);
339 }
340
341 /* Function to enable the hardware */
342 static void iommu_enable(struct amd_iommu *iommu)
343 {
344 iommu_feature_enable(iommu, CONTROL_IOMMU_EN);
345 }
346
347 static void iommu_disable(struct amd_iommu *iommu)
348 {
349 /* Disable command buffer */
350 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
351
352 /* Disable event logging and event interrupts */
353 iommu_feature_disable(iommu, CONTROL_EVT_INT_EN);
354 iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN);
355
356 /* Disable IOMMU hardware itself */
357 iommu_feature_disable(iommu, CONTROL_IOMMU_EN);
358 }
359
360 /*
361 * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in
362 * the system has one.
363 */
364 static u8 __iomem * __init iommu_map_mmio_space(u64 address)
365 {
366 if (!request_mem_region(address, MMIO_REGION_LENGTH, "amd_iommu")) {
367 pr_err("AMD-Vi: Can not reserve memory region %llx for mmio\n",
368 address);
369 pr_err("AMD-Vi: This is a BIOS bug. Please contact your hardware vendor\n");
370 return NULL;
371 }
372
373 return (u8 __iomem *)ioremap_nocache(address, MMIO_REGION_LENGTH);
374 }
375
376 static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu)
377 {
378 if (iommu->mmio_base)
379 iounmap(iommu->mmio_base);
380 release_mem_region(iommu->mmio_phys, MMIO_REGION_LENGTH);
381 }
382
383 /****************************************************************************
384 *
385 * The functions below belong to the first pass of AMD IOMMU ACPI table
386 * parsing. In this pass we try to find out the highest device id this
387 * code has to handle. Upon this information the size of the shared data
388 * structures is determined later.
389 *
390 ****************************************************************************/
391
392 /*
393 * This function calculates the length of a given IVHD entry
394 */
395 static inline int ivhd_entry_length(u8 *ivhd)
396 {
397 return 0x04 << (*ivhd >> 6);
398 }
399
400 /*
401 * This function reads the last device id the IOMMU has to handle from the PCI
402 * capability header for this IOMMU
403 */
404 static int __init find_last_devid_on_pci(int bus, int dev, int fn, int cap_ptr)
405 {
406 u32 cap;
407
408 cap = read_pci_config(bus, dev, fn, cap_ptr+MMIO_RANGE_OFFSET);
409 update_last_devid(calc_devid(MMIO_GET_BUS(cap), MMIO_GET_LD(cap)));
410
411 return 0;
412 }
413
414 /*
415 * After reading the highest device id from the IOMMU PCI capability header
416 * this function looks if there is a higher device id defined in the ACPI table
417 */
418 static int __init find_last_devid_from_ivhd(struct ivhd_header *h)
419 {
420 u8 *p = (void *)h, *end = (void *)h;
421 struct ivhd_entry *dev;
422
423 p += sizeof(*h);
424 end += h->length;
425
426 find_last_devid_on_pci(PCI_BUS(h->devid),
427 PCI_SLOT(h->devid),
428 PCI_FUNC(h->devid),
429 h->cap_ptr);
430
431 while (p < end) {
432 dev = (struct ivhd_entry *)p;
433 switch (dev->type) {
434 case IVHD_DEV_SELECT:
435 case IVHD_DEV_RANGE_END:
436 case IVHD_DEV_ALIAS:
437 case IVHD_DEV_EXT_SELECT:
438 /* all the above subfield types refer to device ids */
439 update_last_devid(dev->devid);
440 break;
441 default:
442 break;
443 }
444 p += ivhd_entry_length(p);
445 }
446
447 WARN_ON(p != end);
448
449 return 0;
450 }
451
452 /*
453 * Iterate over all IVHD entries in the ACPI table and find the highest device
454 * id which we need to handle. This is the first of three functions which parse
455 * the ACPI table. So we check the checksum here.
456 */
457 static int __init find_last_devid_acpi(struct acpi_table_header *table)
458 {
459 int i;
460 u8 checksum = 0, *p = (u8 *)table, *end = (u8 *)table;
461 struct ivhd_header *h;
462
463 /*
464 * Validate checksum here so we don't need to do it when
465 * we actually parse the table
466 */
467 for (i = 0; i < table->length; ++i)
468 checksum += p[i];
469 if (checksum != 0)
470 /* ACPI table corrupt */
471 return -ENODEV;
472
473 p += IVRS_HEADER_LENGTH;
474
475 end += table->length;
476 while (p < end) {
477 h = (struct ivhd_header *)p;
478 switch (h->type) {
479 case ACPI_IVHD_TYPE:
480 find_last_devid_from_ivhd(h);
481 break;
482 default:
483 break;
484 }
485 p += h->length;
486 }
487 WARN_ON(p != end);
488
489 return 0;
490 }
491
492 /****************************************************************************
493 *
494 * The following functions belong to the code path which parses the ACPI table
495 * the second time. In this ACPI parsing iteration we allocate IOMMU specific
496 * data structures, initialize the device/alias/rlookup table and also
497 * basically initialize the hardware.
498 *
499 ****************************************************************************/
500
501 /*
502 * Allocates the command buffer. This buffer is per AMD IOMMU. We can
503 * write commands to that buffer later and the IOMMU will execute them
504 * asynchronously
505 */
506 static u8 * __init alloc_command_buffer(struct amd_iommu *iommu)
507 {
508 u8 *cmd_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
509 get_order(CMD_BUFFER_SIZE));
510
511 if (cmd_buf == NULL)
512 return NULL;
513
514 iommu->cmd_buf_size = CMD_BUFFER_SIZE | CMD_BUFFER_UNINITIALIZED;
515
516 return cmd_buf;
517 }
518
519 /*
520 * This function resets the command buffer if the IOMMU stopped fetching
521 * commands from it.
522 */
523 void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu)
524 {
525 iommu_feature_disable(iommu, CONTROL_CMDBUF_EN);
526
527 writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET);
528 writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET);
529
530 iommu_feature_enable(iommu, CONTROL_CMDBUF_EN);
531 }
532
533 /*
534 * This function writes the command buffer address to the hardware and
535 * enables it.
536 */
537 static void iommu_enable_command_buffer(struct amd_iommu *iommu)
538 {
539 u64 entry;
540
541 BUG_ON(iommu->cmd_buf == NULL);
542
543 entry = (u64)virt_to_phys(iommu->cmd_buf);
544 entry |= MMIO_CMD_SIZE_512;
545
546 memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET,
547 &entry, sizeof(entry));
548
549 amd_iommu_reset_cmd_buffer(iommu);
550 iommu->cmd_buf_size &= ~(CMD_BUFFER_UNINITIALIZED);
551 }
552
553 static void __init free_command_buffer(struct amd_iommu *iommu)
554 {
555 free_pages((unsigned long)iommu->cmd_buf,
556 get_order(iommu->cmd_buf_size & ~(CMD_BUFFER_UNINITIALIZED)));
557 }
558
559 /* allocates the memory where the IOMMU will log its events to */
560 static u8 * __init alloc_event_buffer(struct amd_iommu *iommu)
561 {
562 iommu->evt_buf = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
563 get_order(EVT_BUFFER_SIZE));
564
565 if (iommu->evt_buf == NULL)
566 return NULL;
567
568 iommu->evt_buf_size = EVT_BUFFER_SIZE;
569
570 return iommu->evt_buf;
571 }
572
573 static void iommu_enable_event_buffer(struct amd_iommu *iommu)
574 {
575 u64 entry;
576
577 BUG_ON(iommu->evt_buf == NULL);
578
579 entry = (u64)virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK;
580
581 memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET,
582 &entry, sizeof(entry));
583
584 /* set head and tail to zero manually */
585 writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
586 writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET);
587
588 iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN);
589 }
590
591 static void __init free_event_buffer(struct amd_iommu *iommu)
592 {
593 free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
594 }
595
596 /* allocates the memory where the IOMMU will log its events to */
597 static u8 * __init alloc_ppr_log(struct amd_iommu *iommu)
598 {
599 iommu->ppr_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
600 get_order(PPR_LOG_SIZE));
601
602 if (iommu->ppr_log == NULL)
603 return NULL;
604
605 return iommu->ppr_log;
606 }
607
608 static void iommu_enable_ppr_log(struct amd_iommu *iommu)
609 {
610 u64 entry;
611
612 if (iommu->ppr_log == NULL)
613 return;
614
615 entry = (u64)virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;
616
617 memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
618 &entry, sizeof(entry));
619
620 /* set head and tail to zero manually */
621 writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
622 writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
623
624 iommu_feature_enable(iommu, CONTROL_PPFLOG_EN);
625 iommu_feature_enable(iommu, CONTROL_PPR_EN);
626 }
627
628 static void __init free_ppr_log(struct amd_iommu *iommu)
629 {
630 if (iommu->ppr_log == NULL)
631 return;
632
633 free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));
634 }
635
636 static void iommu_enable_gt(struct amd_iommu *iommu)
637 {
638 if (!iommu_feature(iommu, FEATURE_GT))
639 return;
640
641 iommu_feature_enable(iommu, CONTROL_GT_EN);
642 }
643
644 /* sets a specific bit in the device table entry. */
645 static void set_dev_entry_bit(u16 devid, u8 bit)
646 {
647 int i = (bit >> 6) & 0x03;
648 int _bit = bit & 0x3f;
649
650 amd_iommu_dev_table[devid].data[i] |= (1UL << _bit);
651 }
652
653 static int get_dev_entry_bit(u16 devid, u8 bit)
654 {
655 int i = (bit >> 6) & 0x03;
656 int _bit = bit & 0x3f;
657
658 return (amd_iommu_dev_table[devid].data[i] & (1UL << _bit)) >> _bit;
659 }
660
661
662 void amd_iommu_apply_erratum_63(u16 devid)
663 {
664 int sysmgt;
665
666 sysmgt = get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1) |
667 (get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2) << 1);
668
669 if (sysmgt == 0x01)
670 set_dev_entry_bit(devid, DEV_ENTRY_IW);
671 }
672
673 /* Writes the specific IOMMU for a device into the rlookup table */
674 static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid)
675 {
676 amd_iommu_rlookup_table[devid] = iommu;
677 }
678
679 /*
680 * This function takes the device specific flags read from the ACPI
681 * table and sets up the device table entry with that information
682 */
683 static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu,
684 u16 devid, u32 flags, u32 ext_flags)
685 {
686 if (flags & ACPI_DEVFLAG_INITPASS)
687 set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS);
688 if (flags & ACPI_DEVFLAG_EXTINT)
689 set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS);
690 if (flags & ACPI_DEVFLAG_NMI)
691 set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS);
692 if (flags & ACPI_DEVFLAG_SYSMGT1)
693 set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1);
694 if (flags & ACPI_DEVFLAG_SYSMGT2)
695 set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2);
696 if (flags & ACPI_DEVFLAG_LINT0)
697 set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS);
698 if (flags & ACPI_DEVFLAG_LINT1)
699 set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS);
700
701 amd_iommu_apply_erratum_63(devid);
702
703 set_iommu_for_device(iommu, devid);
704 }
705
706 static int add_special_device(u8 type, u8 id, u16 devid)
707 {
708 struct devid_map *entry;
709 struct list_head *list;
710
711 if (type != IVHD_SPECIAL_IOAPIC && type != IVHD_SPECIAL_HPET)
712 return -EINVAL;
713
714 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
715 if (!entry)
716 return -ENOMEM;
717
718 entry->id = id;
719 entry->devid = devid;
720
721 if (type == IVHD_SPECIAL_IOAPIC)
722 list = &ioapic_map;
723 else
724 list = &hpet_map;
725
726 list_add_tail(&entry->list, list);
727
728 return 0;
729 }
730
731 /*
732 * Reads the device exclusion range from ACPI and initializes the IOMMU with
733 * it
734 */
735 static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m)
736 {
737 struct amd_iommu *iommu = amd_iommu_rlookup_table[devid];
738
739 if (!(m->flags & IVMD_FLAG_EXCL_RANGE))
740 return;
741
742 if (iommu) {
743 /*
744 * We only can configure exclusion ranges per IOMMU, not
745 * per device. But we can enable the exclusion range per
746 * device. This is done here
747 */
748 set_dev_entry_bit(m->devid, DEV_ENTRY_EX);
749 iommu->exclusion_start = m->range_start;
750 iommu->exclusion_length = m->range_length;
751 }
752 }
753
754 /*
755 * Takes a pointer to an AMD IOMMU entry in the ACPI table and
756 * initializes the hardware and our data structures with it.
757 */
758 static int __init init_iommu_from_acpi(struct amd_iommu *iommu,
759 struct ivhd_header *h)
760 {
761 u8 *p = (u8 *)h;
762 u8 *end = p, flags = 0;
763 u16 devid = 0, devid_start = 0, devid_to = 0;
764 u32 dev_i, ext_flags = 0;
765 bool alias = false;
766 struct ivhd_entry *e;
767
768 /*
769 * First save the recommended feature enable bits from ACPI
770 */
771 iommu->acpi_flags = h->flags;
772
773 /*
774 * Done. Now parse the device entries
775 */
776 p += sizeof(struct ivhd_header);
777 end += h->length;
778
779
780 while (p < end) {
781 e = (struct ivhd_entry *)p;
782 switch (e->type) {
783 case IVHD_DEV_ALL:
784
785 DUMP_printk(" DEV_ALL\t\t\t first devid: %02x:%02x.%x"
786 " last device %02x:%02x.%x flags: %02x\n",
787 PCI_BUS(iommu->first_device),
788 PCI_SLOT(iommu->first_device),
789 PCI_FUNC(iommu->first_device),
790 PCI_BUS(iommu->last_device),
791 PCI_SLOT(iommu->last_device),
792 PCI_FUNC(iommu->last_device),
793 e->flags);
794
795 for (dev_i = iommu->first_device;
796 dev_i <= iommu->last_device; ++dev_i)
797 set_dev_entry_from_acpi(iommu, dev_i,
798 e->flags, 0);
799 break;
800 case IVHD_DEV_SELECT:
801
802 DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x "
803 "flags: %02x\n",
804 PCI_BUS(e->devid),
805 PCI_SLOT(e->devid),
806 PCI_FUNC(e->devid),
807 e->flags);
808
809 devid = e->devid;
810 set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
811 break;
812 case IVHD_DEV_SELECT_RANGE_START:
813
814 DUMP_printk(" DEV_SELECT_RANGE_START\t "
815 "devid: %02x:%02x.%x flags: %02x\n",
816 PCI_BUS(e->devid),
817 PCI_SLOT(e->devid),
818 PCI_FUNC(e->devid),
819 e->flags);
820
821 devid_start = e->devid;
822 flags = e->flags;
823 ext_flags = 0;
824 alias = false;
825 break;
826 case IVHD_DEV_ALIAS:
827
828 DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x "
829 "flags: %02x devid_to: %02x:%02x.%x\n",
830 PCI_BUS(e->devid),
831 PCI_SLOT(e->devid),
832 PCI_FUNC(e->devid),
833 e->flags,
834 PCI_BUS(e->ext >> 8),
835 PCI_SLOT(e->ext >> 8),
836 PCI_FUNC(e->ext >> 8));
837
838 devid = e->devid;
839 devid_to = e->ext >> 8;
840 set_dev_entry_from_acpi(iommu, devid , e->flags, 0);
841 set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0);
842 amd_iommu_alias_table[devid] = devid_to;
843 break;
844 case IVHD_DEV_ALIAS_RANGE:
845
846 DUMP_printk(" DEV_ALIAS_RANGE\t\t "
847 "devid: %02x:%02x.%x flags: %02x "
848 "devid_to: %02x:%02x.%x\n",
849 PCI_BUS(e->devid),
850 PCI_SLOT(e->devid),
851 PCI_FUNC(e->devid),
852 e->flags,
853 PCI_BUS(e->ext >> 8),
854 PCI_SLOT(e->ext >> 8),
855 PCI_FUNC(e->ext >> 8));
856
857 devid_start = e->devid;
858 flags = e->flags;
859 devid_to = e->ext >> 8;
860 ext_flags = 0;
861 alias = true;
862 break;
863 case IVHD_DEV_EXT_SELECT:
864
865 DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x "
866 "flags: %02x ext: %08x\n",
867 PCI_BUS(e->devid),
868 PCI_SLOT(e->devid),
869 PCI_FUNC(e->devid),
870 e->flags, e->ext);
871
872 devid = e->devid;
873 set_dev_entry_from_acpi(iommu, devid, e->flags,
874 e->ext);
875 break;
876 case IVHD_DEV_EXT_SELECT_RANGE:
877
878 DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: "
879 "%02x:%02x.%x flags: %02x ext: %08x\n",
880 PCI_BUS(e->devid),
881 PCI_SLOT(e->devid),
882 PCI_FUNC(e->devid),
883 e->flags, e->ext);
884
885 devid_start = e->devid;
886 flags = e->flags;
887 ext_flags = e->ext;
888 alias = false;
889 break;
890 case IVHD_DEV_RANGE_END:
891
892 DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n",
893 PCI_BUS(e->devid),
894 PCI_SLOT(e->devid),
895 PCI_FUNC(e->devid));
896
897 devid = e->devid;
898 for (dev_i = devid_start; dev_i <= devid; ++dev_i) {
899 if (alias) {
900 amd_iommu_alias_table[dev_i] = devid_to;
901 set_dev_entry_from_acpi(iommu,
902 devid_to, flags, ext_flags);
903 }
904 set_dev_entry_from_acpi(iommu, dev_i,
905 flags, ext_flags);
906 }
907 break;
908 case IVHD_DEV_SPECIAL: {
909 u8 handle, type;
910 const char *var;
911 u16 devid;
912 int ret;
913
914 handle = e->ext & 0xff;
915 devid = (e->ext >> 8) & 0xffff;
916 type = (e->ext >> 24) & 0xff;
917
918 if (type == IVHD_SPECIAL_IOAPIC)
919 var = "IOAPIC";
920 else if (type == IVHD_SPECIAL_HPET)
921 var = "HPET";
922 else
923 var = "UNKNOWN";
924
925 DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %02x:%02x.%x\n",
926 var, (int)handle,
927 PCI_BUS(devid),
928 PCI_SLOT(devid),
929 PCI_FUNC(devid));
930
931 set_dev_entry_from_acpi(iommu, devid, e->flags, 0);
932 ret = add_special_device(type, handle, devid);
933 if (ret)
934 return ret;
935 break;
936 }
937 default:
938 break;
939 }
940
941 p += ivhd_entry_length(p);
942 }
943
944 return 0;
945 }
946
947 /* Initializes the device->iommu mapping for the driver */
948 static int __init init_iommu_devices(struct amd_iommu *iommu)
949 {
950 u32 i;
951
952 for (i = iommu->first_device; i <= iommu->last_device; ++i)
953 set_iommu_for_device(iommu, i);
954
955 return 0;
956 }
957
958 static void __init free_iommu_one(struct amd_iommu *iommu)
959 {
960 free_command_buffer(iommu);
961 free_event_buffer(iommu);
962 free_ppr_log(iommu);
963 iommu_unmap_mmio_space(iommu);
964 }
965
966 static void __init free_iommu_all(void)
967 {
968 struct amd_iommu *iommu, *next;
969
970 for_each_iommu_safe(iommu, next) {
971 list_del(&iommu->list);
972 free_iommu_one(iommu);
973 kfree(iommu);
974 }
975 }
976
977 /*
978 * This function clues the initialization function for one IOMMU
979 * together and also allocates the command buffer and programs the
980 * hardware. It does NOT enable the IOMMU. This is done afterwards.
981 */
982 static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)
983 {
984 int ret;
985
986 spin_lock_init(&iommu->lock);
987
988 /* Add IOMMU to internal data structures */
989 list_add_tail(&iommu->list, &amd_iommu_list);
990 iommu->index = amd_iommus_present++;
991
992 if (unlikely(iommu->index >= MAX_IOMMUS)) {
993 WARN(1, "AMD-Vi: System has more IOMMUs than supported by this driver\n");
994 return -ENOSYS;
995 }
996
997 /* Index is fine - add IOMMU to the array */
998 amd_iommus[iommu->index] = iommu;
999
1000 /*
1001 * Copy data from ACPI table entry to the iommu struct
1002 */
1003 iommu->devid = h->devid;
1004 iommu->cap_ptr = h->cap_ptr;
1005 iommu->pci_seg = h->pci_seg;
1006 iommu->mmio_phys = h->mmio_phys;
1007 iommu->mmio_base = iommu_map_mmio_space(h->mmio_phys);
1008 if (!iommu->mmio_base)
1009 return -ENOMEM;
1010
1011 iommu->cmd_buf = alloc_command_buffer(iommu);
1012 if (!iommu->cmd_buf)
1013 return -ENOMEM;
1014
1015 iommu->evt_buf = alloc_event_buffer(iommu);
1016 if (!iommu->evt_buf)
1017 return -ENOMEM;
1018
1019 iommu->int_enabled = false;
1020
1021 ret = init_iommu_from_acpi(iommu, h);
1022 if (ret)
1023 return ret;
1024
1025 /*
1026 * Make sure IOMMU is not considered to translate itself. The IVRS
1027 * table tells us so, but this is a lie!
1028 */
1029 amd_iommu_rlookup_table[iommu->devid] = NULL;
1030
1031 init_iommu_devices(iommu);
1032
1033 return 0;
1034 }
1035
1036 /*
1037 * Iterates over all IOMMU entries in the ACPI table, allocates the
1038 * IOMMU structure and initializes it with init_iommu_one()
1039 */
1040 static int __init init_iommu_all(struct acpi_table_header *table)
1041 {
1042 u8 *p = (u8 *)table, *end = (u8 *)table;
1043 struct ivhd_header *h;
1044 struct amd_iommu *iommu;
1045 int ret;
1046
1047 end += table->length;
1048 p += IVRS_HEADER_LENGTH;
1049
1050 while (p < end) {
1051 h = (struct ivhd_header *)p;
1052 switch (*p) {
1053 case ACPI_IVHD_TYPE:
1054
1055 DUMP_printk("device: %02x:%02x.%01x cap: %04x "
1056 "seg: %d flags: %01x info %04x\n",
1057 PCI_BUS(h->devid), PCI_SLOT(h->devid),
1058 PCI_FUNC(h->devid), h->cap_ptr,
1059 h->pci_seg, h->flags, h->info);
1060 DUMP_printk(" mmio-addr: %016llx\n",
1061 h->mmio_phys);
1062
1063 iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL);
1064 if (iommu == NULL)
1065 return -ENOMEM;
1066
1067 ret = init_iommu_one(iommu, h);
1068 if (ret)
1069 return ret;
1070 break;
1071 default:
1072 break;
1073 }
1074 p += h->length;
1075
1076 }
1077 WARN_ON(p != end);
1078
1079 return 0;
1080 }
1081
1082 static int iommu_init_pci(struct amd_iommu *iommu)
1083 {
1084 int cap_ptr = iommu->cap_ptr;
1085 u32 range, misc, low, high;
1086
1087 iommu->dev = pci_get_bus_and_slot(PCI_BUS(iommu->devid),
1088 iommu->devid & 0xff);
1089 if (!iommu->dev)
1090 return -ENODEV;
1091
1092 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET,
1093 &iommu->cap);
1094 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET,
1095 &range);
1096 pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET,
1097 &misc);
1098
1099 iommu->first_device = calc_devid(MMIO_GET_BUS(range),
1100 MMIO_GET_FD(range));
1101 iommu->last_device = calc_devid(MMIO_GET_BUS(range),
1102 MMIO_GET_LD(range));
1103
1104 if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB)))
1105 amd_iommu_iotlb_sup = false;
1106
1107 /* read extended feature bits */
1108 low = readl(iommu->mmio_base + MMIO_EXT_FEATURES);
1109 high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4);
1110
1111 iommu->features = ((u64)high << 32) | low;
1112
1113 if (iommu_feature(iommu, FEATURE_GT)) {
1114 int glxval;
1115 u32 pasids;
1116 u64 shift;
1117
1118 shift = iommu->features & FEATURE_PASID_MASK;
1119 shift >>= FEATURE_PASID_SHIFT;
1120 pasids = (1 << shift);
1121
1122 amd_iommu_max_pasids = min(amd_iommu_max_pasids, pasids);
1123
1124 glxval = iommu->features & FEATURE_GLXVAL_MASK;
1125 glxval >>= FEATURE_GLXVAL_SHIFT;
1126
1127 if (amd_iommu_max_glx_val == -1)
1128 amd_iommu_max_glx_val = glxval;
1129 else
1130 amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);
1131 }
1132
1133 if (iommu_feature(iommu, FEATURE_GT) &&
1134 iommu_feature(iommu, FEATURE_PPR)) {
1135 iommu->is_iommu_v2 = true;
1136 amd_iommu_v2_present = true;
1137 }
1138
1139 if (iommu_feature(iommu, FEATURE_PPR)) {
1140 iommu->ppr_log = alloc_ppr_log(iommu);
1141 if (!iommu->ppr_log)
1142 return -ENOMEM;
1143 }
1144
1145 if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE))
1146 amd_iommu_np_cache = true;
1147
1148 if (is_rd890_iommu(iommu->dev)) {
1149 int i, j;
1150
1151 iommu->root_pdev = pci_get_bus_and_slot(iommu->dev->bus->number,
1152 PCI_DEVFN(0, 0));
1153
1154 /*
1155 * Some rd890 systems may not be fully reconfigured by the
1156 * BIOS, so it's necessary for us to store this information so
1157 * it can be reprogrammed on resume
1158 */
1159 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4,
1160 &iommu->stored_addr_lo);
1161 pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8,
1162 &iommu->stored_addr_hi);
1163
1164 /* Low bit locks writes to configuration space */
1165 iommu->stored_addr_lo &= ~1;
1166
1167 for (i = 0; i < 6; i++)
1168 for (j = 0; j < 0x12; j++)
1169 iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j);
1170
1171 for (i = 0; i < 0x83; i++)
1172 iommu->stored_l2[i] = iommu_read_l2(iommu, i);
1173 }
1174
1175 return pci_enable_device(iommu->dev);
1176 }
1177
1178 static void print_iommu_info(void)
1179 {
1180 static const char * const feat_str[] = {
1181 "PreF", "PPR", "X2APIC", "NX", "GT", "[5]",
1182 "IA", "GA", "HE", "PC"
1183 };
1184 struct amd_iommu *iommu;
1185
1186 for_each_iommu(iommu) {
1187 int i;
1188
1189 pr_info("AMD-Vi: Found IOMMU at %s cap 0x%hx\n",
1190 dev_name(&iommu->dev->dev), iommu->cap_ptr);
1191
1192 if (iommu->cap & (1 << IOMMU_CAP_EFR)) {
1193 pr_info("AMD-Vi: Extended features: ");
1194 for (i = 0; i < ARRAY_SIZE(feat_str); ++i) {
1195 if (iommu_feature(iommu, (1ULL << i)))
1196 pr_cont(" %s", feat_str[i]);
1197 }
1198 pr_cont("\n");
1199 }
1200 }
1201 if (irq_remapping_enabled)
1202 pr_info("AMD-Vi: Interrupt remapping enabled\n");
1203 }
1204
1205 static int __init amd_iommu_init_pci(void)
1206 {
1207 struct amd_iommu *iommu;
1208 int ret = 0;
1209
1210 for_each_iommu(iommu) {
1211 ret = iommu_init_pci(iommu);
1212 if (ret)
1213 break;
1214 }
1215
1216 ret = amd_iommu_init_devices();
1217
1218 print_iommu_info();
1219
1220 return ret;
1221 }
1222
1223 /****************************************************************************
1224 *
1225 * The following functions initialize the MSI interrupts for all IOMMUs
1226 * in the system. It's a bit challenging because there could be multiple
1227 * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per
1228 * pci_dev.
1229 *
1230 ****************************************************************************/
1231
1232 static int iommu_setup_msi(struct amd_iommu *iommu)
1233 {
1234 int r;
1235
1236 r = pci_enable_msi(iommu->dev);
1237 if (r)
1238 return r;
1239
1240 r = request_threaded_irq(iommu->dev->irq,
1241 amd_iommu_int_handler,
1242 amd_iommu_int_thread,
1243 0, "AMD-Vi",
1244 iommu->dev);
1245
1246 if (r) {
1247 pci_disable_msi(iommu->dev);
1248 return r;
1249 }
1250
1251 iommu->int_enabled = true;
1252
1253 return 0;
1254 }
1255
1256 static int iommu_init_msi(struct amd_iommu *iommu)
1257 {
1258 int ret;
1259
1260 if (iommu->int_enabled)
1261 goto enable_faults;
1262
1263 if (pci_find_capability(iommu->dev, PCI_CAP_ID_MSI))
1264 ret = iommu_setup_msi(iommu);
1265 else
1266 ret = -ENODEV;
1267
1268 if (ret)
1269 return ret;
1270
1271 enable_faults:
1272 iommu_feature_enable(iommu, CONTROL_EVT_INT_EN);
1273
1274 if (iommu->ppr_log != NULL)
1275 iommu_feature_enable(iommu, CONTROL_PPFINT_EN);
1276
1277 return 0;
1278 }
1279
1280 /****************************************************************************
1281 *
1282 * The next functions belong to the third pass of parsing the ACPI
1283 * table. In this last pass the memory mapping requirements are
1284 * gathered (like exclusion and unity mapping ranges).
1285 *
1286 ****************************************************************************/
1287
1288 static void __init free_unity_maps(void)
1289 {
1290 struct unity_map_entry *entry, *next;
1291
1292 list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) {
1293 list_del(&entry->list);
1294 kfree(entry);
1295 }
1296 }
1297
1298 /* called when we find an exclusion range definition in ACPI */
1299 static int __init init_exclusion_range(struct ivmd_header *m)
1300 {
1301 int i;
1302
1303 switch (m->type) {
1304 case ACPI_IVMD_TYPE:
1305 set_device_exclusion_range(m->devid, m);
1306 break;
1307 case ACPI_IVMD_TYPE_ALL:
1308 for (i = 0; i <= amd_iommu_last_bdf; ++i)
1309 set_device_exclusion_range(i, m);
1310 break;
1311 case ACPI_IVMD_TYPE_RANGE:
1312 for (i = m->devid; i <= m->aux; ++i)
1313 set_device_exclusion_range(i, m);
1314 break;
1315 default:
1316 break;
1317 }
1318
1319 return 0;
1320 }
1321
1322 /* called for unity map ACPI definition */
1323 static int __init init_unity_map_range(struct ivmd_header *m)
1324 {
1325 struct unity_map_entry *e = NULL;
1326 char *s;
1327
1328 e = kzalloc(sizeof(*e), GFP_KERNEL);
1329 if (e == NULL)
1330 return -ENOMEM;
1331
1332 switch (m->type) {
1333 default:
1334 kfree(e);
1335 return 0;
1336 case ACPI_IVMD_TYPE:
1337 s = "IVMD_TYPEi\t\t\t";
1338 e->devid_start = e->devid_end = m->devid;
1339 break;
1340 case ACPI_IVMD_TYPE_ALL:
1341 s = "IVMD_TYPE_ALL\t\t";
1342 e->devid_start = 0;
1343 e->devid_end = amd_iommu_last_bdf;
1344 break;
1345 case ACPI_IVMD_TYPE_RANGE:
1346 s = "IVMD_TYPE_RANGE\t\t";
1347 e->devid_start = m->devid;
1348 e->devid_end = m->aux;
1349 break;
1350 }
1351 e->address_start = PAGE_ALIGN(m->range_start);
1352 e->address_end = e->address_start + PAGE_ALIGN(m->range_length);
1353 e->prot = m->flags >> 1;
1354
1355 DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x"
1356 " range_start: %016llx range_end: %016llx flags: %x\n", s,
1357 PCI_BUS(e->devid_start), PCI_SLOT(e->devid_start),
1358 PCI_FUNC(e->devid_start), PCI_BUS(e->devid_end),
1359 PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end),
1360 e->address_start, e->address_end, m->flags);
1361
1362 list_add_tail(&e->list, &amd_iommu_unity_map);
1363
1364 return 0;
1365 }
1366
1367 /* iterates over all memory definitions we find in the ACPI table */
1368 static int __init init_memory_definitions(struct acpi_table_header *table)
1369 {
1370 u8 *p = (u8 *)table, *end = (u8 *)table;
1371 struct ivmd_header *m;
1372
1373 end += table->length;
1374 p += IVRS_HEADER_LENGTH;
1375
1376 while (p < end) {
1377 m = (struct ivmd_header *)p;
1378 if (m->flags & IVMD_FLAG_EXCL_RANGE)
1379 init_exclusion_range(m);
1380 else if (m->flags & IVMD_FLAG_UNITY_MAP)
1381 init_unity_map_range(m);
1382
1383 p += m->length;
1384 }
1385
1386 return 0;
1387 }
1388
1389 /*
1390 * Init the device table to not allow DMA access for devices and
1391 * suppress all page faults
1392 */
1393 static void init_device_table_dma(void)
1394 {
1395 u32 devid;
1396
1397 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
1398 set_dev_entry_bit(devid, DEV_ENTRY_VALID);
1399 set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION);
1400 }
1401 }
1402
1403 static void __init uninit_device_table_dma(void)
1404 {
1405 u32 devid;
1406
1407 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) {
1408 amd_iommu_dev_table[devid].data[0] = 0ULL;
1409 amd_iommu_dev_table[devid].data[1] = 0ULL;
1410 }
1411 }
1412
1413 static void init_device_table(void)
1414 {
1415 u32 devid;
1416
1417 if (!amd_iommu_irq_remap)
1418 return;
1419
1420 for (devid = 0; devid <= amd_iommu_last_bdf; ++devid)
1421 set_dev_entry_bit(devid, DEV_ENTRY_IRQ_TBL_EN);
1422 }
1423
1424 static void iommu_init_flags(struct amd_iommu *iommu)
1425 {
1426 iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ?
1427 iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) :
1428 iommu_feature_disable(iommu, CONTROL_HT_TUN_EN);
1429
1430 iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ?
1431 iommu_feature_enable(iommu, CONTROL_PASSPW_EN) :
1432 iommu_feature_disable(iommu, CONTROL_PASSPW_EN);
1433
1434 iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ?
1435 iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) :
1436 iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN);
1437
1438 iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ?
1439 iommu_feature_enable(iommu, CONTROL_ISOC_EN) :
1440 iommu_feature_disable(iommu, CONTROL_ISOC_EN);
1441
1442 /*
1443 * make IOMMU memory accesses cache coherent
1444 */
1445 iommu_feature_enable(iommu, CONTROL_COHERENT_EN);
1446
1447 /* Set IOTLB invalidation timeout to 1s */
1448 iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S);
1449 }
1450
1451 static void iommu_apply_resume_quirks(struct amd_iommu *iommu)
1452 {
1453 int i, j;
1454 u32 ioc_feature_control;
1455 struct pci_dev *pdev = iommu->root_pdev;
1456
1457 /* RD890 BIOSes may not have completely reconfigured the iommu */
1458 if (!is_rd890_iommu(iommu->dev) || !pdev)
1459 return;
1460
1461 /*
1462 * First, we need to ensure that the iommu is enabled. This is
1463 * controlled by a register in the northbridge
1464 */
1465
1466 /* Select Northbridge indirect register 0x75 and enable writing */
1467 pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7));
1468 pci_read_config_dword(pdev, 0x64, &ioc_feature_control);
1469
1470 /* Enable the iommu */
1471 if (!(ioc_feature_control & 0x1))
1472 pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1);
1473
1474 /* Restore the iommu BAR */
1475 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
1476 iommu->stored_addr_lo);
1477 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8,
1478 iommu->stored_addr_hi);
1479
1480 /* Restore the l1 indirect regs for each of the 6 l1s */
1481 for (i = 0; i < 6; i++)
1482 for (j = 0; j < 0x12; j++)
1483 iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]);
1484
1485 /* Restore the l2 indirect regs */
1486 for (i = 0; i < 0x83; i++)
1487 iommu_write_l2(iommu, i, iommu->stored_l2[i]);
1488
1489 /* Lock PCI setup registers */
1490 pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4,
1491 iommu->stored_addr_lo | 1);
1492 }
1493
1494 /*
1495 * This function finally enables all IOMMUs found in the system after
1496 * they have been initialized
1497 */
1498 static void early_enable_iommus(void)
1499 {
1500 struct amd_iommu *iommu;
1501
1502 for_each_iommu(iommu) {
1503 iommu_disable(iommu);
1504 iommu_init_flags(iommu);
1505 iommu_set_device_table(iommu);
1506 iommu_enable_command_buffer(iommu);
1507 iommu_enable_event_buffer(iommu);
1508 iommu_set_exclusion_range(iommu);
1509 iommu_enable(iommu);
1510 iommu_flush_all_caches(iommu);
1511 }
1512 }
1513
1514 static void enable_iommus_v2(void)
1515 {
1516 struct amd_iommu *iommu;
1517
1518 for_each_iommu(iommu) {
1519 iommu_enable_ppr_log(iommu);
1520 iommu_enable_gt(iommu);
1521 }
1522 }
1523
1524 static void enable_iommus(void)
1525 {
1526 early_enable_iommus();
1527
1528 enable_iommus_v2();
1529 }
1530
1531 static void disable_iommus(void)
1532 {
1533 struct amd_iommu *iommu;
1534
1535 for_each_iommu(iommu)
1536 iommu_disable(iommu);
1537 }
1538
1539 /*
1540 * Suspend/Resume support
1541 * disable suspend until real resume implemented
1542 */
1543
1544 static void amd_iommu_resume(void)
1545 {
1546 struct amd_iommu *iommu;
1547
1548 for_each_iommu(iommu)
1549 iommu_apply_resume_quirks(iommu);
1550
1551 /* re-load the hardware */
1552 enable_iommus();
1553
1554 amd_iommu_enable_interrupts();
1555 }
1556
1557 static int amd_iommu_suspend(void)
1558 {
1559 /* disable IOMMUs to go out of the way for BIOS */
1560 disable_iommus();
1561
1562 return 0;
1563 }
1564
1565 static struct syscore_ops amd_iommu_syscore_ops = {
1566 .suspend = amd_iommu_suspend,
1567 .resume = amd_iommu_resume,
1568 };
1569
1570 static void __init free_on_init_error(void)
1571 {
1572 free_pages((unsigned long)irq_lookup_table,
1573 get_order(rlookup_table_size));
1574
1575 if (amd_iommu_irq_cache) {
1576 kmem_cache_destroy(amd_iommu_irq_cache);
1577 amd_iommu_irq_cache = NULL;
1578
1579 }
1580
1581 free_pages((unsigned long)amd_iommu_rlookup_table,
1582 get_order(rlookup_table_size));
1583
1584 free_pages((unsigned long)amd_iommu_alias_table,
1585 get_order(alias_table_size));
1586
1587 free_pages((unsigned long)amd_iommu_dev_table,
1588 get_order(dev_table_size));
1589
1590 free_iommu_all();
1591
1592 #ifdef CONFIG_GART_IOMMU
1593 /*
1594 * We failed to initialize the AMD IOMMU - try fallback to GART
1595 * if possible.
1596 */
1597 gart_iommu_init();
1598
1599 #endif
1600 }
1601
1602 /* SB IOAPIC is always on this device in AMD systems */
1603 #define IOAPIC_SB_DEVID ((0x00 << 8) | PCI_DEVFN(0x14, 0))
1604
1605 static bool __init check_ioapic_information(void)
1606 {
1607 bool ret, has_sb_ioapic;
1608 int idx;
1609
1610 has_sb_ioapic = false;
1611 ret = false;
1612
1613 for (idx = 0; idx < nr_ioapics; idx++) {
1614 int devid, id = mpc_ioapic_id(idx);
1615
1616 devid = get_ioapic_devid(id);
1617 if (devid < 0) {
1618 pr_err(FW_BUG "AMD-Vi: IOAPIC[%d] not in IVRS table\n", id);
1619 ret = false;
1620 } else if (devid == IOAPIC_SB_DEVID) {
1621 has_sb_ioapic = true;
1622 ret = true;
1623 }
1624 }
1625
1626 if (!has_sb_ioapic) {
1627 /*
1628 * We expect the SB IOAPIC to be listed in the IVRS
1629 * table. The system timer is connected to the SB IOAPIC
1630 * and if we don't have it in the list the system will
1631 * panic at boot time. This situation usually happens
1632 * when the BIOS is buggy and provides us the wrong
1633 * device id for the IOAPIC in the system.
1634 */
1635 pr_err(FW_BUG "AMD-Vi: No southbridge IOAPIC found in IVRS table\n");
1636 }
1637
1638 if (!ret)
1639 pr_err("AMD-Vi: Disabling interrupt remapping due to BIOS Bug(s)\n");
1640
1641 return ret;
1642 }
1643
1644 static void __init free_dma_resources(void)
1645 {
1646 amd_iommu_uninit_devices();
1647
1648 free_pages((unsigned long)amd_iommu_pd_alloc_bitmap,
1649 get_order(MAX_DOMAIN_ID/8));
1650
1651 free_unity_maps();
1652 }
1653
1654 /*
1655 * This is the hardware init function for AMD IOMMU in the system.
1656 * This function is called either from amd_iommu_init or from the interrupt
1657 * remapping setup code.
1658 *
1659 * This function basically parses the ACPI table for AMD IOMMU (IVRS)
1660 * three times:
1661 *
1662 * 1 pass) Find the highest PCI device id the driver has to handle.
1663 * Upon this information the size of the data structures is
1664 * determined that needs to be allocated.
1665 *
1666 * 2 pass) Initialize the data structures just allocated with the
1667 * information in the ACPI table about available AMD IOMMUs
1668 * in the system. It also maps the PCI devices in the
1669 * system to specific IOMMUs
1670 *
1671 * 3 pass) After the basic data structures are allocated and
1672 * initialized we update them with information about memory
1673 * remapping requirements parsed out of the ACPI table in
1674 * this last pass.
1675 *
1676 * After everything is set up the IOMMUs are enabled and the necessary
1677 * hotplug and suspend notifiers are registered.
1678 */
1679 static int __init early_amd_iommu_init(void)
1680 {
1681 struct acpi_table_header *ivrs_base;
1682 acpi_size ivrs_size;
1683 acpi_status status;
1684 int i, ret = 0;
1685
1686 if (!amd_iommu_detected)
1687 return -ENODEV;
1688
1689 status = acpi_get_table_with_size("IVRS", 0, &ivrs_base, &ivrs_size);
1690 if (status == AE_NOT_FOUND)
1691 return -ENODEV;
1692 else if (ACPI_FAILURE(status)) {
1693 const char *err = acpi_format_exception(status);
1694 pr_err("AMD-Vi: IVRS table error: %s\n", err);
1695 return -EINVAL;
1696 }
1697
1698 /*
1699 * First parse ACPI tables to find the largest Bus/Dev/Func
1700 * we need to handle. Upon this information the shared data
1701 * structures for the IOMMUs in the system will be allocated
1702 */
1703 ret = find_last_devid_acpi(ivrs_base);
1704 if (ret)
1705 goto out;
1706
1707 dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE);
1708 alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE);
1709 rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE);
1710
1711 /* Device table - directly used by all IOMMUs */
1712 ret = -ENOMEM;
1713 amd_iommu_dev_table = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1714 get_order(dev_table_size));
1715 if (amd_iommu_dev_table == NULL)
1716 goto out;
1717
1718 /*
1719 * Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the
1720 * IOMMU see for that device
1721 */
1722 amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL,
1723 get_order(alias_table_size));
1724 if (amd_iommu_alias_table == NULL)
1725 goto out;
1726
1727 /* IOMMU rlookup table - find the IOMMU for a specific device */
1728 amd_iommu_rlookup_table = (void *)__get_free_pages(
1729 GFP_KERNEL | __GFP_ZERO,
1730 get_order(rlookup_table_size));
1731 if (amd_iommu_rlookup_table == NULL)
1732 goto out;
1733
1734 amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages(
1735 GFP_KERNEL | __GFP_ZERO,
1736 get_order(MAX_DOMAIN_ID/8));
1737 if (amd_iommu_pd_alloc_bitmap == NULL)
1738 goto out;
1739
1740 /*
1741 * let all alias entries point to itself
1742 */
1743 for (i = 0; i <= amd_iommu_last_bdf; ++i)
1744 amd_iommu_alias_table[i] = i;
1745
1746 /*
1747 * never allocate domain 0 because its used as the non-allocated and
1748 * error value placeholder
1749 */
1750 amd_iommu_pd_alloc_bitmap[0] = 1;
1751
1752 spin_lock_init(&amd_iommu_pd_lock);
1753
1754 /*
1755 * now the data structures are allocated and basically initialized
1756 * start the real acpi table scan
1757 */
1758 ret = init_iommu_all(ivrs_base);
1759 if (ret)
1760 goto out;
1761
1762 if (amd_iommu_irq_remap)
1763 amd_iommu_irq_remap = check_ioapic_information();
1764
1765 if (amd_iommu_irq_remap) {
1766 /*
1767 * Interrupt remapping enabled, create kmem_cache for the
1768 * remapping tables.
1769 */
1770 amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache",
1771 MAX_IRQS_PER_TABLE * sizeof(u32),
1772 IRQ_TABLE_ALIGNMENT,
1773 0, NULL);
1774 if (!amd_iommu_irq_cache)
1775 goto out;
1776
1777 irq_lookup_table = (void *)__get_free_pages(
1778 GFP_KERNEL | __GFP_ZERO,
1779 get_order(rlookup_table_size));
1780 if (!irq_lookup_table)
1781 goto out;
1782 }
1783
1784 ret = init_memory_definitions(ivrs_base);
1785 if (ret)
1786 goto out;
1787
1788 /* init the device table */
1789 init_device_table();
1790
1791 out:
1792 /* Don't leak any ACPI memory */
1793 early_acpi_os_unmap_memory((char __iomem *)ivrs_base, ivrs_size);
1794 ivrs_base = NULL;
1795
1796 return ret;
1797 }
1798
1799 static int amd_iommu_enable_interrupts(void)
1800 {
1801 struct amd_iommu *iommu;
1802 int ret = 0;
1803
1804 for_each_iommu(iommu) {
1805 ret = iommu_init_msi(iommu);
1806 if (ret)
1807 goto out;
1808 }
1809
1810 out:
1811 return ret;
1812 }
1813
1814 static bool detect_ivrs(void)
1815 {
1816 struct acpi_table_header *ivrs_base;
1817 acpi_size ivrs_size;
1818 acpi_status status;
1819
1820 status = acpi_get_table_with_size("IVRS", 0, &ivrs_base, &ivrs_size);
1821 if (status == AE_NOT_FOUND)
1822 return false;
1823 else if (ACPI_FAILURE(status)) {
1824 const char *err = acpi_format_exception(status);
1825 pr_err("AMD-Vi: IVRS table error: %s\n", err);
1826 return false;
1827 }
1828
1829 early_acpi_os_unmap_memory((char __iomem *)ivrs_base, ivrs_size);
1830
1831 /* Make sure ACS will be enabled during PCI probe */
1832 pci_request_acs();
1833
1834 if (!disable_irq_remap)
1835 amd_iommu_irq_remap = true;
1836
1837 return true;
1838 }
1839
1840 static int amd_iommu_init_dma(void)
1841 {
1842 struct amd_iommu *iommu;
1843 int ret;
1844
1845 init_device_table_dma();
1846
1847 for_each_iommu(iommu)
1848 iommu_flush_all_caches(iommu);
1849
1850 if (iommu_pass_through)
1851 ret = amd_iommu_init_passthrough();
1852 else
1853 ret = amd_iommu_init_dma_ops();
1854
1855 if (ret)
1856 return ret;
1857
1858 amd_iommu_init_api();
1859
1860 amd_iommu_init_notifier();
1861
1862 return 0;
1863 }
1864
1865 /****************************************************************************
1866 *
1867 * AMD IOMMU Initialization State Machine
1868 *
1869 ****************************************************************************/
1870
1871 static int __init state_next(void)
1872 {
1873 int ret = 0;
1874
1875 switch (init_state) {
1876 case IOMMU_START_STATE:
1877 if (!detect_ivrs()) {
1878 init_state = IOMMU_NOT_FOUND;
1879 ret = -ENODEV;
1880 } else {
1881 init_state = IOMMU_IVRS_DETECTED;
1882 }
1883 break;
1884 case IOMMU_IVRS_DETECTED:
1885 ret = early_amd_iommu_init();
1886 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED;
1887 break;
1888 case IOMMU_ACPI_FINISHED:
1889 early_enable_iommus();
1890 register_syscore_ops(&amd_iommu_syscore_ops);
1891 x86_platform.iommu_shutdown = disable_iommus;
1892 init_state = IOMMU_ENABLED;
1893 break;
1894 case IOMMU_ENABLED:
1895 ret = amd_iommu_init_pci();
1896 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT;
1897 enable_iommus_v2();
1898 break;
1899 case IOMMU_PCI_INIT:
1900 ret = amd_iommu_enable_interrupts();
1901 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN;
1902 break;
1903 case IOMMU_INTERRUPTS_EN:
1904 ret = amd_iommu_init_dma();
1905 init_state = ret ? IOMMU_INIT_ERROR : IOMMU_DMA_OPS;
1906 break;
1907 case IOMMU_DMA_OPS:
1908 init_state = IOMMU_INITIALIZED;
1909 break;
1910 case IOMMU_INITIALIZED:
1911 /* Nothing to do */
1912 break;
1913 case IOMMU_NOT_FOUND:
1914 case IOMMU_INIT_ERROR:
1915 /* Error states => do nothing */
1916 ret = -EINVAL;
1917 break;
1918 default:
1919 /* Unknown state */
1920 BUG();
1921 }
1922
1923 return ret;
1924 }
1925
1926 static int __init iommu_go_to_state(enum iommu_init_state state)
1927 {
1928 int ret = 0;
1929
1930 while (init_state != state) {
1931 ret = state_next();
1932 if (init_state == IOMMU_NOT_FOUND ||
1933 init_state == IOMMU_INIT_ERROR)
1934 break;
1935 }
1936
1937 return ret;
1938 }
1939
1940 #ifdef CONFIG_IRQ_REMAP
1941 int __init amd_iommu_prepare(void)
1942 {
1943 return iommu_go_to_state(IOMMU_ACPI_FINISHED);
1944 }
1945
1946 int __init amd_iommu_supported(void)
1947 {
1948 return amd_iommu_irq_remap ? 1 : 0;
1949 }
1950
1951 int __init amd_iommu_enable(void)
1952 {
1953 int ret;
1954
1955 ret = iommu_go_to_state(IOMMU_ENABLED);
1956 if (ret)
1957 return ret;
1958
1959 irq_remapping_enabled = 1;
1960
1961 return 0;
1962 }
1963
1964 void amd_iommu_disable(void)
1965 {
1966 amd_iommu_suspend();
1967 }
1968
1969 int amd_iommu_reenable(int mode)
1970 {
1971 amd_iommu_resume();
1972
1973 return 0;
1974 }
1975
1976 int __init amd_iommu_enable_faulting(void)
1977 {
1978 /* We enable MSI later when PCI is initialized */
1979 return 0;
1980 }
1981 #endif
1982
1983 /*
1984 * This is the core init function for AMD IOMMU hardware in the system.
1985 * This function is called from the generic x86 DMA layer initialization
1986 * code.
1987 */
1988 static int __init amd_iommu_init(void)
1989 {
1990 int ret;
1991
1992 ret = iommu_go_to_state(IOMMU_INITIALIZED);
1993 if (ret) {
1994 free_dma_resources();
1995 if (!irq_remapping_enabled) {
1996 disable_iommus();
1997 free_on_init_error();
1998 } else {
1999 struct amd_iommu *iommu;
2000
2001 uninit_device_table_dma();
2002 for_each_iommu(iommu)
2003 iommu_flush_all_caches(iommu);
2004 }
2005 }
2006
2007 return ret;
2008 }
2009
2010 /****************************************************************************
2011 *
2012 * Early detect code. This code runs at IOMMU detection time in the DMA
2013 * layer. It just looks if there is an IVRS ACPI table to detect AMD
2014 * IOMMUs
2015 *
2016 ****************************************************************************/
2017 int __init amd_iommu_detect(void)
2018 {
2019 int ret;
2020
2021 if (no_iommu || (iommu_detected && !gart_iommu_aperture))
2022 return -ENODEV;
2023
2024 if (amd_iommu_disabled)
2025 return -ENODEV;
2026
2027 ret = iommu_go_to_state(IOMMU_IVRS_DETECTED);
2028 if (ret)
2029 return ret;
2030
2031 amd_iommu_detected = true;
2032 iommu_detected = 1;
2033 x86_init.iommu.iommu_init = amd_iommu_init;
2034
2035 return 0;
2036 }
2037
2038 /****************************************************************************
2039 *
2040 * Parsing functions for the AMD IOMMU specific kernel command line
2041 * options.
2042 *
2043 ****************************************************************************/
2044
2045 static int __init parse_amd_iommu_dump(char *str)
2046 {
2047 amd_iommu_dump = true;
2048
2049 return 1;
2050 }
2051
2052 static int __init parse_amd_iommu_options(char *str)
2053 {
2054 for (; *str; ++str) {
2055 if (strncmp(str, "fullflush", 9) == 0)
2056 amd_iommu_unmap_flush = true;
2057 if (strncmp(str, "off", 3) == 0)
2058 amd_iommu_disabled = true;
2059 if (strncmp(str, "force_isolation", 15) == 0)
2060 amd_iommu_force_isolation = true;
2061 }
2062
2063 return 1;
2064 }
2065
2066 __setup("amd_iommu_dump", parse_amd_iommu_dump);
2067 __setup("amd_iommu=", parse_amd_iommu_options);
2068
2069 IOMMU_INIT_FINISH(amd_iommu_detect,
2070 gart_iommu_hole_init,
2071 NULL,
2072 NULL);
2073
2074 bool amd_iommu_v2_supported(void)
2075 {
2076 return amd_iommu_v2_present;
2077 }
2078 EXPORT_SYMBOL(amd_iommu_v2_supported);
Ubuntu Artful kernel mirror