]>
Commit | Line | Data |
---|---|---|
45051539 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
f6e2e6b6 | 2 | /* |
5d0d7156 | 3 | * Copyright (C) 2007-2010 Advanced Micro Devices, Inc. |
63ce3ae8 | 4 | * Author: Joerg Roedel <jroedel@suse.de> |
f6e2e6b6 | 5 | * Leo Duran <leo.duran@amd.com> |
f6e2e6b6 JR |
6 | */ |
7 | ||
101fa037 | 8 | #define pr_fmt(fmt) "AMD-Vi: " fmt |
5f226da1 | 9 | #define dev_fmt(fmt) pr_fmt(fmt) |
101fa037 | 10 | |
f6e2e6b6 JR |
11 | #include <linux/pci.h> |
12 | #include <linux/acpi.h> | |
f6e2e6b6 | 13 | #include <linux/list.h> |
5c87f62d | 14 | #include <linux/bitmap.h> |
5a0e3ad6 | 15 | #include <linux/slab.h> |
f3c6ea1b | 16 | #include <linux/syscore_ops.h> |
a80dc3e0 JR |
17 | #include <linux/interrupt.h> |
18 | #include <linux/msi.h> | |
403f81d8 | 19 | #include <linux/amd-iommu.h> |
400a28a0 | 20 | #include <linux/export.h> |
066f2e98 | 21 | #include <linux/iommu.h> |
ebcfa284 | 22 | #include <linux/kmemleak.h> |
2543a786 | 23 | #include <linux/mem_encrypt.h> |
f6e2e6b6 | 24 | #include <asm/pci-direct.h> |
46a7fa27 | 25 | #include <asm/iommu.h> |
1d9b16d1 | 26 | #include <asm/gart.h> |
ea1b0d39 | 27 | #include <asm/x86_init.h> |
22e6daf4 | 28 | #include <asm/iommu_table.h> |
eb1eb7ae | 29 | #include <asm/io_apic.h> |
6b474b82 | 30 | #include <asm/irq_remapping.h> |
403f81d8 | 31 | |
3ac3e5ee | 32 | #include <linux/crash_dump.h> |
403f81d8 JR |
33 | #include "amd_iommu_proto.h" |
34 | #include "amd_iommu_types.h" | |
05152a04 | 35 | #include "irq_remapping.h" |
403f81d8 | 36 | |
f6e2e6b6 JR |
37 | /* |
38 | * definitions for the ACPI scanning code | |
39 | */ | |
f6e2e6b6 | 40 | #define IVRS_HEADER_LENGTH 48 |
f6e2e6b6 | 41 | |
8c7142f5 | 42 | #define ACPI_IVHD_TYPE_MAX_SUPPORTED 0x40 |
f6e2e6b6 JR |
43 | #define ACPI_IVMD_TYPE_ALL 0x20 |
44 | #define ACPI_IVMD_TYPE 0x21 | |
45 | #define ACPI_IVMD_TYPE_RANGE 0x22 | |
46 | ||
47 | #define IVHD_DEV_ALL 0x01 | |
48 | #define IVHD_DEV_SELECT 0x02 | |
49 | #define IVHD_DEV_SELECT_RANGE_START 0x03 | |
50 | #define IVHD_DEV_RANGE_END 0x04 | |
51 | #define IVHD_DEV_ALIAS 0x42 | |
52 | #define IVHD_DEV_ALIAS_RANGE 0x43 | |
53 | #define IVHD_DEV_EXT_SELECT 0x46 | |
54 | #define IVHD_DEV_EXT_SELECT_RANGE 0x47 | |
6efed63b | 55 | #define IVHD_DEV_SPECIAL 0x48 |
8c7142f5 | 56 | #define IVHD_DEV_ACPI_HID 0xf0 |
6efed63b | 57 | |
2a0cb4e2 WZ |
58 | #define UID_NOT_PRESENT 0 |
59 | #define UID_IS_INTEGER 1 | |
60 | #define UID_IS_CHARACTER 2 | |
61 | ||
6efed63b JR |
62 | #define IVHD_SPECIAL_IOAPIC 1 |
63 | #define IVHD_SPECIAL_HPET 2 | |
f6e2e6b6 | 64 | |
6da7342f JR |
65 | #define IVHD_FLAG_HT_TUN_EN_MASK 0x01 |
66 | #define IVHD_FLAG_PASSPW_EN_MASK 0x02 | |
67 | #define IVHD_FLAG_RESPASSPW_EN_MASK 0x04 | |
68 | #define IVHD_FLAG_ISOC_EN_MASK 0x08 | |
f6e2e6b6 JR |
69 | |
70 | #define IVMD_FLAG_EXCL_RANGE 0x08 | |
71 | #define IVMD_FLAG_UNITY_MAP 0x01 | |
72 | ||
73 | #define ACPI_DEVFLAG_INITPASS 0x01 | |
74 | #define ACPI_DEVFLAG_EXTINT 0x02 | |
75 | #define ACPI_DEVFLAG_NMI 0x04 | |
76 | #define ACPI_DEVFLAG_SYSMGT1 0x10 | |
77 | #define ACPI_DEVFLAG_SYSMGT2 0x20 | |
78 | #define ACPI_DEVFLAG_LINT0 0x40 | |
79 | #define ACPI_DEVFLAG_LINT1 0x80 | |
80 | #define ACPI_DEVFLAG_ATSDIS 0x10000000 | |
81 | ||
8bda0cfb | 82 | #define LOOP_TIMEOUT 100000 |
b65233a9 JR |
83 | /* |
84 | * ACPI table definitions | |
85 | * | |
86 | * These data structures are laid over the table to parse the important values | |
87 | * out of it. | |
88 | */ | |
89 | ||
b0119e87 JR |
90 | extern const struct iommu_ops amd_iommu_ops; |
91 | ||
b65233a9 JR |
92 | /* |
93 | * structure describing one IOMMU in the ACPI table. Typically followed by one | |
94 | * or more ivhd_entrys. | |
95 | */ | |
f6e2e6b6 JR |
96 | struct ivhd_header { |
97 | u8 type; | |
98 | u8 flags; | |
99 | u16 length; | |
100 | u16 devid; | |
101 | u16 cap_ptr; | |
102 | u64 mmio_phys; | |
103 | u16 pci_seg; | |
104 | u16 info; | |
7d7d38af SS |
105 | u32 efr_attr; |
106 | ||
107 | /* Following only valid on IVHD type 11h and 40h */ | |
108 | u64 efr_reg; /* Exact copy of MMIO_EXT_FEATURES */ | |
109 | u64 res; | |
f6e2e6b6 JR |
110 | } __attribute__((packed)); |
111 | ||
b65233a9 JR |
112 | /* |
113 | * A device entry describing which devices a specific IOMMU translates and | |
114 | * which requestor ids they use. | |
115 | */ | |
f6e2e6b6 JR |
116 | struct ivhd_entry { |
117 | u8 type; | |
118 | u16 devid; | |
119 | u8 flags; | |
120 | u32 ext; | |
2a0cb4e2 WZ |
121 | u32 hidh; |
122 | u64 cid; | |
123 | u8 uidf; | |
124 | u8 uidl; | |
125 | u8 uid; | |
f6e2e6b6 JR |
126 | } __attribute__((packed)); |
127 | ||
b65233a9 JR |
128 | /* |
129 | * An AMD IOMMU memory definition structure. It defines things like exclusion | |
130 | * ranges for devices and regions that should be unity mapped. | |
131 | */ | |
f6e2e6b6 JR |
132 | struct ivmd_header { |
133 | u8 type; | |
134 | u8 flags; | |
135 | u16 length; | |
136 | u16 devid; | |
137 | u16 aux; | |
138 | u64 resv; | |
139 | u64 range_start; | |
140 | u64 range_length; | |
141 | } __attribute__((packed)); | |
142 | ||
fefda117 | 143 | bool amd_iommu_dump; |
05152a04 | 144 | bool amd_iommu_irq_remap __read_mostly; |
fefda117 | 145 | |
d98de49a | 146 | int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; |
90fcffd9 | 147 | static int amd_iommu_xt_mode = IRQ_REMAP_X2APIC_MODE; |
3928aa3f | 148 | |
02f3b3f5 | 149 | static bool amd_iommu_detected; |
a5235725 | 150 | static bool __initdata amd_iommu_disabled; |
8c7142f5 | 151 | static int amd_iommu_target_ivhd_type; |
c1cbebee | 152 | |
b65233a9 JR |
153 | u16 amd_iommu_last_bdf; /* largest PCI device id we have |
154 | to handle */ | |
2e22847f | 155 | LIST_HEAD(amd_iommu_unity_map); /* a list of required unity mappings |
b65233a9 | 156 | we find in ACPI */ |
621a5f7a | 157 | bool amd_iommu_unmap_flush; /* if true, flush on every unmap */ |
928abd25 | 158 | |
2e22847f | 159 | LIST_HEAD(amd_iommu_list); /* list of all AMD IOMMUs in the |
b65233a9 | 160 | system */ |
928abd25 | 161 | |
bb52777e JR |
162 | /* Array to assign indices to IOMMUs*/ |
163 | struct amd_iommu *amd_iommus[MAX_IOMMUS]; | |
6b9376e3 SS |
164 | |
165 | /* Number of IOMMUs present in the system */ | |
166 | static int amd_iommus_present; | |
bb52777e | 167 | |
318afd41 JR |
168 | /* IOMMUs have a non-present cache? */ |
169 | bool amd_iommu_np_cache __read_mostly; | |
60f723b4 | 170 | bool amd_iommu_iotlb_sup __read_mostly = true; |
318afd41 | 171 | |
a919a018 | 172 | u32 amd_iommu_max_pasid __read_mostly = ~0; |
62f71abb | 173 | |
400a28a0 | 174 | bool amd_iommu_v2_present __read_mostly; |
4160cd9e | 175 | static bool amd_iommu_pc_present __read_mostly; |
400a28a0 | 176 | |
5abcdba4 JR |
177 | bool amd_iommu_force_isolation __read_mostly; |
178 | ||
b65233a9 JR |
179 | /* |
180 | * Pointer to the device table which is shared by all AMD IOMMUs | |
181 | * it is indexed by the PCI device id or the HT unit id and contains | |
182 | * information about the domain the device belongs to as well as the | |
183 | * page table root pointer. | |
184 | */ | |
928abd25 | 185 | struct dev_table_entry *amd_iommu_dev_table; |
45a01c42 BH |
186 | /* |
187 | * Pointer to a device table which the content of old device table | |
188 | * will be copied to. It's only be used in kdump kernel. | |
189 | */ | |
190 | static struct dev_table_entry *old_dev_tbl_cpy; | |
b65233a9 JR |
191 | |
192 | /* | |
193 | * The alias table is a driver specific data structure which contains the | |
194 | * mappings of the PCI device ids to the actual requestor ids on the IOMMU. | |
195 | * More than one device can share the same requestor id. | |
196 | */ | |
928abd25 | 197 | u16 *amd_iommu_alias_table; |
b65233a9 JR |
198 | |
199 | /* | |
200 | * The rlookup table is used to find the IOMMU which is responsible | |
201 | * for a specific device. It is also indexed by the PCI device id. | |
202 | */ | |
928abd25 | 203 | struct amd_iommu **amd_iommu_rlookup_table; |
daae2d25 | 204 | EXPORT_SYMBOL(amd_iommu_rlookup_table); |
b65233a9 | 205 | |
b65233a9 | 206 | /* |
0ea2c422 JR |
207 | * This table is used to find the irq remapping table for a given device id |
208 | * quickly. | |
209 | */ | |
210 | struct irq_remap_table **irq_lookup_table; | |
211 | ||
b65233a9 | 212 | /* |
df805abb | 213 | * AMD IOMMU allows up to 2^16 different protection domains. This is a bitmap |
b65233a9 JR |
214 | * to know which ones are already in use. |
215 | */ | |
928abd25 JR |
216 | unsigned long *amd_iommu_pd_alloc_bitmap; |
217 | ||
b65233a9 JR |
218 | static u32 dev_table_size; /* size of the device table */ |
219 | static u32 alias_table_size; /* size of the alias table */ | |
220 | static u32 rlookup_table_size; /* size if the rlookup table */ | |
3e8064ba | 221 | |
2c0ae172 JR |
222 | enum iommu_init_state { |
223 | IOMMU_START_STATE, | |
224 | IOMMU_IVRS_DETECTED, | |
225 | IOMMU_ACPI_FINISHED, | |
226 | IOMMU_ENABLED, | |
227 | IOMMU_PCI_INIT, | |
228 | IOMMU_INTERRUPTS_EN, | |
229 | IOMMU_DMA_OPS, | |
230 | IOMMU_INITIALIZED, | |
231 | IOMMU_NOT_FOUND, | |
232 | IOMMU_INIT_ERROR, | |
1b1e942e | 233 | IOMMU_CMDLINE_DISABLED, |
2c0ae172 JR |
234 | }; |
235 | ||
235dacbc JR |
236 | /* Early ioapic and hpet maps from kernel command line */ |
237 | #define EARLY_MAP_SIZE 4 | |
238 | static struct devid_map __initdata early_ioapic_map[EARLY_MAP_SIZE]; | |
239 | static struct devid_map __initdata early_hpet_map[EARLY_MAP_SIZE]; | |
2a0cb4e2 WZ |
240 | static struct acpihid_map_entry __initdata early_acpihid_map[EARLY_MAP_SIZE]; |
241 | ||
235dacbc JR |
242 | static int __initdata early_ioapic_map_size; |
243 | static int __initdata early_hpet_map_size; | |
2a0cb4e2 WZ |
244 | static int __initdata early_acpihid_map_size; |
245 | ||
dfbb6d47 | 246 | static bool __initdata cmdline_maps; |
235dacbc | 247 | |
2c0ae172 JR |
248 | static enum iommu_init_state init_state = IOMMU_START_STATE; |
249 | ||
ae295142 | 250 | static int amd_iommu_enable_interrupts(void); |
2c0ae172 | 251 | static int __init iommu_go_to_state(enum iommu_init_state state); |
aafd8ba0 | 252 | static void init_device_table_dma(void); |
3d9761e7 | 253 | |
2479c631 | 254 | static bool amd_iommu_pre_enabled = true; |
3ac3e5ee | 255 | |
4c232a70 BH |
256 | bool translation_pre_enabled(struct amd_iommu *iommu) |
257 | { | |
258 | return (iommu->flags & AMD_IOMMU_FLAG_TRANS_PRE_ENABLED); | |
259 | } | |
daae2d25 | 260 | EXPORT_SYMBOL(translation_pre_enabled); |
4c232a70 BH |
261 | |
262 | static void clear_translation_pre_enabled(struct amd_iommu *iommu) | |
263 | { | |
264 | iommu->flags &= ~AMD_IOMMU_FLAG_TRANS_PRE_ENABLED; | |
265 | } | |
266 | ||
267 | static void init_translation_status(struct amd_iommu *iommu) | |
268 | { | |
e881dbd5 | 269 | u64 ctrl; |
4c232a70 | 270 | |
e881dbd5 | 271 | ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); |
4c232a70 BH |
272 | if (ctrl & (1<<CONTROL_IOMMU_EN)) |
273 | iommu->flags |= AMD_IOMMU_FLAG_TRANS_PRE_ENABLED; | |
274 | } | |
275 | ||
208ec8c9 JR |
276 | static inline void update_last_devid(u16 devid) |
277 | { | |
278 | if (devid > amd_iommu_last_bdf) | |
279 | amd_iommu_last_bdf = devid; | |
280 | } | |
281 | ||
c571484e JR |
282 | static inline unsigned long tbl_size(int entry_size) |
283 | { | |
284 | unsigned shift = PAGE_SHIFT + | |
421f909c | 285 | get_order(((int)amd_iommu_last_bdf + 1) * entry_size); |
c571484e JR |
286 | |
287 | return 1UL << shift; | |
288 | } | |
289 | ||
6b9376e3 SS |
290 | int amd_iommu_get_num_iommus(void) |
291 | { | |
292 | return amd_iommus_present; | |
293 | } | |
294 | ||
5bcd757f MG |
295 | /* Access to l1 and l2 indexed register spaces */ |
296 | ||
297 | static u32 iommu_read_l1(struct amd_iommu *iommu, u16 l1, u8 address) | |
298 | { | |
299 | u32 val; | |
300 | ||
301 | pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); | |
302 | pci_read_config_dword(iommu->dev, 0xfc, &val); | |
303 | return val; | |
304 | } | |
305 | ||
306 | static void iommu_write_l1(struct amd_iommu *iommu, u16 l1, u8 address, u32 val) | |
307 | { | |
308 | pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16 | 1 << 31)); | |
309 | pci_write_config_dword(iommu->dev, 0xfc, val); | |
310 | pci_write_config_dword(iommu->dev, 0xf8, (address | l1 << 16)); | |
311 | } | |
312 | ||
313 | static u32 iommu_read_l2(struct amd_iommu *iommu, u8 address) | |
314 | { | |
315 | u32 val; | |
316 | ||
317 | pci_write_config_dword(iommu->dev, 0xf0, address); | |
318 | pci_read_config_dword(iommu->dev, 0xf4, &val); | |
319 | return val; | |
320 | } | |
321 | ||
322 | static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val) | |
323 | { | |
324 | pci_write_config_dword(iommu->dev, 0xf0, (address | 1 << 8)); | |
325 | pci_write_config_dword(iommu->dev, 0xf4, val); | |
326 | } | |
327 | ||
b65233a9 JR |
328 | /**************************************************************************** |
329 | * | |
330 | * AMD IOMMU MMIO register space handling functions | |
331 | * | |
332 | * These functions are used to program the IOMMU device registers in | |
333 | * MMIO space required for that driver. | |
334 | * | |
335 | ****************************************************************************/ | |
3e8064ba | 336 | |
b65233a9 JR |
337 | /* |
338 | * This function set the exclusion range in the IOMMU. DMA accesses to the | |
339 | * exclusion range are passed through untranslated | |
340 | */ | |
05f92db9 | 341 | static void iommu_set_exclusion_range(struct amd_iommu *iommu) |
b2026aa2 JR |
342 | { |
343 | u64 start = iommu->exclusion_start & PAGE_MASK; | |
3c677d20 | 344 | u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK; |
b2026aa2 JR |
345 | u64 entry; |
346 | ||
347 | if (!iommu->exclusion_start) | |
348 | return; | |
349 | ||
350 | entry = start | MMIO_EXCL_ENABLE_MASK; | |
351 | memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET, | |
352 | &entry, sizeof(entry)); | |
353 | ||
354 | entry = limit; | |
355 | memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET, | |
356 | &entry, sizeof(entry)); | |
357 | } | |
358 | ||
b65233a9 | 359 | /* Programs the physical address of the device table into the IOMMU hardware */ |
6b7f000e | 360 | static void iommu_set_device_table(struct amd_iommu *iommu) |
b2026aa2 | 361 | { |
f609891f | 362 | u64 entry; |
b2026aa2 JR |
363 | |
364 | BUG_ON(iommu->mmio_base == NULL); | |
365 | ||
2543a786 | 366 | entry = iommu_virt_to_phys(amd_iommu_dev_table); |
b2026aa2 JR |
367 | entry |= (dev_table_size >> 12) - 1; |
368 | memcpy_toio(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET, | |
369 | &entry, sizeof(entry)); | |
370 | } | |
371 | ||
b65233a9 | 372 | /* Generic functions to enable/disable certain features of the IOMMU. */ |
05f92db9 | 373 | static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit) |
b2026aa2 | 374 | { |
e881dbd5 | 375 | u64 ctrl; |
b2026aa2 | 376 | |
e881dbd5 SS |
377 | ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); |
378 | ctrl |= (1ULL << bit); | |
379 | writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); | |
b2026aa2 JR |
380 | } |
381 | ||
ca020711 | 382 | static void iommu_feature_disable(struct amd_iommu *iommu, u8 bit) |
b2026aa2 | 383 | { |
e881dbd5 | 384 | u64 ctrl; |
b2026aa2 | 385 | |
e881dbd5 SS |
386 | ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); |
387 | ctrl &= ~(1ULL << bit); | |
388 | writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); | |
b2026aa2 JR |
389 | } |
390 | ||
1456e9d2 JR |
391 | static void iommu_set_inv_tlb_timeout(struct amd_iommu *iommu, int timeout) |
392 | { | |
e881dbd5 | 393 | u64 ctrl; |
1456e9d2 | 394 | |
e881dbd5 | 395 | ctrl = readq(iommu->mmio_base + MMIO_CONTROL_OFFSET); |
1456e9d2 JR |
396 | ctrl &= ~CTRL_INV_TO_MASK; |
397 | ctrl |= (timeout << CONTROL_INV_TIMEOUT) & CTRL_INV_TO_MASK; | |
e881dbd5 | 398 | writeq(ctrl, iommu->mmio_base + MMIO_CONTROL_OFFSET); |
1456e9d2 JR |
399 | } |
400 | ||
b65233a9 | 401 | /* Function to enable the hardware */ |
05f92db9 | 402 | static void iommu_enable(struct amd_iommu *iommu) |
b2026aa2 | 403 | { |
b2026aa2 | 404 | iommu_feature_enable(iommu, CONTROL_IOMMU_EN); |
b2026aa2 JR |
405 | } |
406 | ||
92ac4320 | 407 | static void iommu_disable(struct amd_iommu *iommu) |
126c52be | 408 | { |
a8c485bb CW |
409 | /* Disable command buffer */ |
410 | iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); | |
411 | ||
412 | /* Disable event logging and event interrupts */ | |
413 | iommu_feature_disable(iommu, CONTROL_EVT_INT_EN); | |
414 | iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN); | |
415 | ||
8bda0cfb SS |
416 | /* Disable IOMMU GA_LOG */ |
417 | iommu_feature_disable(iommu, CONTROL_GALOG_EN); | |
418 | iommu_feature_disable(iommu, CONTROL_GAINT_EN); | |
419 | ||
a8c485bb | 420 | /* Disable IOMMU hardware itself */ |
92ac4320 | 421 | iommu_feature_disable(iommu, CONTROL_IOMMU_EN); |
126c52be JR |
422 | } |
423 | ||
b65233a9 JR |
424 | /* |
425 | * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in | |
426 | * the system has one. | |
427 | */ | |
30861ddc | 428 | static u8 __iomem * __init iommu_map_mmio_space(u64 address, u64 end) |
6c56747b | 429 | { |
30861ddc | 430 | if (!request_mem_region(address, end, "amd_iommu")) { |
101fa037 | 431 | pr_err("Can not reserve memory region %llx-%llx for mmio\n", |
30861ddc | 432 | address, end); |
101fa037 | 433 | pr_err("This is a BIOS bug. Please contact your hardware vendor\n"); |
6c56747b | 434 | return NULL; |
e82752d8 | 435 | } |
6c56747b | 436 | |
30861ddc | 437 | return (u8 __iomem *)ioremap_nocache(address, end); |
6c56747b JR |
438 | } |
439 | ||
440 | static void __init iommu_unmap_mmio_space(struct amd_iommu *iommu) | |
441 | { | |
442 | if (iommu->mmio_base) | |
443 | iounmap(iommu->mmio_base); | |
30861ddc | 444 | release_mem_region(iommu->mmio_phys, iommu->mmio_phys_end); |
6c56747b JR |
445 | } |
446 | ||
ac7ccf67 SS |
447 | static inline u32 get_ivhd_header_size(struct ivhd_header *h) |
448 | { | |
449 | u32 size = 0; | |
450 | ||
451 | switch (h->type) { | |
452 | case 0x10: | |
453 | size = 24; | |
454 | break; | |
455 | case 0x11: | |
456 | case 0x40: | |
457 | size = 40; | |
458 | break; | |
459 | } | |
460 | return size; | |
461 | } | |
462 | ||
b65233a9 JR |
463 | /**************************************************************************** |
464 | * | |
465 | * The functions below belong to the first pass of AMD IOMMU ACPI table | |
466 | * parsing. In this pass we try to find out the highest device id this | |
467 | * code has to handle. Upon this information the size of the shared data | |
468 | * structures is determined later. | |
469 | * | |
470 | ****************************************************************************/ | |
471 | ||
b514e555 JR |
472 | /* |
473 | * This function calculates the length of a given IVHD entry | |
474 | */ | |
475 | static inline int ivhd_entry_length(u8 *ivhd) | |
476 | { | |
8c7142f5 SS |
477 | u32 type = ((struct ivhd_entry *)ivhd)->type; |
478 | ||
479 | if (type < 0x80) { | |
480 | return 0x04 << (*ivhd >> 6); | |
481 | } else if (type == IVHD_DEV_ACPI_HID) { | |
482 | /* For ACPI_HID, offset 21 is uid len */ | |
483 | return *((u8 *)ivhd + 21) + 22; | |
484 | } | |
485 | return 0; | |
b514e555 JR |
486 | } |
487 | ||
b65233a9 JR |
488 | /* |
489 | * After reading the highest device id from the IOMMU PCI capability header | |
490 | * this function looks if there is a higher device id defined in the ACPI table | |
491 | */ | |
3e8064ba JR |
492 | static int __init find_last_devid_from_ivhd(struct ivhd_header *h) |
493 | { | |
494 | u8 *p = (void *)h, *end = (void *)h; | |
495 | struct ivhd_entry *dev; | |
496 | ||
ac7ccf67 SS |
497 | u32 ivhd_size = get_ivhd_header_size(h); |
498 | ||
499 | if (!ivhd_size) { | |
101fa037 | 500 | pr_err("Unsupported IVHD type %#x\n", h->type); |
ac7ccf67 SS |
501 | return -EINVAL; |
502 | } | |
503 | ||
504 | p += ivhd_size; | |
3e8064ba JR |
505 | end += h->length; |
506 | ||
3e8064ba JR |
507 | while (p < end) { |
508 | dev = (struct ivhd_entry *)p; | |
509 | switch (dev->type) { | |
d1259416 JR |
510 | case IVHD_DEV_ALL: |
511 | /* Use maximum BDF value for DEV_ALL */ | |
512 | update_last_devid(0xffff); | |
513 | break; | |
3e8064ba JR |
514 | case IVHD_DEV_SELECT: |
515 | case IVHD_DEV_RANGE_END: | |
516 | case IVHD_DEV_ALIAS: | |
517 | case IVHD_DEV_EXT_SELECT: | |
b65233a9 | 518 | /* all the above subfield types refer to device ids */ |
208ec8c9 | 519 | update_last_devid(dev->devid); |
3e8064ba JR |
520 | break; |
521 | default: | |
522 | break; | |
523 | } | |
b514e555 | 524 | p += ivhd_entry_length(p); |
3e8064ba JR |
525 | } |
526 | ||
527 | WARN_ON(p != end); | |
528 | ||
529 | return 0; | |
530 | } | |
531 | ||
8c7142f5 SS |
532 | static int __init check_ivrs_checksum(struct acpi_table_header *table) |
533 | { | |
534 | int i; | |
535 | u8 checksum = 0, *p = (u8 *)table; | |
536 | ||
537 | for (i = 0; i < table->length; ++i) | |
538 | checksum += p[i]; | |
539 | if (checksum != 0) { | |
540 | /* ACPI table corrupt */ | |
101fa037 | 541 | pr_err(FW_BUG "IVRS invalid checksum\n"); |
8c7142f5 SS |
542 | return -ENODEV; |
543 | } | |
544 | ||
545 | return 0; | |
546 | } | |
547 | ||
b65233a9 JR |
548 | /* |
549 | * Iterate over all IVHD entries in the ACPI table and find the highest device | |
550 | * id which we need to handle. This is the first of three functions which parse | |
551 | * the ACPI table. So we check the checksum here. | |
552 | */ | |
3e8064ba JR |
553 | static int __init find_last_devid_acpi(struct acpi_table_header *table) |
554 | { | |
8c7142f5 | 555 | u8 *p = (u8 *)table, *end = (u8 *)table; |
3e8064ba JR |
556 | struct ivhd_header *h; |
557 | ||
3e8064ba JR |
558 | p += IVRS_HEADER_LENGTH; |
559 | ||
560 | end += table->length; | |
561 | while (p < end) { | |
562 | h = (struct ivhd_header *)p; | |
8c7142f5 SS |
563 | if (h->type == amd_iommu_target_ivhd_type) { |
564 | int ret = find_last_devid_from_ivhd(h); | |
565 | ||
566 | if (ret) | |
567 | return ret; | |
3e8064ba JR |
568 | } |
569 | p += h->length; | |
570 | } | |
571 | WARN_ON(p != end); | |
572 | ||
573 | return 0; | |
574 | } | |
575 | ||
b65233a9 JR |
576 | /**************************************************************************** |
577 | * | |
df805abb | 578 | * The following functions belong to the code path which parses the ACPI table |
b65233a9 JR |
579 | * the second time. In this ACPI parsing iteration we allocate IOMMU specific |
580 | * data structures, initialize the device/alias/rlookup table and also | |
581 | * basically initialize the hardware. | |
582 | * | |
583 | ****************************************************************************/ | |
584 | ||
585 | /* | |
586 | * Allocates the command buffer. This buffer is per AMD IOMMU. We can | |
587 | * write commands to that buffer later and the IOMMU will execute them | |
588 | * asynchronously | |
589 | */ | |
f2c2db53 | 590 | static int __init alloc_command_buffer(struct amd_iommu *iommu) |
b36ca91e | 591 | { |
f2c2db53 JR |
592 | iommu->cmd_buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, |
593 | get_order(CMD_BUFFER_SIZE)); | |
b36ca91e | 594 | |
f2c2db53 | 595 | return iommu->cmd_buf ? 0 : -ENOMEM; |
58492e12 JR |
596 | } |
597 | ||
93f1cc67 JR |
598 | /* |
599 | * This function resets the command buffer if the IOMMU stopped fetching | |
600 | * commands from it. | |
601 | */ | |
602 | void amd_iommu_reset_cmd_buffer(struct amd_iommu *iommu) | |
603 | { | |
604 | iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); | |
605 | ||
606 | writel(0x00, iommu->mmio_base + MMIO_CMD_HEAD_OFFSET); | |
607 | writel(0x00, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); | |
d334a563 TL |
608 | iommu->cmd_buf_head = 0; |
609 | iommu->cmd_buf_tail = 0; | |
93f1cc67 JR |
610 | |
611 | iommu_feature_enable(iommu, CONTROL_CMDBUF_EN); | |
612 | } | |
613 | ||
58492e12 JR |
614 | /* |
615 | * This function writes the command buffer address to the hardware and | |
616 | * enables it. | |
617 | */ | |
618 | static void iommu_enable_command_buffer(struct amd_iommu *iommu) | |
619 | { | |
620 | u64 entry; | |
621 | ||
622 | BUG_ON(iommu->cmd_buf == NULL); | |
623 | ||
2543a786 | 624 | entry = iommu_virt_to_phys(iommu->cmd_buf); |
b36ca91e | 625 | entry |= MMIO_CMD_SIZE_512; |
58492e12 | 626 | |
b36ca91e | 627 | memcpy_toio(iommu->mmio_base + MMIO_CMD_BUF_OFFSET, |
58492e12 | 628 | &entry, sizeof(entry)); |
b36ca91e | 629 | |
93f1cc67 | 630 | amd_iommu_reset_cmd_buffer(iommu); |
b36ca91e JR |
631 | } |
632 | ||
78d313c6 BH |
633 | /* |
634 | * This function disables the command buffer | |
635 | */ | |
636 | static void iommu_disable_command_buffer(struct amd_iommu *iommu) | |
637 | { | |
638 | iommu_feature_disable(iommu, CONTROL_CMDBUF_EN); | |
639 | } | |
640 | ||
b36ca91e JR |
641 | static void __init free_command_buffer(struct amd_iommu *iommu) |
642 | { | |
deba4bce | 643 | free_pages((unsigned long)iommu->cmd_buf, get_order(CMD_BUFFER_SIZE)); |
b36ca91e JR |
644 | } |
645 | ||
335503e5 | 646 | /* allocates the memory where the IOMMU will log its events to */ |
f2c2db53 | 647 | static int __init alloc_event_buffer(struct amd_iommu *iommu) |
335503e5 | 648 | { |
f2c2db53 JR |
649 | iommu->evt_buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, |
650 | get_order(EVT_BUFFER_SIZE)); | |
335503e5 | 651 | |
f2c2db53 | 652 | return iommu->evt_buf ? 0 : -ENOMEM; |
58492e12 JR |
653 | } |
654 | ||
655 | static void iommu_enable_event_buffer(struct amd_iommu *iommu) | |
656 | { | |
657 | u64 entry; | |
658 | ||
659 | BUG_ON(iommu->evt_buf == NULL); | |
660 | ||
2543a786 | 661 | entry = iommu_virt_to_phys(iommu->evt_buf) | EVT_LEN_MASK; |
58492e12 | 662 | |
335503e5 JR |
663 | memcpy_toio(iommu->mmio_base + MMIO_EVT_BUF_OFFSET, |
664 | &entry, sizeof(entry)); | |
665 | ||
09067207 JR |
666 | /* set head and tail to zero manually */ |
667 | writel(0x00, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET); | |
668 | writel(0x00, iommu->mmio_base + MMIO_EVT_TAIL_OFFSET); | |
669 | ||
58492e12 | 670 | iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN); |
335503e5 JR |
671 | } |
672 | ||
78d313c6 BH |
673 | /* |
674 | * This function disables the event log buffer | |
675 | */ | |
676 | static void iommu_disable_event_buffer(struct amd_iommu *iommu) | |
677 | { | |
678 | iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN); | |
679 | } | |
680 | ||
335503e5 JR |
681 | static void __init free_event_buffer(struct amd_iommu *iommu) |
682 | { | |
683 | free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE)); | |
684 | } | |
685 | ||
1a29ac01 | 686 | /* allocates the memory where the IOMMU will log its events to */ |
f2c2db53 | 687 | static int __init alloc_ppr_log(struct amd_iommu *iommu) |
1a29ac01 | 688 | { |
f2c2db53 JR |
689 | iommu->ppr_log = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, |
690 | get_order(PPR_LOG_SIZE)); | |
1a29ac01 | 691 | |
f2c2db53 | 692 | return iommu->ppr_log ? 0 : -ENOMEM; |
1a29ac01 JR |
693 | } |
694 | ||
695 | static void iommu_enable_ppr_log(struct amd_iommu *iommu) | |
696 | { | |
697 | u64 entry; | |
698 | ||
699 | if (iommu->ppr_log == NULL) | |
700 | return; | |
701 | ||
2543a786 | 702 | entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512; |
1a29ac01 JR |
703 | |
704 | memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET, | |
705 | &entry, sizeof(entry)); | |
706 | ||
707 | /* set head and tail to zero manually */ | |
708 | writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET); | |
709 | writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); | |
710 | ||
711 | iommu_feature_enable(iommu, CONTROL_PPFLOG_EN); | |
712 | iommu_feature_enable(iommu, CONTROL_PPR_EN); | |
713 | } | |
714 | ||
715 | static void __init free_ppr_log(struct amd_iommu *iommu) | |
716 | { | |
717 | if (iommu->ppr_log == NULL) | |
718 | return; | |
719 | ||
720 | free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE)); | |
721 | } | |
722 | ||
8bda0cfb SS |
723 | static void free_ga_log(struct amd_iommu *iommu) |
724 | { | |
725 | #ifdef CONFIG_IRQ_REMAP | |
726 | if (iommu->ga_log) | |
727 | free_pages((unsigned long)iommu->ga_log, | |
728 | get_order(GA_LOG_SIZE)); | |
729 | if (iommu->ga_log_tail) | |
730 | free_pages((unsigned long)iommu->ga_log_tail, | |
731 | get_order(8)); | |
732 | #endif | |
733 | } | |
734 | ||
735 | static int iommu_ga_log_enable(struct amd_iommu *iommu) | |
736 | { | |
737 | #ifdef CONFIG_IRQ_REMAP | |
738 | u32 status, i; | |
739 | ||
740 | if (!iommu->ga_log) | |
741 | return -EINVAL; | |
742 | ||
743 | status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); | |
744 | ||
745 | /* Check if already running */ | |
746 | if (status & (MMIO_STATUS_GALOG_RUN_MASK)) | |
747 | return 0; | |
748 | ||
749 | iommu_feature_enable(iommu, CONTROL_GAINT_EN); | |
750 | iommu_feature_enable(iommu, CONTROL_GALOG_EN); | |
751 | ||
752 | for (i = 0; i < LOOP_TIMEOUT; ++i) { | |
753 | status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); | |
754 | if (status & (MMIO_STATUS_GALOG_RUN_MASK)) | |
755 | break; | |
756 | } | |
757 | ||
758 | if (i >= LOOP_TIMEOUT) | |
759 | return -EINVAL; | |
760 | #endif /* CONFIG_IRQ_REMAP */ | |
761 | return 0; | |
762 | } | |
763 | ||
764 | #ifdef CONFIG_IRQ_REMAP | |
765 | static int iommu_init_ga_log(struct amd_iommu *iommu) | |
766 | { | |
767 | u64 entry; | |
768 | ||
769 | if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) | |
770 | return 0; | |
771 | ||
772 | iommu->ga_log = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, | |
773 | get_order(GA_LOG_SIZE)); | |
774 | if (!iommu->ga_log) | |
775 | goto err_out; | |
776 | ||
777 | iommu->ga_log_tail = (u8 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, | |
778 | get_order(8)); | |
779 | if (!iommu->ga_log_tail) | |
780 | goto err_out; | |
781 | ||
2543a786 | 782 | entry = iommu_virt_to_phys(iommu->ga_log) | GA_LOG_SIZE_512; |
8bda0cfb SS |
783 | memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_BASE_OFFSET, |
784 | &entry, sizeof(entry)); | |
ab99be46 FS |
785 | entry = (iommu_virt_to_phys(iommu->ga_log_tail) & |
786 | (BIT_ULL(52)-1)) & ~7ULL; | |
8bda0cfb SS |
787 | memcpy_toio(iommu->mmio_base + MMIO_GA_LOG_TAIL_OFFSET, |
788 | &entry, sizeof(entry)); | |
789 | writel(0x00, iommu->mmio_base + MMIO_GA_HEAD_OFFSET); | |
790 | writel(0x00, iommu->mmio_base + MMIO_GA_TAIL_OFFSET); | |
791 | ||
792 | return 0; | |
793 | err_out: | |
794 | free_ga_log(iommu); | |
795 | return -EINVAL; | |
796 | } | |
797 | #endif /* CONFIG_IRQ_REMAP */ | |
798 | ||
799 | static int iommu_init_ga(struct amd_iommu *iommu) | |
800 | { | |
801 | int ret = 0; | |
802 | ||
803 | #ifdef CONFIG_IRQ_REMAP | |
804 | /* Note: We have already checked GASup from IVRS table. | |
805 | * Now, we need to make sure that GAMSup is set. | |
806 | */ | |
807 | if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) && | |
808 | !iommu_feature(iommu, FEATURE_GAM_VAPIC)) | |
809 | amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA; | |
810 | ||
811 | ret = iommu_init_ga_log(iommu); | |
812 | #endif /* CONFIG_IRQ_REMAP */ | |
813 | ||
814 | return ret; | |
815 | } | |
816 | ||
90fcffd9 SS |
817 | static void iommu_enable_xt(struct amd_iommu *iommu) |
818 | { | |
819 | #ifdef CONFIG_IRQ_REMAP | |
820 | /* | |
821 | * XT mode (32-bit APIC destination ID) requires | |
822 | * GA mode (128-bit IRTE support) as a prerequisite. | |
823 | */ | |
824 | if (AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir) && | |
825 | amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) | |
826 | iommu_feature_enable(iommu, CONTROL_XT_EN); | |
827 | #endif /* CONFIG_IRQ_REMAP */ | |
828 | } | |
829 | ||
cbc33a90 JR |
830 | static void iommu_enable_gt(struct amd_iommu *iommu) |
831 | { | |
832 | if (!iommu_feature(iommu, FEATURE_GT)) | |
833 | return; | |
834 | ||
835 | iommu_feature_enable(iommu, CONTROL_GT_EN); | |
836 | } | |
837 | ||
b65233a9 | 838 | /* sets a specific bit in the device table entry. */ |
3566b778 JR |
839 | static void set_dev_entry_bit(u16 devid, u8 bit) |
840 | { | |
ee6c2868 JR |
841 | int i = (bit >> 6) & 0x03; |
842 | int _bit = bit & 0x3f; | |
3566b778 | 843 | |
ee6c2868 | 844 | amd_iommu_dev_table[devid].data[i] |= (1UL << _bit); |
3566b778 JR |
845 | } |
846 | ||
c5cca146 JR |
847 | static int get_dev_entry_bit(u16 devid, u8 bit) |
848 | { | |
ee6c2868 JR |
849 | int i = (bit >> 6) & 0x03; |
850 | int _bit = bit & 0x3f; | |
c5cca146 | 851 | |
ee6c2868 | 852 | return (amd_iommu_dev_table[devid].data[i] & (1UL << _bit)) >> _bit; |
c5cca146 JR |
853 | } |
854 | ||
855 | ||
45a01c42 BH |
856 | static bool copy_device_table(void) |
857 | { | |
ae162efb | 858 | u64 int_ctl, int_tab_len, entry = 0, last_entry = 0; |
45a01c42 BH |
859 | struct dev_table_entry *old_devtb = NULL; |
860 | u32 lo, hi, devid, old_devtb_size; | |
861 | phys_addr_t old_devtb_phys; | |
45a01c42 | 862 | struct amd_iommu *iommu; |
53019a9e | 863 | u16 dom_id, dte_v, irq_v; |
45a01c42 | 864 | gfp_t gfp_flag; |
daae2d25 | 865 | u64 tmp; |
45a01c42 | 866 | |
3ac3e5ee BH |
867 | if (!amd_iommu_pre_enabled) |
868 | return false; | |
45a01c42 BH |
869 | |
870 | pr_warn("Translation is already enabled - trying to copy translation structures\n"); | |
871 | for_each_iommu(iommu) { | |
872 | /* All IOMMUs should use the same device table with the same size */ | |
873 | lo = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET); | |
874 | hi = readl(iommu->mmio_base + MMIO_DEV_TABLE_OFFSET + 4); | |
875 | entry = (((u64) hi) << 32) + lo; | |
876 | if (last_entry && last_entry != entry) { | |
3c6bae62 | 877 | pr_err("IOMMU:%d should use the same dev table as others!\n", |
45a01c42 BH |
878 | iommu->index); |
879 | return false; | |
880 | } | |
881 | last_entry = entry; | |
882 | ||
883 | old_devtb_size = ((entry & ~PAGE_MASK) + 1) << 12; | |
884 | if (old_devtb_size != dev_table_size) { | |
3c6bae62 | 885 | pr_err("The device table size of IOMMU:%d is not expected!\n", |
45a01c42 BH |
886 | iommu->index); |
887 | return false; | |
888 | } | |
889 | } | |
890 | ||
8780158c LJ |
891 | /* |
892 | * When SME is enabled in the first kernel, the entry includes the | |
893 | * memory encryption mask(sme_me_mask), we must remove the memory | |
894 | * encryption mask to obtain the true physical address in kdump kernel. | |
895 | */ | |
896 | old_devtb_phys = __sme_clr(entry) & PAGE_MASK; | |
897 | ||
b336781b | 898 | if (old_devtb_phys >= 0x100000000ULL) { |
3c6bae62 | 899 | pr_err("The address of old device table is above 4G, not trustworthy!\n"); |
b336781b BH |
900 | return false; |
901 | } | |
8780158c LJ |
902 | old_devtb = (sme_active() && is_kdump_kernel()) |
903 | ? (__force void *)ioremap_encrypted(old_devtb_phys, | |
904 | dev_table_size) | |
905 | : memremap(old_devtb_phys, dev_table_size, MEMREMAP_WB); | |
906 | ||
45a01c42 BH |
907 | if (!old_devtb) |
908 | return false; | |
909 | ||
b336781b | 910 | gfp_flag = GFP_KERNEL | __GFP_ZERO | GFP_DMA32; |
45a01c42 BH |
911 | old_dev_tbl_cpy = (void *)__get_free_pages(gfp_flag, |
912 | get_order(dev_table_size)); | |
913 | if (old_dev_tbl_cpy == NULL) { | |
3c6bae62 | 914 | pr_err("Failed to allocate memory for copying old device table!\n"); |
45a01c42 BH |
915 | return false; |
916 | } | |
917 | ||
918 | for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { | |
919 | old_dev_tbl_cpy[devid] = old_devtb[devid]; | |
920 | dom_id = old_devtb[devid].data[1] & DEV_DOMID_MASK; | |
921 | dte_v = old_devtb[devid].data[0] & DTE_FLAG_V; | |
53019a9e BH |
922 | |
923 | if (dte_v && dom_id) { | |
924 | old_dev_tbl_cpy[devid].data[0] = old_devtb[devid].data[0]; | |
925 | old_dev_tbl_cpy[devid].data[1] = old_devtb[devid].data[1]; | |
45a01c42 | 926 | __set_bit(dom_id, amd_iommu_pd_alloc_bitmap); |
daae2d25 BH |
927 | /* If gcr3 table existed, mask it out */ |
928 | if (old_devtb[devid].data[0] & DTE_FLAG_GV) { | |
929 | tmp = DTE_GCR3_VAL_B(~0ULL) << DTE_GCR3_SHIFT_B; | |
930 | tmp |= DTE_GCR3_VAL_C(~0ULL) << DTE_GCR3_SHIFT_C; | |
931 | old_dev_tbl_cpy[devid].data[1] &= ~tmp; | |
932 | tmp = DTE_GCR3_VAL_A(~0ULL) << DTE_GCR3_SHIFT_A; | |
933 | tmp |= DTE_FLAG_GV; | |
934 | old_dev_tbl_cpy[devid].data[0] &= ~tmp; | |
935 | } | |
53019a9e BH |
936 | } |
937 | ||
938 | irq_v = old_devtb[devid].data[2] & DTE_IRQ_REMAP_ENABLE; | |
939 | int_ctl = old_devtb[devid].data[2] & DTE_IRQ_REMAP_INTCTL_MASK; | |
940 | int_tab_len = old_devtb[devid].data[2] & DTE_IRQ_TABLE_LEN_MASK; | |
941 | if (irq_v && (int_ctl || int_tab_len)) { | |
942 | if ((int_ctl != DTE_IRQ_REMAP_INTCTL) || | |
943 | (int_tab_len != DTE_IRQ_TABLE_LEN)) { | |
944 | pr_err("Wrong old irq remapping flag: %#x\n", devid); | |
945 | return false; | |
946 | } | |
947 | ||
948 | old_dev_tbl_cpy[devid].data[2] = old_devtb[devid].data[2]; | |
949 | } | |
45a01c42 BH |
950 | } |
951 | memunmap(old_devtb); | |
952 | ||
953 | return true; | |
954 | } | |
955 | ||
c5cca146 JR |
956 | void amd_iommu_apply_erratum_63(u16 devid) |
957 | { | |
958 | int sysmgt; | |
959 | ||
960 | sysmgt = get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1) | | |
961 | (get_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2) << 1); | |
962 | ||
963 | if (sysmgt == 0x01) | |
964 | set_dev_entry_bit(devid, DEV_ENTRY_IW); | |
965 | } | |
966 | ||
5ff4789d JR |
967 | /* Writes the specific IOMMU for a device into the rlookup table */ |
968 | static void __init set_iommu_for_device(struct amd_iommu *iommu, u16 devid) | |
969 | { | |
970 | amd_iommu_rlookup_table[devid] = iommu; | |
971 | } | |
972 | ||
b65233a9 JR |
973 | /* |
974 | * This function takes the device specific flags read from the ACPI | |
975 | * table and sets up the device table entry with that information | |
976 | */ | |
5ff4789d JR |
977 | static void __init set_dev_entry_from_acpi(struct amd_iommu *iommu, |
978 | u16 devid, u32 flags, u32 ext_flags) | |
3566b778 JR |
979 | { |
980 | if (flags & ACPI_DEVFLAG_INITPASS) | |
981 | set_dev_entry_bit(devid, DEV_ENTRY_INIT_PASS); | |
982 | if (flags & ACPI_DEVFLAG_EXTINT) | |
983 | set_dev_entry_bit(devid, DEV_ENTRY_EINT_PASS); | |
984 | if (flags & ACPI_DEVFLAG_NMI) | |
985 | set_dev_entry_bit(devid, DEV_ENTRY_NMI_PASS); | |
986 | if (flags & ACPI_DEVFLAG_SYSMGT1) | |
987 | set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT1); | |
988 | if (flags & ACPI_DEVFLAG_SYSMGT2) | |
989 | set_dev_entry_bit(devid, DEV_ENTRY_SYSMGT2); | |
990 | if (flags & ACPI_DEVFLAG_LINT0) | |
991 | set_dev_entry_bit(devid, DEV_ENTRY_LINT0_PASS); | |
992 | if (flags & ACPI_DEVFLAG_LINT1) | |
993 | set_dev_entry_bit(devid, DEV_ENTRY_LINT1_PASS); | |
3566b778 | 994 | |
c5cca146 JR |
995 | amd_iommu_apply_erratum_63(devid); |
996 | ||
5ff4789d | 997 | set_iommu_for_device(iommu, devid); |
3566b778 JR |
998 | } |
999 | ||
c50e3247 | 1000 | static int __init add_special_device(u8 type, u8 id, u16 *devid, bool cmd_line) |
6efed63b JR |
1001 | { |
1002 | struct devid_map *entry; | |
1003 | struct list_head *list; | |
1004 | ||
31cff67f JR |
1005 | if (type == IVHD_SPECIAL_IOAPIC) |
1006 | list = &ioapic_map; | |
1007 | else if (type == IVHD_SPECIAL_HPET) | |
1008 | list = &hpet_map; | |
1009 | else | |
6efed63b JR |
1010 | return -EINVAL; |
1011 | ||
31cff67f JR |
1012 | list_for_each_entry(entry, list, list) { |
1013 | if (!(entry->id == id && entry->cmd_line)) | |
1014 | continue; | |
1015 | ||
101fa037 | 1016 | pr_info("Command-line override present for %s id %d - ignoring\n", |
31cff67f JR |
1017 | type == IVHD_SPECIAL_IOAPIC ? "IOAPIC" : "HPET", id); |
1018 | ||
c50e3247 JR |
1019 | *devid = entry->devid; |
1020 | ||
31cff67f JR |
1021 | return 0; |
1022 | } | |
1023 | ||
6efed63b JR |
1024 | entry = kzalloc(sizeof(*entry), GFP_KERNEL); |
1025 | if (!entry) | |
1026 | return -ENOMEM; | |
1027 | ||
31cff67f | 1028 | entry->id = id; |
c50e3247 | 1029 | entry->devid = *devid; |
31cff67f | 1030 | entry->cmd_line = cmd_line; |
6efed63b JR |
1031 | |
1032 | list_add_tail(&entry->list, list); | |
1033 | ||
1034 | return 0; | |
1035 | } | |
1036 | ||
2a0cb4e2 WZ |
1037 | static int __init add_acpi_hid_device(u8 *hid, u8 *uid, u16 *devid, |
1038 | bool cmd_line) | |
1039 | { | |
1040 | struct acpihid_map_entry *entry; | |
1041 | struct list_head *list = &acpihid_map; | |
1042 | ||
1043 | list_for_each_entry(entry, list, list) { | |
1044 | if (strcmp(entry->hid, hid) || | |
1045 | (*uid && *entry->uid && strcmp(entry->uid, uid)) || | |
1046 | !entry->cmd_line) | |
1047 | continue; | |
1048 | ||
101fa037 | 1049 | pr_info("Command-line override for hid:%s uid:%s\n", |
2a0cb4e2 WZ |
1050 | hid, uid); |
1051 | *devid = entry->devid; | |
1052 | return 0; | |
1053 | } | |
1054 | ||
1055 | entry = kzalloc(sizeof(*entry), GFP_KERNEL); | |
1056 | if (!entry) | |
1057 | return -ENOMEM; | |
1058 | ||
1059 | memcpy(entry->uid, uid, strlen(uid)); | |
1060 | memcpy(entry->hid, hid, strlen(hid)); | |
1061 | entry->devid = *devid; | |
1062 | entry->cmd_line = cmd_line; | |
1063 | entry->root_devid = (entry->devid & (~0x7)); | |
1064 | ||
101fa037 | 1065 | pr_info("%s, add hid:%s, uid:%s, rdevid:%d\n", |
2a0cb4e2 WZ |
1066 | entry->cmd_line ? "cmd" : "ivrs", |
1067 | entry->hid, entry->uid, entry->root_devid); | |
1068 | ||
1069 | list_add_tail(&entry->list, list); | |
1070 | return 0; | |
1071 | } | |
1072 | ||
235dacbc JR |
1073 | static int __init add_early_maps(void) |
1074 | { | |
1075 | int i, ret; | |
1076 | ||
1077 | for (i = 0; i < early_ioapic_map_size; ++i) { | |
1078 | ret = add_special_device(IVHD_SPECIAL_IOAPIC, | |
1079 | early_ioapic_map[i].id, | |
c50e3247 | 1080 | &early_ioapic_map[i].devid, |
235dacbc JR |
1081 | early_ioapic_map[i].cmd_line); |
1082 | if (ret) | |
1083 | return ret; | |
1084 | } | |
1085 | ||
1086 | for (i = 0; i < early_hpet_map_size; ++i) { | |
1087 | ret = add_special_device(IVHD_SPECIAL_HPET, | |
1088 | early_hpet_map[i].id, | |
c50e3247 | 1089 | &early_hpet_map[i].devid, |
235dacbc JR |
1090 | early_hpet_map[i].cmd_line); |
1091 | if (ret) | |
1092 | return ret; | |
1093 | } | |
1094 | ||
2a0cb4e2 WZ |
1095 | for (i = 0; i < early_acpihid_map_size; ++i) { |
1096 | ret = add_acpi_hid_device(early_acpihid_map[i].hid, | |
1097 | early_acpihid_map[i].uid, | |
1098 | &early_acpihid_map[i].devid, | |
1099 | early_acpihid_map[i].cmd_line); | |
1100 | if (ret) | |
1101 | return ret; | |
1102 | } | |
1103 | ||
235dacbc JR |
1104 | return 0; |
1105 | } | |
1106 | ||
b65233a9 | 1107 | /* |
df805abb | 1108 | * Reads the device exclusion range from ACPI and initializes the IOMMU with |
b65233a9 JR |
1109 | * it |
1110 | */ | |
3566b778 JR |
1111 | static void __init set_device_exclusion_range(u16 devid, struct ivmd_header *m) |
1112 | { | |
1113 | struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; | |
1114 | ||
1115 | if (!(m->flags & IVMD_FLAG_EXCL_RANGE)) | |
1116 | return; | |
1117 | ||
1118 | if (iommu) { | |
b65233a9 JR |
1119 | /* |
1120 | * We only can configure exclusion ranges per IOMMU, not | |
1121 | * per device. But we can enable the exclusion range per | |
1122 | * device. This is done here | |
1123 | */ | |
2c16c9fd | 1124 | set_dev_entry_bit(devid, DEV_ENTRY_EX); |
3566b778 JR |
1125 | iommu->exclusion_start = m->range_start; |
1126 | iommu->exclusion_length = m->range_length; | |
1127 | } | |
1128 | } | |
1129 | ||
b65233a9 JR |
1130 | /* |
1131 | * Takes a pointer to an AMD IOMMU entry in the ACPI table and | |
1132 | * initializes the hardware and our data structures with it. | |
1133 | */ | |
6efed63b | 1134 | static int __init init_iommu_from_acpi(struct amd_iommu *iommu, |
5d0c8e49 JR |
1135 | struct ivhd_header *h) |
1136 | { | |
1137 | u8 *p = (u8 *)h; | |
1138 | u8 *end = p, flags = 0; | |
0de66d5b JR |
1139 | u16 devid = 0, devid_start = 0, devid_to = 0; |
1140 | u32 dev_i, ext_flags = 0; | |
58a3bee5 | 1141 | bool alias = false; |
5d0c8e49 | 1142 | struct ivhd_entry *e; |
ac7ccf67 | 1143 | u32 ivhd_size; |
235dacbc JR |
1144 | int ret; |
1145 | ||
1146 | ||
1147 | ret = add_early_maps(); | |
1148 | if (ret) | |
1149 | return ret; | |
5d0c8e49 JR |
1150 | |
1151 | /* | |
e9bf5197 | 1152 | * First save the recommended feature enable bits from ACPI |
5d0c8e49 | 1153 | */ |
e9bf5197 | 1154 | iommu->acpi_flags = h->flags; |
5d0c8e49 JR |
1155 | |
1156 | /* | |
1157 | * Done. Now parse the device entries | |
1158 | */ | |
ac7ccf67 SS |
1159 | ivhd_size = get_ivhd_header_size(h); |
1160 | if (!ivhd_size) { | |
101fa037 | 1161 | pr_err("Unsupported IVHD type %#x\n", h->type); |
ac7ccf67 SS |
1162 | return -EINVAL; |
1163 | } | |
1164 | ||
1165 | p += ivhd_size; | |
1166 | ||
5d0c8e49 JR |
1167 | end += h->length; |
1168 | ||
42a698f4 | 1169 | |
5d0c8e49 JR |
1170 | while (p < end) { |
1171 | e = (struct ivhd_entry *)p; | |
1172 | switch (e->type) { | |
1173 | case IVHD_DEV_ALL: | |
42a698f4 | 1174 | |
226e889b | 1175 | DUMP_printk(" DEV_ALL\t\t\tflags: %02x\n", e->flags); |
42a698f4 | 1176 | |
226e889b JR |
1177 | for (dev_i = 0; dev_i <= amd_iommu_last_bdf; ++dev_i) |
1178 | set_dev_entry_from_acpi(iommu, dev_i, e->flags, 0); | |
5d0c8e49 JR |
1179 | break; |
1180 | case IVHD_DEV_SELECT: | |
42a698f4 JR |
1181 | |
1182 | DUMP_printk(" DEV_SELECT\t\t\t devid: %02x:%02x.%x " | |
1183 | "flags: %02x\n", | |
c5081cd7 | 1184 | PCI_BUS_NUM(e->devid), |
42a698f4 JR |
1185 | PCI_SLOT(e->devid), |
1186 | PCI_FUNC(e->devid), | |
1187 | e->flags); | |
1188 | ||
5d0c8e49 | 1189 | devid = e->devid; |
5ff4789d | 1190 | set_dev_entry_from_acpi(iommu, devid, e->flags, 0); |
5d0c8e49 JR |
1191 | break; |
1192 | case IVHD_DEV_SELECT_RANGE_START: | |
42a698f4 JR |
1193 | |
1194 | DUMP_printk(" DEV_SELECT_RANGE_START\t " | |
1195 | "devid: %02x:%02x.%x flags: %02x\n", | |
c5081cd7 | 1196 | PCI_BUS_NUM(e->devid), |
42a698f4 JR |
1197 | PCI_SLOT(e->devid), |
1198 | PCI_FUNC(e->devid), | |
1199 | e->flags); | |
1200 | ||
5d0c8e49 JR |
1201 | devid_start = e->devid; |
1202 | flags = e->flags; | |
1203 | ext_flags = 0; | |
58a3bee5 | 1204 | alias = false; |
5d0c8e49 JR |
1205 | break; |
1206 | case IVHD_DEV_ALIAS: | |
42a698f4 JR |
1207 | |
1208 | DUMP_printk(" DEV_ALIAS\t\t\t devid: %02x:%02x.%x " | |
1209 | "flags: %02x devid_to: %02x:%02x.%x\n", | |
c5081cd7 | 1210 | PCI_BUS_NUM(e->devid), |
42a698f4 JR |
1211 | PCI_SLOT(e->devid), |
1212 | PCI_FUNC(e->devid), | |
1213 | e->flags, | |
c5081cd7 | 1214 | PCI_BUS_NUM(e->ext >> 8), |
42a698f4 JR |
1215 | PCI_SLOT(e->ext >> 8), |
1216 | PCI_FUNC(e->ext >> 8)); | |
1217 | ||
5d0c8e49 JR |
1218 | devid = e->devid; |
1219 | devid_to = e->ext >> 8; | |
7a6a3a08 | 1220 | set_dev_entry_from_acpi(iommu, devid , e->flags, 0); |
7455aab1 | 1221 | set_dev_entry_from_acpi(iommu, devid_to, e->flags, 0); |
5d0c8e49 JR |
1222 | amd_iommu_alias_table[devid] = devid_to; |
1223 | break; | |
1224 | case IVHD_DEV_ALIAS_RANGE: | |
42a698f4 JR |
1225 | |
1226 | DUMP_printk(" DEV_ALIAS_RANGE\t\t " | |
1227 | "devid: %02x:%02x.%x flags: %02x " | |
1228 | "devid_to: %02x:%02x.%x\n", | |
c5081cd7 | 1229 | PCI_BUS_NUM(e->devid), |
42a698f4 JR |
1230 | PCI_SLOT(e->devid), |
1231 | PCI_FUNC(e->devid), | |
1232 | e->flags, | |
c5081cd7 | 1233 | PCI_BUS_NUM(e->ext >> 8), |
42a698f4 JR |
1234 | PCI_SLOT(e->ext >> 8), |
1235 | PCI_FUNC(e->ext >> 8)); | |
1236 | ||
5d0c8e49 JR |
1237 | devid_start = e->devid; |
1238 | flags = e->flags; | |
1239 | devid_to = e->ext >> 8; | |
1240 | ext_flags = 0; | |
58a3bee5 | 1241 | alias = true; |
5d0c8e49 JR |
1242 | break; |
1243 | case IVHD_DEV_EXT_SELECT: | |
42a698f4 JR |
1244 | |
1245 | DUMP_printk(" DEV_EXT_SELECT\t\t devid: %02x:%02x.%x " | |
1246 | "flags: %02x ext: %08x\n", | |
c5081cd7 | 1247 | PCI_BUS_NUM(e->devid), |
42a698f4 JR |
1248 | PCI_SLOT(e->devid), |
1249 | PCI_FUNC(e->devid), | |
1250 | e->flags, e->ext); | |
1251 | ||
5d0c8e49 | 1252 | devid = e->devid; |
5ff4789d JR |
1253 | set_dev_entry_from_acpi(iommu, devid, e->flags, |
1254 | e->ext); | |
5d0c8e49 JR |
1255 | break; |
1256 | case IVHD_DEV_EXT_SELECT_RANGE: | |
42a698f4 JR |
1257 | |
1258 | DUMP_printk(" DEV_EXT_SELECT_RANGE\t devid: " | |
1259 | "%02x:%02x.%x flags: %02x ext: %08x\n", | |
c5081cd7 | 1260 | PCI_BUS_NUM(e->devid), |
42a698f4 JR |
1261 | PCI_SLOT(e->devid), |
1262 | PCI_FUNC(e->devid), | |
1263 | e->flags, e->ext); | |
1264 | ||
5d0c8e49 JR |
1265 | devid_start = e->devid; |
1266 | flags = e->flags; | |
1267 | ext_flags = e->ext; | |
58a3bee5 | 1268 | alias = false; |
5d0c8e49 JR |
1269 | break; |
1270 | case IVHD_DEV_RANGE_END: | |
42a698f4 JR |
1271 | |
1272 | DUMP_printk(" DEV_RANGE_END\t\t devid: %02x:%02x.%x\n", | |
c5081cd7 | 1273 | PCI_BUS_NUM(e->devid), |
42a698f4 JR |
1274 | PCI_SLOT(e->devid), |
1275 | PCI_FUNC(e->devid)); | |
1276 | ||
5d0c8e49 JR |
1277 | devid = e->devid; |
1278 | for (dev_i = devid_start; dev_i <= devid; ++dev_i) { | |
7a6a3a08 | 1279 | if (alias) { |
5d0c8e49 | 1280 | amd_iommu_alias_table[dev_i] = devid_to; |
7a6a3a08 JR |
1281 | set_dev_entry_from_acpi(iommu, |
1282 | devid_to, flags, ext_flags); | |
1283 | } | |
1284 | set_dev_entry_from_acpi(iommu, dev_i, | |
1285 | flags, ext_flags); | |
5d0c8e49 JR |
1286 | } |
1287 | break; | |
6efed63b JR |
1288 | case IVHD_DEV_SPECIAL: { |
1289 | u8 handle, type; | |
1290 | const char *var; | |
1291 | u16 devid; | |
1292 | int ret; | |
1293 | ||
1294 | handle = e->ext & 0xff; | |
1295 | devid = (e->ext >> 8) & 0xffff; | |
1296 | type = (e->ext >> 24) & 0xff; | |
1297 | ||
1298 | if (type == IVHD_SPECIAL_IOAPIC) | |
1299 | var = "IOAPIC"; | |
1300 | else if (type == IVHD_SPECIAL_HPET) | |
1301 | var = "HPET"; | |
1302 | else | |
1303 | var = "UNKNOWN"; | |
1304 | ||
1305 | DUMP_printk(" DEV_SPECIAL(%s[%d])\t\tdevid: %02x:%02x.%x\n", | |
1306 | var, (int)handle, | |
c5081cd7 | 1307 | PCI_BUS_NUM(devid), |
6efed63b JR |
1308 | PCI_SLOT(devid), |
1309 | PCI_FUNC(devid)); | |
1310 | ||
c50e3247 | 1311 | ret = add_special_device(type, handle, &devid, false); |
6efed63b JR |
1312 | if (ret) |
1313 | return ret; | |
c50e3247 JR |
1314 | |
1315 | /* | |
1316 | * add_special_device might update the devid in case a | |
1317 | * command-line override is present. So call | |
1318 | * set_dev_entry_from_acpi after add_special_device. | |
1319 | */ | |
1320 | set_dev_entry_from_acpi(iommu, devid, e->flags, 0); | |
1321 | ||
6efed63b JR |
1322 | break; |
1323 | } | |
2a0cb4e2 WZ |
1324 | case IVHD_DEV_ACPI_HID: { |
1325 | u16 devid; | |
1326 | u8 hid[ACPIHID_HID_LEN] = {0}; | |
1327 | u8 uid[ACPIHID_UID_LEN] = {0}; | |
1328 | int ret; | |
1329 | ||
1330 | if (h->type != 0x40) { | |
1331 | pr_err(FW_BUG "Invalid IVHD device type %#x\n", | |
1332 | e->type); | |
1333 | break; | |
1334 | } | |
1335 | ||
1336 | memcpy(hid, (u8 *)(&e->ext), ACPIHID_HID_LEN - 1); | |
1337 | hid[ACPIHID_HID_LEN - 1] = '\0'; | |
1338 | ||
1339 | if (!(*hid)) { | |
1340 | pr_err(FW_BUG "Invalid HID.\n"); | |
1341 | break; | |
1342 | } | |
1343 | ||
1344 | switch (e->uidf) { | |
1345 | case UID_NOT_PRESENT: | |
1346 | ||
1347 | if (e->uidl != 0) | |
1348 | pr_warn(FW_BUG "Invalid UID length.\n"); | |
1349 | ||
1350 | break; | |
1351 | case UID_IS_INTEGER: | |
1352 | ||
1353 | sprintf(uid, "%d", e->uid); | |
1354 | ||
1355 | break; | |
1356 | case UID_IS_CHARACTER: | |
1357 | ||
1358 | memcpy(uid, (u8 *)(&e->uid), ACPIHID_UID_LEN - 1); | |
1359 | uid[ACPIHID_UID_LEN - 1] = '\0'; | |
1360 | ||
1361 | break; | |
1362 | default: | |
1363 | break; | |
1364 | } | |
1365 | ||
6082ee72 | 1366 | devid = e->devid; |
2a0cb4e2 WZ |
1367 | DUMP_printk(" DEV_ACPI_HID(%s[%s])\t\tdevid: %02x:%02x.%x\n", |
1368 | hid, uid, | |
1369 | PCI_BUS_NUM(devid), | |
1370 | PCI_SLOT(devid), | |
1371 | PCI_FUNC(devid)); | |
1372 | ||
2a0cb4e2 WZ |
1373 | flags = e->flags; |
1374 | ||
1375 | ret = add_acpi_hid_device(hid, uid, &devid, false); | |
1376 | if (ret) | |
1377 | return ret; | |
1378 | ||
1379 | /* | |
1380 | * add_special_device might update the devid in case a | |
1381 | * command-line override is present. So call | |
1382 | * set_dev_entry_from_acpi after add_special_device. | |
1383 | */ | |
1384 | set_dev_entry_from_acpi(iommu, devid, e->flags, 0); | |
1385 | ||
1386 | break; | |
1387 | } | |
5d0c8e49 JR |
1388 | default: |
1389 | break; | |
1390 | } | |
1391 | ||
b514e555 | 1392 | p += ivhd_entry_length(p); |
5d0c8e49 | 1393 | } |
6efed63b JR |
1394 | |
1395 | return 0; | |
5d0c8e49 JR |
1396 | } |
1397 | ||
e47d402d JR |
1398 | static void __init free_iommu_one(struct amd_iommu *iommu) |
1399 | { | |
1400 | free_command_buffer(iommu); | |
335503e5 | 1401 | free_event_buffer(iommu); |
1a29ac01 | 1402 | free_ppr_log(iommu); |
8bda0cfb | 1403 | free_ga_log(iommu); |
e47d402d JR |
1404 | iommu_unmap_mmio_space(iommu); |
1405 | } | |
1406 | ||
1407 | static void __init free_iommu_all(void) | |
1408 | { | |
1409 | struct amd_iommu *iommu, *next; | |
1410 | ||
3bd22172 | 1411 | for_each_iommu_safe(iommu, next) { |
e47d402d JR |
1412 | list_del(&iommu->list); |
1413 | free_iommu_one(iommu); | |
1414 | kfree(iommu); | |
1415 | } | |
1416 | } | |
1417 | ||
318fe782 SS |
1418 | /* |
1419 | * Family15h Model 10h-1fh erratum 746 (IOMMU Logging May Stall Translations) | |
1420 | * Workaround: | |
1421 | * BIOS should disable L2B micellaneous clock gating by setting | |
1422 | * L2_L2B_CK_GATE_CONTROL[CKGateL2BMiscDisable](D0F2xF4_x90[2]) = 1b | |
1423 | */ | |
e2f1a3bd | 1424 | static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu) |
318fe782 SS |
1425 | { |
1426 | u32 value; | |
1427 | ||
1428 | if ((boot_cpu_data.x86 != 0x15) || | |
1429 | (boot_cpu_data.x86_model < 0x10) || | |
1430 | (boot_cpu_data.x86_model > 0x1f)) | |
1431 | return; | |
1432 | ||
1433 | pci_write_config_dword(iommu->dev, 0xf0, 0x90); | |
1434 | pci_read_config_dword(iommu->dev, 0xf4, &value); | |
1435 | ||
1436 | if (value & BIT(2)) | |
1437 | return; | |
1438 | ||
1439 | /* Select NB indirect register 0x90 and enable writing */ | |
1440 | pci_write_config_dword(iommu->dev, 0xf0, 0x90 | (1 << 8)); | |
1441 | ||
1442 | pci_write_config_dword(iommu->dev, 0xf4, value | 0x4); | |
5f226da1 | 1443 | pci_info(iommu->dev, "Applying erratum 746 workaround\n"); |
318fe782 SS |
1444 | |
1445 | /* Clear the enable writing bit */ | |
1446 | pci_write_config_dword(iommu->dev, 0xf0, 0x90); | |
1447 | } | |
1448 | ||
358875fd JC |
1449 | /* |
1450 | * Family15h Model 30h-3fh (IOMMU Mishandles ATS Write Permission) | |
1451 | * Workaround: | |
1452 | * BIOS should enable ATS write permission check by setting | |
1453 | * L2_DEBUG_3[AtsIgnoreIWDis](D0F2xF4_x47[0]) = 1b | |
1454 | */ | |
1455 | static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu) | |
1456 | { | |
1457 | u32 value; | |
1458 | ||
1459 | if ((boot_cpu_data.x86 != 0x15) || | |
1460 | (boot_cpu_data.x86_model < 0x30) || | |
1461 | (boot_cpu_data.x86_model > 0x3f)) | |
1462 | return; | |
1463 | ||
1464 | /* Test L2_DEBUG_3[AtsIgnoreIWDis] == 1 */ | |
1465 | value = iommu_read_l2(iommu, 0x47); | |
1466 | ||
1467 | if (value & BIT(0)) | |
1468 | return; | |
1469 | ||
1470 | /* Set L2_DEBUG_3[AtsIgnoreIWDis] = 1 */ | |
1471 | iommu_write_l2(iommu, 0x47, value | BIT(0)); | |
1472 | ||
5f226da1 | 1473 | pci_info(iommu->dev, "Applying ATS write check workaround\n"); |
358875fd JC |
1474 | } |
1475 | ||
b65233a9 JR |
1476 | /* |
1477 | * This function clues the initialization function for one IOMMU | |
1478 | * together and also allocates the command buffer and programs the | |
1479 | * hardware. It does NOT enable the IOMMU. This is done afterwards. | |
1480 | */ | |
e47d402d JR |
1481 | static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h) |
1482 | { | |
6efed63b JR |
1483 | int ret; |
1484 | ||
27790398 | 1485 | raw_spin_lock_init(&iommu->lock); |
bb52777e JR |
1486 | |
1487 | /* Add IOMMU to internal data structures */ | |
e47d402d | 1488 | list_add_tail(&iommu->list, &amd_iommu_list); |
6b9376e3 | 1489 | iommu->index = amd_iommus_present++; |
bb52777e JR |
1490 | |
1491 | if (unlikely(iommu->index >= MAX_IOMMUS)) { | |
101fa037 | 1492 | WARN(1, "System has more IOMMUs than supported by this driver\n"); |
bb52777e JR |
1493 | return -ENOSYS; |
1494 | } | |
1495 | ||
1496 | /* Index is fine - add IOMMU to the array */ | |
1497 | amd_iommus[iommu->index] = iommu; | |
e47d402d JR |
1498 | |
1499 | /* | |
1500 | * Copy data from ACPI table entry to the iommu struct | |
1501 | */ | |
23c742db | 1502 | iommu->devid = h->devid; |
e47d402d | 1503 | iommu->cap_ptr = h->cap_ptr; |
ee893c24 | 1504 | iommu->pci_seg = h->pci_seg; |
e47d402d | 1505 | iommu->mmio_phys = h->mmio_phys; |
30861ddc | 1506 | |
7d7d38af SS |
1507 | switch (h->type) { |
1508 | case 0x10: | |
1509 | /* Check if IVHD EFR contains proper max banks/counters */ | |
1510 | if ((h->efr_attr != 0) && | |
1511 | ((h->efr_attr & (0xF << 13)) != 0) && | |
1512 | ((h->efr_attr & (0x3F << 17)) != 0)) | |
1513 | iommu->mmio_phys_end = MMIO_REG_END_OFFSET; | |
1514 | else | |
1515 | iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; | |
3928aa3f SS |
1516 | if (((h->efr_attr & (0x1 << IOMMU_FEAT_GASUP_SHIFT)) == 0)) |
1517 | amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; | |
90fcffd9 SS |
1518 | if (((h->efr_attr & (0x1 << IOMMU_FEAT_XTSUP_SHIFT)) == 0)) |
1519 | amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; | |
7d7d38af SS |
1520 | break; |
1521 | case 0x11: | |
1522 | case 0x40: | |
1523 | if (h->efr_reg & (1 << 9)) | |
1524 | iommu->mmio_phys_end = MMIO_REG_END_OFFSET; | |
1525 | else | |
1526 | iommu->mmio_phys_end = MMIO_CNTR_CONF_OFFSET; | |
3928aa3f SS |
1527 | if (((h->efr_reg & (0x1 << IOMMU_EFR_GASUP_SHIFT)) == 0)) |
1528 | amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; | |
90fcffd9 SS |
1529 | if (((h->efr_reg & (0x1 << IOMMU_EFR_XTSUP_SHIFT)) == 0)) |
1530 | amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; | |
7d7d38af SS |
1531 | break; |
1532 | default: | |
1533 | return -EINVAL; | |
30861ddc SK |
1534 | } |
1535 | ||
1536 | iommu->mmio_base = iommu_map_mmio_space(iommu->mmio_phys, | |
1537 | iommu->mmio_phys_end); | |
e47d402d JR |
1538 | if (!iommu->mmio_base) |
1539 | return -ENOMEM; | |
1540 | ||
f2c2db53 | 1541 | if (alloc_command_buffer(iommu)) |
e47d402d JR |
1542 | return -ENOMEM; |
1543 | ||
f2c2db53 | 1544 | if (alloc_event_buffer(iommu)) |
335503e5 JR |
1545 | return -ENOMEM; |
1546 | ||
a80dc3e0 JR |
1547 | iommu->int_enabled = false; |
1548 | ||
4c232a70 | 1549 | init_translation_status(iommu); |
3ac3e5ee BH |
1550 | if (translation_pre_enabled(iommu) && !is_kdump_kernel()) { |
1551 | iommu_disable(iommu); | |
1552 | clear_translation_pre_enabled(iommu); | |
1553 | pr_warn("Translation was enabled for IOMMU:%d but we are not in kdump mode\n", | |
1554 | iommu->index); | |
1555 | } | |
1556 | if (amd_iommu_pre_enabled) | |
1557 | amd_iommu_pre_enabled = translation_pre_enabled(iommu); | |
4c232a70 | 1558 | |
6efed63b JR |
1559 | ret = init_iommu_from_acpi(iommu, h); |
1560 | if (ret) | |
1561 | return ret; | |
f6fec00a | 1562 | |
7c71d306 JL |
1563 | ret = amd_iommu_create_irq_domain(iommu); |
1564 | if (ret) | |
1565 | return ret; | |
1566 | ||
f6fec00a JR |
1567 | /* |
1568 | * Make sure IOMMU is not considered to translate itself. The IVRS | |
1569 | * table tells us so, but this is a lie! | |
1570 | */ | |
1571 | amd_iommu_rlookup_table[iommu->devid] = NULL; | |
1572 | ||
23c742db | 1573 | return 0; |
e47d402d JR |
1574 | } |
1575 | ||
8c7142f5 SS |
1576 | /** |
1577 | * get_highest_supported_ivhd_type - Look up the appropriate IVHD type | |
1578 | * @ivrs Pointer to the IVRS header | |
1579 | * | |
1580 | * This function search through all IVDB of the maximum supported IVHD | |
1581 | */ | |
1582 | static u8 get_highest_supported_ivhd_type(struct acpi_table_header *ivrs) | |
1583 | { | |
1584 | u8 *base = (u8 *)ivrs; | |
1585 | struct ivhd_header *ivhd = (struct ivhd_header *) | |
1586 | (base + IVRS_HEADER_LENGTH); | |
1587 | u8 last_type = ivhd->type; | |
1588 | u16 devid = ivhd->devid; | |
1589 | ||
1590 | while (((u8 *)ivhd - base < ivrs->length) && | |
1591 | (ivhd->type <= ACPI_IVHD_TYPE_MAX_SUPPORTED)) { | |
1592 | u8 *p = (u8 *) ivhd; | |
1593 | ||
1594 | if (ivhd->devid == devid) | |
1595 | last_type = ivhd->type; | |
1596 | ivhd = (struct ivhd_header *)(p + ivhd->length); | |
1597 | } | |
1598 | ||
1599 | return last_type; | |
1600 | } | |
1601 | ||
b65233a9 JR |
1602 | /* |
1603 | * Iterates over all IOMMU entries in the ACPI table, allocates the | |
1604 | * IOMMU structure and initializes it with init_iommu_one() | |
1605 | */ | |
e47d402d JR |
1606 | static int __init init_iommu_all(struct acpi_table_header *table) |
1607 | { | |
1608 | u8 *p = (u8 *)table, *end = (u8 *)table; | |
1609 | struct ivhd_header *h; | |
1610 | struct amd_iommu *iommu; | |
1611 | int ret; | |
1612 | ||
e47d402d JR |
1613 | end += table->length; |
1614 | p += IVRS_HEADER_LENGTH; | |
1615 | ||
1616 | while (p < end) { | |
1617 | h = (struct ivhd_header *)p; | |
8c7142f5 | 1618 | if (*p == amd_iommu_target_ivhd_type) { |
9c72041f | 1619 | |
ae908c22 | 1620 | DUMP_printk("device: %02x:%02x.%01x cap: %04x " |
9c72041f | 1621 | "seg: %d flags: %01x info %04x\n", |
c5081cd7 | 1622 | PCI_BUS_NUM(h->devid), PCI_SLOT(h->devid), |
9c72041f JR |
1623 | PCI_FUNC(h->devid), h->cap_ptr, |
1624 | h->pci_seg, h->flags, h->info); | |
1625 | DUMP_printk(" mmio-addr: %016llx\n", | |
1626 | h->mmio_phys); | |
1627 | ||
e47d402d | 1628 | iommu = kzalloc(sizeof(struct amd_iommu), GFP_KERNEL); |
02f3b3f5 JR |
1629 | if (iommu == NULL) |
1630 | return -ENOMEM; | |
3551a708 | 1631 | |
e47d402d | 1632 | ret = init_iommu_one(iommu, h); |
02f3b3f5 JR |
1633 | if (ret) |
1634 | return ret; | |
e47d402d JR |
1635 | } |
1636 | p += h->length; | |
1637 | ||
1638 | } | |
1639 | WARN_ON(p != end); | |
1640 | ||
1641 | return 0; | |
1642 | } | |
1643 | ||
1650dfd1 SS |
1644 | static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, |
1645 | u8 fxn, u64 *value, bool is_write); | |
30861ddc SK |
1646 | |
1647 | static void init_iommu_perf_ctr(struct amd_iommu *iommu) | |
1648 | { | |
5f226da1 | 1649 | struct pci_dev *pdev = iommu->dev; |
30861ddc SK |
1650 | u64 val = 0xabcd, val2 = 0; |
1651 | ||
1652 | if (!iommu_feature(iommu, FEATURE_PC)) | |
1653 | return; | |
1654 | ||
1655 | amd_iommu_pc_present = true; | |
1656 | ||
1657 | /* Check if the performance counters can be written to */ | |
1650dfd1 SS |
1658 | if ((iommu_pc_get_set_reg(iommu, 0, 0, 0, &val, true)) || |
1659 | (iommu_pc_get_set_reg(iommu, 0, 0, 0, &val2, false)) || | |
30861ddc | 1660 | (val != val2)) { |
5f226da1 | 1661 | pci_err(pdev, "Unable to write to IOMMU perf counter.\n"); |
30861ddc SK |
1662 | amd_iommu_pc_present = false; |
1663 | return; | |
1664 | } | |
1665 | ||
5f226da1 | 1666 | pci_info(pdev, "IOMMU performance counters supported\n"); |
30861ddc SK |
1667 | |
1668 | val = readl(iommu->mmio_base + MMIO_CNTR_CONF_OFFSET); | |
1669 | iommu->max_banks = (u8) ((val >> 12) & 0x3f); | |
1670 | iommu->max_counters = (u8) ((val >> 7) & 0xf); | |
1671 | } | |
1672 | ||
066f2e98 AW |
1673 | static ssize_t amd_iommu_show_cap(struct device *dev, |
1674 | struct device_attribute *attr, | |
1675 | char *buf) | |
1676 | { | |
b7a42b9d | 1677 | struct amd_iommu *iommu = dev_to_amd_iommu(dev); |
066f2e98 AW |
1678 | return sprintf(buf, "%x\n", iommu->cap); |
1679 | } | |
1680 | static DEVICE_ATTR(cap, S_IRUGO, amd_iommu_show_cap, NULL); | |
1681 | ||
1682 | static ssize_t amd_iommu_show_features(struct device *dev, | |
1683 | struct device_attribute *attr, | |
1684 | char *buf) | |
1685 | { | |
b7a42b9d | 1686 | struct amd_iommu *iommu = dev_to_amd_iommu(dev); |
066f2e98 AW |
1687 | return sprintf(buf, "%llx\n", iommu->features); |
1688 | } | |
1689 | static DEVICE_ATTR(features, S_IRUGO, amd_iommu_show_features, NULL); | |
1690 | ||
1691 | static struct attribute *amd_iommu_attrs[] = { | |
1692 | &dev_attr_cap.attr, | |
1693 | &dev_attr_features.attr, | |
1694 | NULL, | |
1695 | }; | |
1696 | ||
1697 | static struct attribute_group amd_iommu_group = { | |
1698 | .name = "amd-iommu", | |
1699 | .attrs = amd_iommu_attrs, | |
1700 | }; | |
1701 | ||
1702 | static const struct attribute_group *amd_iommu_groups[] = { | |
1703 | &amd_iommu_group, | |
1704 | NULL, | |
1705 | }; | |
30861ddc | 1706 | |
24d2c521 | 1707 | static int __init iommu_init_pci(struct amd_iommu *iommu) |
23c742db JR |
1708 | { |
1709 | int cap_ptr = iommu->cap_ptr; | |
1710 | u32 range, misc, low, high; | |
8bda0cfb | 1711 | int ret; |
23c742db | 1712 | |
d5bf0f4f SK |
1713 | iommu->dev = pci_get_domain_bus_and_slot(0, PCI_BUS_NUM(iommu->devid), |
1714 | iommu->devid & 0xff); | |
23c742db JR |
1715 | if (!iommu->dev) |
1716 | return -ENODEV; | |
1717 | ||
cbbc00be JL |
1718 | /* Prevent binding other PCI device drivers to IOMMU devices */ |
1719 | iommu->dev->match_driver = false; | |
1720 | ||
23c742db JR |
1721 | pci_read_config_dword(iommu->dev, cap_ptr + MMIO_CAP_HDR_OFFSET, |
1722 | &iommu->cap); | |
1723 | pci_read_config_dword(iommu->dev, cap_ptr + MMIO_RANGE_OFFSET, | |
1724 | &range); | |
1725 | pci_read_config_dword(iommu->dev, cap_ptr + MMIO_MISC_OFFSET, | |
1726 | &misc); | |
1727 | ||
23c742db JR |
1728 | if (!(iommu->cap & (1 << IOMMU_CAP_IOTLB))) |
1729 | amd_iommu_iotlb_sup = false; | |
1730 | ||
1731 | /* read extended feature bits */ | |
1732 | low = readl(iommu->mmio_base + MMIO_EXT_FEATURES); | |
1733 | high = readl(iommu->mmio_base + MMIO_EXT_FEATURES + 4); | |
1734 | ||
1735 | iommu->features = ((u64)high << 32) | low; | |
1736 | ||
1737 | if (iommu_feature(iommu, FEATURE_GT)) { | |
1738 | int glxval; | |
a919a018 SS |
1739 | u32 max_pasid; |
1740 | u64 pasmax; | |
23c742db | 1741 | |
a919a018 SS |
1742 | pasmax = iommu->features & FEATURE_PASID_MASK; |
1743 | pasmax >>= FEATURE_PASID_SHIFT; | |
1744 | max_pasid = (1 << (pasmax + 1)) - 1; | |
23c742db | 1745 | |
a919a018 SS |
1746 | amd_iommu_max_pasid = min(amd_iommu_max_pasid, max_pasid); |
1747 | ||
1748 | BUG_ON(amd_iommu_max_pasid & ~PASID_MASK); | |
23c742db JR |
1749 | |
1750 | glxval = iommu->features & FEATURE_GLXVAL_MASK; | |
1751 | glxval >>= FEATURE_GLXVAL_SHIFT; | |
1752 | ||
1753 | if (amd_iommu_max_glx_val == -1) | |
1754 | amd_iommu_max_glx_val = glxval; | |
1755 | else | |
1756 | amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval); | |
1757 | } | |
1758 | ||
1759 | if (iommu_feature(iommu, FEATURE_GT) && | |
1760 | iommu_feature(iommu, FEATURE_PPR)) { | |
1761 | iommu->is_iommu_v2 = true; | |
1762 | amd_iommu_v2_present = true; | |
1763 | } | |
1764 | ||
f2c2db53 JR |
1765 | if (iommu_feature(iommu, FEATURE_PPR) && alloc_ppr_log(iommu)) |
1766 | return -ENOMEM; | |
23c742db | 1767 | |
8bda0cfb SS |
1768 | ret = iommu_init_ga(iommu); |
1769 | if (ret) | |
1770 | return ret; | |
3928aa3f | 1771 | |
23c742db JR |
1772 | if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) |
1773 | amd_iommu_np_cache = true; | |
1774 | ||
30861ddc SK |
1775 | init_iommu_perf_ctr(iommu); |
1776 | ||
23c742db JR |
1777 | if (is_rd890_iommu(iommu->dev)) { |
1778 | int i, j; | |
1779 | ||
d5bf0f4f SK |
1780 | iommu->root_pdev = |
1781 | pci_get_domain_bus_and_slot(0, iommu->dev->bus->number, | |
1782 | PCI_DEVFN(0, 0)); | |
23c742db JR |
1783 | |
1784 | /* | |
1785 | * Some rd890 systems may not be fully reconfigured by the | |
1786 | * BIOS, so it's necessary for us to store this information so | |
1787 | * it can be reprogrammed on resume | |
1788 | */ | |
1789 | pci_read_config_dword(iommu->dev, iommu->cap_ptr + 4, | |
1790 | &iommu->stored_addr_lo); | |
1791 | pci_read_config_dword(iommu->dev, iommu->cap_ptr + 8, | |
1792 | &iommu->stored_addr_hi); | |
1793 | ||
1794 | /* Low bit locks writes to configuration space */ | |
1795 | iommu->stored_addr_lo &= ~1; | |
1796 | ||
1797 | for (i = 0; i < 6; i++) | |
1798 | for (j = 0; j < 0x12; j++) | |
1799 | iommu->stored_l1[i][j] = iommu_read_l1(iommu, i, j); | |
1800 | ||
1801 | for (i = 0; i < 0x83; i++) | |
1802 | iommu->stored_l2[i] = iommu_read_l2(iommu, i); | |
1803 | } | |
1804 | ||
318fe782 | 1805 | amd_iommu_erratum_746_workaround(iommu); |
358875fd | 1806 | amd_iommu_ats_write_check_workaround(iommu); |
318fe782 | 1807 | |
39ab9555 JR |
1808 | iommu_device_sysfs_add(&iommu->iommu, &iommu->dev->dev, |
1809 | amd_iommu_groups, "ivhd%d", iommu->index); | |
b0119e87 JR |
1810 | iommu_device_set_ops(&iommu->iommu, &amd_iommu_ops); |
1811 | iommu_device_register(&iommu->iommu); | |
066f2e98 | 1812 | |
23c742db JR |
1813 | return pci_enable_device(iommu->dev); |
1814 | } | |
1815 | ||
4d121c32 JR |
1816 | static void print_iommu_info(void) |
1817 | { | |
1818 | static const char * const feat_str[] = { | |
1819 | "PreF", "PPR", "X2APIC", "NX", "GT", "[5]", | |
1820 | "IA", "GA", "HE", "PC" | |
1821 | }; | |
1822 | struct amd_iommu *iommu; | |
1823 | ||
1824 | for_each_iommu(iommu) { | |
5f226da1 | 1825 | struct pci_dev *pdev = iommu->dev; |
4d121c32 JR |
1826 | int i; |
1827 | ||
5f226da1 | 1828 | pci_info(pdev, "Found IOMMU cap 0x%hx\n", iommu->cap_ptr); |
4d121c32 JR |
1829 | |
1830 | if (iommu->cap & (1 << IOMMU_CAP_EFR)) { | |
5f226da1 BH |
1831 | pci_info(pdev, "Extended features (%#llx):\n", |
1832 | iommu->features); | |
2bd5ed00 | 1833 | for (i = 0; i < ARRAY_SIZE(feat_str); ++i) { |
4d121c32 JR |
1834 | if (iommu_feature(iommu, (1ULL << i))) |
1835 | pr_cont(" %s", feat_str[i]); | |
1836 | } | |
3928aa3f SS |
1837 | |
1838 | if (iommu->features & FEATURE_GAM_VAPIC) | |
1839 | pr_cont(" GA_vAPIC"); | |
1840 | ||
30861ddc | 1841 | pr_cont("\n"); |
500c25ed | 1842 | } |
4d121c32 | 1843 | } |
3928aa3f | 1844 | if (irq_remapping_enabled) { |
101fa037 | 1845 | pr_info("Interrupt remapping enabled\n"); |
3928aa3f | 1846 | if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) |
101fa037 | 1847 | pr_info("Virtual APIC enabled\n"); |
90fcffd9 | 1848 | if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) |
101fa037 | 1849 | pr_info("X2APIC enabled\n"); |
3928aa3f | 1850 | } |
4d121c32 JR |
1851 | } |
1852 | ||
2c0ae172 | 1853 | static int __init amd_iommu_init_pci(void) |
23c742db JR |
1854 | { |
1855 | struct amd_iommu *iommu; | |
1856 | int ret = 0; | |
1857 | ||
1858 | for_each_iommu(iommu) { | |
1859 | ret = iommu_init_pci(iommu); | |
1860 | if (ret) | |
1861 | break; | |
1862 | } | |
1863 | ||
522e5cb7 JR |
1864 | /* |
1865 | * Order is important here to make sure any unity map requirements are | |
1866 | * fulfilled. The unity mappings are created and written to the device | |
1867 | * table during the amd_iommu_init_api() call. | |
1868 | * | |
1869 | * After that we call init_device_table_dma() to make sure any | |
1870 | * uninitialized DTE will block DMA, and in the end we flush the caches | |
1871 | * of all IOMMUs to make sure the changes to the device table are | |
1872 | * active. | |
1873 | */ | |
1874 | ret = amd_iommu_init_api(); | |
1875 | ||
aafd8ba0 JR |
1876 | init_device_table_dma(); |
1877 | ||
1878 | for_each_iommu(iommu) | |
1879 | iommu_flush_all_caches(iommu); | |
1880 | ||
3a18404c JR |
1881 | if (!ret) |
1882 | print_iommu_info(); | |
4d121c32 | 1883 | |
23c742db JR |
1884 | return ret; |
1885 | } | |
1886 | ||
a80dc3e0 JR |
1887 | /**************************************************************************** |
1888 | * | |
1889 | * The following functions initialize the MSI interrupts for all IOMMUs | |
df805abb | 1890 | * in the system. It's a bit challenging because there could be multiple |
a80dc3e0 JR |
1891 | * IOMMUs per PCI BDF but we can call pci_enable_msi(x) only once per |
1892 | * pci_dev. | |
1893 | * | |
1894 | ****************************************************************************/ | |
1895 | ||
9f800de3 | 1896 | static int iommu_setup_msi(struct amd_iommu *iommu) |
a80dc3e0 JR |
1897 | { |
1898 | int r; | |
a80dc3e0 | 1899 | |
9ddd592a JR |
1900 | r = pci_enable_msi(iommu->dev); |
1901 | if (r) | |
1902 | return r; | |
a80dc3e0 | 1903 | |
72fe00f0 JR |
1904 | r = request_threaded_irq(iommu->dev->irq, |
1905 | amd_iommu_int_handler, | |
1906 | amd_iommu_int_thread, | |
1907 | 0, "AMD-Vi", | |
3f398bc7 | 1908 | iommu); |
a80dc3e0 JR |
1909 | |
1910 | if (r) { | |
1911 | pci_disable_msi(iommu->dev); | |
9ddd592a | 1912 | return r; |
a80dc3e0 JR |
1913 | } |
1914 | ||
fab6afa3 | 1915 | iommu->int_enabled = true; |
1a29ac01 | 1916 | |
a80dc3e0 JR |
1917 | return 0; |
1918 | } | |
1919 | ||
05f92db9 | 1920 | static int iommu_init_msi(struct amd_iommu *iommu) |
a80dc3e0 | 1921 | { |
9ddd592a JR |
1922 | int ret; |
1923 | ||
a80dc3e0 | 1924 | if (iommu->int_enabled) |
9ddd592a | 1925 | goto enable_faults; |
a80dc3e0 | 1926 | |
82fcfc67 | 1927 | if (iommu->dev->msi_cap) |
9ddd592a JR |
1928 | ret = iommu_setup_msi(iommu); |
1929 | else | |
1930 | ret = -ENODEV; | |
1931 | ||
1932 | if (ret) | |
1933 | return ret; | |
a80dc3e0 | 1934 | |
9ddd592a JR |
1935 | enable_faults: |
1936 | iommu_feature_enable(iommu, CONTROL_EVT_INT_EN); | |
a80dc3e0 | 1937 | |
9ddd592a JR |
1938 | if (iommu->ppr_log != NULL) |
1939 | iommu_feature_enable(iommu, CONTROL_PPFINT_EN); | |
1940 | ||
8bda0cfb SS |
1941 | iommu_ga_log_enable(iommu); |
1942 | ||
9ddd592a | 1943 | return 0; |
a80dc3e0 JR |
1944 | } |
1945 | ||
b65233a9 JR |
1946 | /**************************************************************************** |
1947 | * | |
1948 | * The next functions belong to the third pass of parsing the ACPI | |
1949 | * table. In this last pass the memory mapping requirements are | |
df805abb | 1950 | * gathered (like exclusion and unity mapping ranges). |
b65233a9 JR |
1951 | * |
1952 | ****************************************************************************/ | |
1953 | ||
be2a022c JR |
1954 | static void __init free_unity_maps(void) |
1955 | { | |
1956 | struct unity_map_entry *entry, *next; | |
1957 | ||
1958 | list_for_each_entry_safe(entry, next, &amd_iommu_unity_map, list) { | |
1959 | list_del(&entry->list); | |
1960 | kfree(entry); | |
1961 | } | |
1962 | } | |
1963 | ||
b65233a9 | 1964 | /* called when we find an exclusion range definition in ACPI */ |
be2a022c JR |
1965 | static int __init init_exclusion_range(struct ivmd_header *m) |
1966 | { | |
1967 | int i; | |
1968 | ||
1969 | switch (m->type) { | |
1970 | case ACPI_IVMD_TYPE: | |
1971 | set_device_exclusion_range(m->devid, m); | |
1972 | break; | |
1973 | case ACPI_IVMD_TYPE_ALL: | |
3a61ec38 | 1974 | for (i = 0; i <= amd_iommu_last_bdf; ++i) |
be2a022c JR |
1975 | set_device_exclusion_range(i, m); |
1976 | break; | |
1977 | case ACPI_IVMD_TYPE_RANGE: | |
1978 | for (i = m->devid; i <= m->aux; ++i) | |
1979 | set_device_exclusion_range(i, m); | |
1980 | break; | |
1981 | default: | |
1982 | break; | |
1983 | } | |
1984 | ||
1985 | return 0; | |
1986 | } | |
1987 | ||
b65233a9 | 1988 | /* called for unity map ACPI definition */ |
be2a022c JR |
1989 | static int __init init_unity_map_range(struct ivmd_header *m) |
1990 | { | |
98f1ad25 | 1991 | struct unity_map_entry *e = NULL; |
02acc43a | 1992 | char *s; |
be2a022c JR |
1993 | |
1994 | e = kzalloc(sizeof(*e), GFP_KERNEL); | |
1995 | if (e == NULL) | |
1996 | return -ENOMEM; | |
1997 | ||
8aafaaf2 JR |
1998 | if (m->flags & IVMD_FLAG_EXCL_RANGE) |
1999 | init_exclusion_range(m); | |
2000 | ||
be2a022c JR |
2001 | switch (m->type) { |
2002 | default: | |
0bc252f4 JR |
2003 | kfree(e); |
2004 | return 0; | |
be2a022c | 2005 | case ACPI_IVMD_TYPE: |
02acc43a | 2006 | s = "IVMD_TYPEi\t\t\t"; |
be2a022c JR |
2007 | e->devid_start = e->devid_end = m->devid; |
2008 | break; | |
2009 | case ACPI_IVMD_TYPE_ALL: | |
02acc43a | 2010 | s = "IVMD_TYPE_ALL\t\t"; |
be2a022c JR |
2011 | e->devid_start = 0; |
2012 | e->devid_end = amd_iommu_last_bdf; | |
2013 | break; | |
2014 | case ACPI_IVMD_TYPE_RANGE: | |
02acc43a | 2015 | s = "IVMD_TYPE_RANGE\t\t"; |
be2a022c JR |
2016 | e->devid_start = m->devid; |
2017 | e->devid_end = m->aux; | |
2018 | break; | |
2019 | } | |
2020 | e->address_start = PAGE_ALIGN(m->range_start); | |
2021 | e->address_end = e->address_start + PAGE_ALIGN(m->range_length); | |
2022 | e->prot = m->flags >> 1; | |
2023 | ||
02acc43a JR |
2024 | DUMP_printk("%s devid_start: %02x:%02x.%x devid_end: %02x:%02x.%x" |
2025 | " range_start: %016llx range_end: %016llx flags: %x\n", s, | |
c5081cd7 SK |
2026 | PCI_BUS_NUM(e->devid_start), PCI_SLOT(e->devid_start), |
2027 | PCI_FUNC(e->devid_start), PCI_BUS_NUM(e->devid_end), | |
02acc43a JR |
2028 | PCI_SLOT(e->devid_end), PCI_FUNC(e->devid_end), |
2029 | e->address_start, e->address_end, m->flags); | |
2030 | ||
be2a022c JR |
2031 | list_add_tail(&e->list, &amd_iommu_unity_map); |
2032 | ||
2033 | return 0; | |
2034 | } | |
2035 | ||
b65233a9 | 2036 | /* iterates over all memory definitions we find in the ACPI table */ |
be2a022c JR |
2037 | static int __init init_memory_definitions(struct acpi_table_header *table) |
2038 | { | |
2039 | u8 *p = (u8 *)table, *end = (u8 *)table; | |
2040 | struct ivmd_header *m; | |
2041 | ||
be2a022c JR |
2042 | end += table->length; |
2043 | p += IVRS_HEADER_LENGTH; | |
2044 | ||
2045 | while (p < end) { | |
2046 | m = (struct ivmd_header *)p; | |
8aafaaf2 | 2047 | if (m->flags & (IVMD_FLAG_UNITY_MAP | IVMD_FLAG_EXCL_RANGE)) |
be2a022c JR |
2048 | init_unity_map_range(m); |
2049 | ||
2050 | p += m->length; | |
2051 | } | |
2052 | ||
2053 | return 0; | |
2054 | } | |
2055 | ||
9f5f5fb3 | 2056 | /* |
3ac3e5ee | 2057 | * Init the device table to not allow DMA access for devices |
9f5f5fb3 | 2058 | */ |
33f28c59 | 2059 | static void init_device_table_dma(void) |
9f5f5fb3 | 2060 | { |
0de66d5b | 2061 | u32 devid; |
9f5f5fb3 JR |
2062 | |
2063 | for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { | |
2064 | set_dev_entry_bit(devid, DEV_ENTRY_VALID); | |
2065 | set_dev_entry_bit(devid, DEV_ENTRY_TRANSLATION); | |
9f5f5fb3 JR |
2066 | } |
2067 | } | |
2068 | ||
d04e0ba3 JR |
2069 | static void __init uninit_device_table_dma(void) |
2070 | { | |
2071 | u32 devid; | |
2072 | ||
2073 | for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) { | |
2074 | amd_iommu_dev_table[devid].data[0] = 0ULL; | |
2075 | amd_iommu_dev_table[devid].data[1] = 0ULL; | |
2076 | } | |
2077 | } | |
2078 | ||
33f28c59 JR |
2079 | static void init_device_table(void) |
2080 | { | |
2081 | u32 devid; | |
2082 | ||
2083 | if (!amd_iommu_irq_remap) | |
2084 | return; | |
2085 | ||
2086 | for (devid = 0; devid <= amd_iommu_last_bdf; ++devid) | |
2087 | set_dev_entry_bit(devid, DEV_ENTRY_IRQ_TBL_EN); | |
2088 | } | |
2089 | ||
e9bf5197 JR |
2090 | static void iommu_init_flags(struct amd_iommu *iommu) |
2091 | { | |
2092 | iommu->acpi_flags & IVHD_FLAG_HT_TUN_EN_MASK ? | |
2093 | iommu_feature_enable(iommu, CONTROL_HT_TUN_EN) : | |
2094 | iommu_feature_disable(iommu, CONTROL_HT_TUN_EN); | |
2095 | ||
2096 | iommu->acpi_flags & IVHD_FLAG_PASSPW_EN_MASK ? | |
2097 | iommu_feature_enable(iommu, CONTROL_PASSPW_EN) : | |
2098 | iommu_feature_disable(iommu, CONTROL_PASSPW_EN); | |
2099 | ||
2100 | iommu->acpi_flags & IVHD_FLAG_RESPASSPW_EN_MASK ? | |
2101 | iommu_feature_enable(iommu, CONTROL_RESPASSPW_EN) : | |
2102 | iommu_feature_disable(iommu, CONTROL_RESPASSPW_EN); | |
2103 | ||
2104 | iommu->acpi_flags & IVHD_FLAG_ISOC_EN_MASK ? | |
2105 | iommu_feature_enable(iommu, CONTROL_ISOC_EN) : | |
2106 | iommu_feature_disable(iommu, CONTROL_ISOC_EN); | |
2107 | ||
2108 | /* | |
2109 | * make IOMMU memory accesses cache coherent | |
2110 | */ | |
2111 | iommu_feature_enable(iommu, CONTROL_COHERENT_EN); | |
1456e9d2 JR |
2112 | |
2113 | /* Set IOTLB invalidation timeout to 1s */ | |
2114 | iommu_set_inv_tlb_timeout(iommu, CTRL_INV_TO_1S); | |
e9bf5197 JR |
2115 | } |
2116 | ||
5bcd757f | 2117 | static void iommu_apply_resume_quirks(struct amd_iommu *iommu) |
4c894f47 | 2118 | { |
5bcd757f MG |
2119 | int i, j; |
2120 | u32 ioc_feature_control; | |
c1bf94ec | 2121 | struct pci_dev *pdev = iommu->root_pdev; |
5bcd757f MG |
2122 | |
2123 | /* RD890 BIOSes may not have completely reconfigured the iommu */ | |
c1bf94ec | 2124 | if (!is_rd890_iommu(iommu->dev) || !pdev) |
5bcd757f MG |
2125 | return; |
2126 | ||
2127 | /* | |
2128 | * First, we need to ensure that the iommu is enabled. This is | |
2129 | * controlled by a register in the northbridge | |
2130 | */ | |
5bcd757f MG |
2131 | |
2132 | /* Select Northbridge indirect register 0x75 and enable writing */ | |
2133 | pci_write_config_dword(pdev, 0x60, 0x75 | (1 << 7)); | |
2134 | pci_read_config_dword(pdev, 0x64, &ioc_feature_control); | |
2135 | ||
2136 | /* Enable the iommu */ | |
2137 | if (!(ioc_feature_control & 0x1)) | |
2138 | pci_write_config_dword(pdev, 0x64, ioc_feature_control | 1); | |
2139 | ||
5bcd757f MG |
2140 | /* Restore the iommu BAR */ |
2141 | pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4, | |
2142 | iommu->stored_addr_lo); | |
2143 | pci_write_config_dword(iommu->dev, iommu->cap_ptr + 8, | |
2144 | iommu->stored_addr_hi); | |
2145 | ||
2146 | /* Restore the l1 indirect regs for each of the 6 l1s */ | |
2147 | for (i = 0; i < 6; i++) | |
2148 | for (j = 0; j < 0x12; j++) | |
2149 | iommu_write_l1(iommu, i, j, iommu->stored_l1[i][j]); | |
2150 | ||
2151 | /* Restore the l2 indirect regs */ | |
2152 | for (i = 0; i < 0x83; i++) | |
2153 | iommu_write_l2(iommu, i, iommu->stored_l2[i]); | |
2154 | ||
2155 | /* Lock PCI setup registers */ | |
2156 | pci_write_config_dword(iommu->dev, iommu->cap_ptr + 4, | |
2157 | iommu->stored_addr_lo | 1); | |
4c894f47 JR |
2158 | } |
2159 | ||
3928aa3f SS |
2160 | static void iommu_enable_ga(struct amd_iommu *iommu) |
2161 | { | |
2162 | #ifdef CONFIG_IRQ_REMAP | |
2163 | switch (amd_iommu_guest_ir) { | |
2164 | case AMD_IOMMU_GUEST_IR_VAPIC: | |
2165 | iommu_feature_enable(iommu, CONTROL_GAM_EN); | |
2166 | /* Fall through */ | |
2167 | case AMD_IOMMU_GUEST_IR_LEGACY_GA: | |
2168 | iommu_feature_enable(iommu, CONTROL_GA_EN); | |
77bdab46 | 2169 | iommu->irte_ops = &irte_128_ops; |
3928aa3f SS |
2170 | break; |
2171 | default: | |
77bdab46 | 2172 | iommu->irte_ops = &irte_32_ops; |
3928aa3f SS |
2173 | break; |
2174 | } | |
2175 | #endif | |
2176 | } | |
2177 | ||
78d313c6 BH |
2178 | static void early_enable_iommu(struct amd_iommu *iommu) |
2179 | { | |
2180 | iommu_disable(iommu); | |
2181 | iommu_init_flags(iommu); | |
2182 | iommu_set_device_table(iommu); | |
2183 | iommu_enable_command_buffer(iommu); | |
2184 | iommu_enable_event_buffer(iommu); | |
2185 | iommu_set_exclusion_range(iommu); | |
2186 | iommu_enable_ga(iommu); | |
90fcffd9 | 2187 | iommu_enable_xt(iommu); |
78d313c6 BH |
2188 | iommu_enable(iommu); |
2189 | iommu_flush_all_caches(iommu); | |
2190 | } | |
2191 | ||
b65233a9 JR |
2192 | /* |
2193 | * This function finally enables all IOMMUs found in the system after | |
3ac3e5ee BH |
2194 | * they have been initialized. |
2195 | * | |
2196 | * Or if in kdump kernel and IOMMUs are all pre-enabled, try to copy | |
2197 | * the old content of device table entries. Not this case or copy failed, | |
2198 | * just continue as normal kernel does. | |
b65233a9 | 2199 | */ |
11ee5ac4 | 2200 | static void early_enable_iommus(void) |
8736197b JR |
2201 | { |
2202 | struct amd_iommu *iommu; | |
2203 | ||
3ac3e5ee BH |
2204 | |
2205 | if (!copy_device_table()) { | |
2206 | /* | |
2207 | * If come here because of failure in copying device table from old | |
2208 | * kernel with all IOMMUs enabled, print error message and try to | |
2209 | * free allocated old_dev_tbl_cpy. | |
2210 | */ | |
2211 | if (amd_iommu_pre_enabled) | |
2212 | pr_err("Failed to copy DEV table from previous kernel.\n"); | |
2213 | if (old_dev_tbl_cpy != NULL) | |
2214 | free_pages((unsigned long)old_dev_tbl_cpy, | |
2215 | get_order(dev_table_size)); | |
2216 | ||
2217 | for_each_iommu(iommu) { | |
2218 | clear_translation_pre_enabled(iommu); | |
2219 | early_enable_iommu(iommu); | |
2220 | } | |
2221 | } else { | |
2222 | pr_info("Copied DEV table from previous kernel.\n"); | |
2223 | free_pages((unsigned long)amd_iommu_dev_table, | |
2224 | get_order(dev_table_size)); | |
2225 | amd_iommu_dev_table = old_dev_tbl_cpy; | |
2226 | for_each_iommu(iommu) { | |
2227 | iommu_disable_command_buffer(iommu); | |
2228 | iommu_disable_event_buffer(iommu); | |
2229 | iommu_enable_command_buffer(iommu); | |
2230 | iommu_enable_event_buffer(iommu); | |
2231 | iommu_enable_ga(iommu); | |
90fcffd9 | 2232 | iommu_enable_xt(iommu); |
3ac3e5ee BH |
2233 | iommu_set_device_table(iommu); |
2234 | iommu_flush_all_caches(iommu); | |
2235 | } | |
8736197b | 2236 | } |
d98de49a SS |
2237 | |
2238 | #ifdef CONFIG_IRQ_REMAP | |
2239 | if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) | |
2240 | amd_iommu_irq_ops.capability |= (1 << IRQ_POSTING_CAP); | |
2241 | #endif | |
8736197b JR |
2242 | } |
2243 | ||
11ee5ac4 JR |
2244 | static void enable_iommus_v2(void) |
2245 | { | |
2246 | struct amd_iommu *iommu; | |
2247 | ||
2248 | for_each_iommu(iommu) { | |
2249 | iommu_enable_ppr_log(iommu); | |
2250 | iommu_enable_gt(iommu); | |
2251 | } | |
2252 | } | |
2253 | ||
2254 | static void enable_iommus(void) | |
2255 | { | |
2256 | early_enable_iommus(); | |
2257 | ||
2258 | enable_iommus_v2(); | |
2259 | } | |
2260 | ||
92ac4320 JR |
2261 | static void disable_iommus(void) |
2262 | { | |
2263 | struct amd_iommu *iommu; | |
2264 | ||
2265 | for_each_iommu(iommu) | |
2266 | iommu_disable(iommu); | |
d98de49a SS |
2267 | |
2268 | #ifdef CONFIG_IRQ_REMAP | |
2269 | if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)) | |
2270 | amd_iommu_irq_ops.capability &= ~(1 << IRQ_POSTING_CAP); | |
2271 | #endif | |
92ac4320 JR |
2272 | } |
2273 | ||
7441e9cb JR |
2274 | /* |
2275 | * Suspend/Resume support | |
2276 | * disable suspend until real resume implemented | |
2277 | */ | |
2278 | ||
f3c6ea1b | 2279 | static void amd_iommu_resume(void) |
7441e9cb | 2280 | { |
5bcd757f MG |
2281 | struct amd_iommu *iommu; |
2282 | ||
2283 | for_each_iommu(iommu) | |
2284 | iommu_apply_resume_quirks(iommu); | |
2285 | ||
736501ee JR |
2286 | /* re-load the hardware */ |
2287 | enable_iommus(); | |
3d9761e7 JR |
2288 | |
2289 | amd_iommu_enable_interrupts(); | |
7441e9cb JR |
2290 | } |
2291 | ||
f3c6ea1b | 2292 | static int amd_iommu_suspend(void) |
7441e9cb | 2293 | { |
736501ee JR |
2294 | /* disable IOMMUs to go out of the way for BIOS */ |
2295 | disable_iommus(); | |
2296 | ||
2297 | return 0; | |
7441e9cb JR |
2298 | } |
2299 | ||
f3c6ea1b | 2300 | static struct syscore_ops amd_iommu_syscore_ops = { |
7441e9cb JR |
2301 | .suspend = amd_iommu_suspend, |
2302 | .resume = amd_iommu_resume, | |
2303 | }; | |
2304 | ||
90b3eb03 | 2305 | static void __init free_iommu_resources(void) |
8704a1ba | 2306 | { |
ebcfa284 | 2307 | kmemleak_free(irq_lookup_table); |
0ea2c422 JR |
2308 | free_pages((unsigned long)irq_lookup_table, |
2309 | get_order(rlookup_table_size)); | |
f6019271 | 2310 | irq_lookup_table = NULL; |
8704a1ba | 2311 | |
a591989a JL |
2312 | kmem_cache_destroy(amd_iommu_irq_cache); |
2313 | amd_iommu_irq_cache = NULL; | |
8704a1ba JR |
2314 | |
2315 | free_pages((unsigned long)amd_iommu_rlookup_table, | |
2316 | get_order(rlookup_table_size)); | |
f6019271 | 2317 | amd_iommu_rlookup_table = NULL; |
8704a1ba JR |
2318 | |
2319 | free_pages((unsigned long)amd_iommu_alias_table, | |
2320 | get_order(alias_table_size)); | |
f6019271 | 2321 | amd_iommu_alias_table = NULL; |
8704a1ba JR |
2322 | |
2323 | free_pages((unsigned long)amd_iommu_dev_table, | |
2324 | get_order(dev_table_size)); | |
f6019271 | 2325 | amd_iommu_dev_table = NULL; |
8704a1ba JR |
2326 | |
2327 | free_iommu_all(); | |
2328 | ||
8704a1ba JR |
2329 | #ifdef CONFIG_GART_IOMMU |
2330 | /* | |
2331 | * We failed to initialize the AMD IOMMU - try fallback to GART | |
2332 | * if possible. | |
2333 | */ | |
2334 | gart_iommu_init(); | |
2335 | ||
2336 | #endif | |
2337 | } | |
2338 | ||
c2ff5cf5 JR |
2339 | /* SB IOAPIC is always on this device in AMD systems */ |
2340 | #define IOAPIC_SB_DEVID ((0x00 << 8) | PCI_DEVFN(0x14, 0)) | |
2341 | ||
eb1eb7ae JR |
2342 | static bool __init check_ioapic_information(void) |
2343 | { | |
dfbb6d47 | 2344 | const char *fw_bug = FW_BUG; |
c2ff5cf5 | 2345 | bool ret, has_sb_ioapic; |
eb1eb7ae JR |
2346 | int idx; |
2347 | ||
c2ff5cf5 JR |
2348 | has_sb_ioapic = false; |
2349 | ret = false; | |
eb1eb7ae | 2350 | |
dfbb6d47 JR |
2351 | /* |
2352 | * If we have map overrides on the kernel command line the | |
2353 | * messages in this function might not describe firmware bugs | |
2354 | * anymore - so be careful | |
2355 | */ | |
2356 | if (cmdline_maps) | |
2357 | fw_bug = ""; | |
2358 | ||
c2ff5cf5 JR |
2359 | for (idx = 0; idx < nr_ioapics; idx++) { |
2360 | int devid, id = mpc_ioapic_id(idx); | |
2361 | ||
2362 | devid = get_ioapic_devid(id); | |
2363 | if (devid < 0) { | |
101fa037 | 2364 | pr_err("%s: IOAPIC[%d] not in IVRS table\n", |
dfbb6d47 | 2365 | fw_bug, id); |
c2ff5cf5 JR |
2366 | ret = false; |
2367 | } else if (devid == IOAPIC_SB_DEVID) { | |
2368 | has_sb_ioapic = true; | |
2369 | ret = true; | |
eb1eb7ae JR |
2370 | } |
2371 | } | |
2372 | ||
c2ff5cf5 JR |
2373 | if (!has_sb_ioapic) { |
2374 | /* | |
2375 | * We expect the SB IOAPIC to be listed in the IVRS | |
2376 | * table. The system timer is connected to the SB IOAPIC | |
2377 | * and if we don't have it in the list the system will | |
2378 | * panic at boot time. This situation usually happens | |
2379 | * when the BIOS is buggy and provides us the wrong | |
2380 | * device id for the IOAPIC in the system. | |
2381 | */ | |
101fa037 | 2382 | pr_err("%s: No southbridge IOAPIC found\n", fw_bug); |
c2ff5cf5 JR |
2383 | } |
2384 | ||
2385 | if (!ret) | |
101fa037 | 2386 | pr_err("Disabling interrupt remapping\n"); |
c2ff5cf5 JR |
2387 | |
2388 | return ret; | |
eb1eb7ae JR |
2389 | } |
2390 | ||
d04e0ba3 JR |
2391 | static void __init free_dma_resources(void) |
2392 | { | |
d04e0ba3 JR |
2393 | free_pages((unsigned long)amd_iommu_pd_alloc_bitmap, |
2394 | get_order(MAX_DOMAIN_ID/8)); | |
f6019271 | 2395 | amd_iommu_pd_alloc_bitmap = NULL; |
d04e0ba3 JR |
2396 | |
2397 | free_unity_maps(); | |
2398 | } | |
2399 | ||
b65233a9 | 2400 | /* |
8704a1ba JR |
2401 | * This is the hardware init function for AMD IOMMU in the system. |
2402 | * This function is called either from amd_iommu_init or from the interrupt | |
2403 | * remapping setup code. | |
b65233a9 JR |
2404 | * |
2405 | * This function basically parses the ACPI table for AMD IOMMU (IVRS) | |
8c7142f5 | 2406 | * four times: |
b65233a9 | 2407 | * |
8c7142f5 SS |
2408 | * 1 pass) Discover the most comprehensive IVHD type to use. |
2409 | * | |
2410 | * 2 pass) Find the highest PCI device id the driver has to handle. | |
b65233a9 JR |
2411 | * Upon this information the size of the data structures is |
2412 | * determined that needs to be allocated. | |
2413 | * | |
8c7142f5 | 2414 | * 3 pass) Initialize the data structures just allocated with the |
b65233a9 JR |
2415 | * information in the ACPI table about available AMD IOMMUs |
2416 | * in the system. It also maps the PCI devices in the | |
2417 | * system to specific IOMMUs | |
2418 | * | |
8c7142f5 | 2419 | * 4 pass) After the basic data structures are allocated and |
b65233a9 JR |
2420 | * initialized we update them with information about memory |
2421 | * remapping requirements parsed out of the ACPI table in | |
2422 | * this last pass. | |
2423 | * | |
8704a1ba JR |
2424 | * After everything is set up the IOMMUs are enabled and the necessary |
2425 | * hotplug and suspend notifiers are registered. | |
b65233a9 | 2426 | */ |
643511b3 | 2427 | static int __init early_amd_iommu_init(void) |
fe74c9cf | 2428 | { |
02f3b3f5 | 2429 | struct acpi_table_header *ivrs_base; |
02f3b3f5 | 2430 | acpi_status status; |
3928aa3f | 2431 | int i, remap_cache_sz, ret = 0; |
fe74c9cf | 2432 | |
643511b3 | 2433 | if (!amd_iommu_detected) |
8704a1ba JR |
2434 | return -ENODEV; |
2435 | ||
6b11d1d6 | 2436 | status = acpi_get_table("IVRS", 0, &ivrs_base); |
02f3b3f5 JR |
2437 | if (status == AE_NOT_FOUND) |
2438 | return -ENODEV; | |
2439 | else if (ACPI_FAILURE(status)) { | |
2440 | const char *err = acpi_format_exception(status); | |
101fa037 | 2441 | pr_err("IVRS table error: %s\n", err); |
02f3b3f5 JR |
2442 | return -EINVAL; |
2443 | } | |
2444 | ||
8c7142f5 SS |
2445 | /* |
2446 | * Validate checksum here so we don't need to do it when | |
2447 | * we actually parse the table | |
2448 | */ | |
2449 | ret = check_ivrs_checksum(ivrs_base); | |
2450 | if (ret) | |
99e8ccd3 | 2451 | goto out; |
8c7142f5 SS |
2452 | |
2453 | amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base); | |
2454 | DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type); | |
2455 | ||
fe74c9cf JR |
2456 | /* |
2457 | * First parse ACPI tables to find the largest Bus/Dev/Func | |
2458 | * we need to handle. Upon this information the shared data | |
2459 | * structures for the IOMMUs in the system will be allocated | |
2460 | */ | |
2c0ae172 JR |
2461 | ret = find_last_devid_acpi(ivrs_base); |
2462 | if (ret) | |
3551a708 JR |
2463 | goto out; |
2464 | ||
c571484e JR |
2465 | dev_table_size = tbl_size(DEV_TABLE_ENTRY_SIZE); |
2466 | alias_table_size = tbl_size(ALIAS_TABLE_ENTRY_SIZE); | |
2467 | rlookup_table_size = tbl_size(RLOOKUP_TABLE_ENTRY_SIZE); | |
fe74c9cf | 2468 | |
fe74c9cf | 2469 | /* Device table - directly used by all IOMMUs */ |
8704a1ba | 2470 | ret = -ENOMEM; |
b336781b BH |
2471 | amd_iommu_dev_table = (void *)__get_free_pages( |
2472 | GFP_KERNEL | __GFP_ZERO | GFP_DMA32, | |
fe74c9cf JR |
2473 | get_order(dev_table_size)); |
2474 | if (amd_iommu_dev_table == NULL) | |
2475 | goto out; | |
2476 | ||
2477 | /* | |
2478 | * Alias table - map PCI Bus/Dev/Func to Bus/Dev/Func the | |
2479 | * IOMMU see for that device | |
2480 | */ | |
2481 | amd_iommu_alias_table = (void *)__get_free_pages(GFP_KERNEL, | |
2482 | get_order(alias_table_size)); | |
2483 | if (amd_iommu_alias_table == NULL) | |
2c0ae172 | 2484 | goto out; |
fe74c9cf JR |
2485 | |
2486 | /* IOMMU rlookup table - find the IOMMU for a specific device */ | |
83fd5cc6 JR |
2487 | amd_iommu_rlookup_table = (void *)__get_free_pages( |
2488 | GFP_KERNEL | __GFP_ZERO, | |
fe74c9cf JR |
2489 | get_order(rlookup_table_size)); |
2490 | if (amd_iommu_rlookup_table == NULL) | |
2c0ae172 | 2491 | goto out; |
fe74c9cf | 2492 | |
5dc8bff0 JR |
2493 | amd_iommu_pd_alloc_bitmap = (void *)__get_free_pages( |
2494 | GFP_KERNEL | __GFP_ZERO, | |
fe74c9cf JR |
2495 | get_order(MAX_DOMAIN_ID/8)); |
2496 | if (amd_iommu_pd_alloc_bitmap == NULL) | |
2c0ae172 | 2497 | goto out; |
fe74c9cf JR |
2498 | |
2499 | /* | |
5dc8bff0 | 2500 | * let all alias entries point to itself |
fe74c9cf | 2501 | */ |
3a61ec38 | 2502 | for (i = 0; i <= amd_iommu_last_bdf; ++i) |
fe74c9cf JR |
2503 | amd_iommu_alias_table[i] = i; |
2504 | ||
fe74c9cf JR |
2505 | /* |
2506 | * never allocate domain 0 because its used as the non-allocated and | |
2507 | * error value placeholder | |
2508 | */ | |
5c87f62d | 2509 | __set_bit(0, amd_iommu_pd_alloc_bitmap); |
fe74c9cf JR |
2510 | |
2511 | /* | |
2512 | * now the data structures are allocated and basically initialized | |
2513 | * start the real acpi table scan | |
2514 | */ | |
02f3b3f5 JR |
2515 | ret = init_iommu_all(ivrs_base); |
2516 | if (ret) | |
2c0ae172 | 2517 | goto out; |
fe74c9cf | 2518 | |
11123741 | 2519 | /* Disable any previously enabled IOMMUs */ |
20b46dff BH |
2520 | if (!is_kdump_kernel() || amd_iommu_disabled) |
2521 | disable_iommus(); | |
11123741 | 2522 | |
eb1eb7ae JR |
2523 | if (amd_iommu_irq_remap) |
2524 | amd_iommu_irq_remap = check_ioapic_information(); | |
2525 | ||
05152a04 JR |
2526 | if (amd_iommu_irq_remap) { |
2527 | /* | |
2528 | * Interrupt remapping enabled, create kmem_cache for the | |
2529 | * remapping tables. | |
2530 | */ | |
83ed9c13 | 2531 | ret = -ENOMEM; |
3928aa3f SS |
2532 | if (!AMD_IOMMU_GUEST_IR_GA(amd_iommu_guest_ir)) |
2533 | remap_cache_sz = MAX_IRQS_PER_TABLE * sizeof(u32); | |
2534 | else | |
2535 | remap_cache_sz = MAX_IRQS_PER_TABLE * (sizeof(u64) * 2); | |
05152a04 | 2536 | amd_iommu_irq_cache = kmem_cache_create("irq_remap_cache", |
3928aa3f SS |
2537 | remap_cache_sz, |
2538 | IRQ_TABLE_ALIGNMENT, | |
2539 | 0, NULL); | |
05152a04 JR |
2540 | if (!amd_iommu_irq_cache) |
2541 | goto out; | |
0ea2c422 JR |
2542 | |
2543 | irq_lookup_table = (void *)__get_free_pages( | |
2544 | GFP_KERNEL | __GFP_ZERO, | |
2545 | get_order(rlookup_table_size)); | |
ebcfa284 LS |
2546 | kmemleak_alloc(irq_lookup_table, rlookup_table_size, |
2547 | 1, GFP_KERNEL); | |
0ea2c422 JR |
2548 | if (!irq_lookup_table) |
2549 | goto out; | |
05152a04 JR |
2550 | } |
2551 | ||
02f3b3f5 JR |
2552 | ret = init_memory_definitions(ivrs_base); |
2553 | if (ret) | |
2c0ae172 | 2554 | goto out; |
3551a708 | 2555 | |
eb1eb7ae JR |
2556 | /* init the device table */ |
2557 | init_device_table(); | |
2558 | ||
8704a1ba | 2559 | out: |
02f3b3f5 | 2560 | /* Don't leak any ACPI memory */ |
6b11d1d6 | 2561 | acpi_put_table(ivrs_base); |
02f3b3f5 JR |
2562 | ivrs_base = NULL; |
2563 | ||
643511b3 JR |
2564 | return ret; |
2565 | } | |
2566 | ||
ae295142 | 2567 | static int amd_iommu_enable_interrupts(void) |
3d9761e7 JR |
2568 | { |
2569 | struct amd_iommu *iommu; | |
2570 | int ret = 0; | |
2571 | ||
2572 | for_each_iommu(iommu) { | |
2573 | ret = iommu_init_msi(iommu); | |
2574 | if (ret) | |
2575 | goto out; | |
2576 | } | |
2577 | ||
2578 | out: | |
2579 | return ret; | |
2580 | } | |
2581 | ||
02f3b3f5 JR |
2582 | static bool detect_ivrs(void) |
2583 | { | |
2584 | struct acpi_table_header *ivrs_base; | |
02f3b3f5 JR |
2585 | acpi_status status; |
2586 | ||
6b11d1d6 | 2587 | status = acpi_get_table("IVRS", 0, &ivrs_base); |
02f3b3f5 JR |
2588 | if (status == AE_NOT_FOUND) |
2589 | return false; | |
2590 | else if (ACPI_FAILURE(status)) { | |
2591 | const char *err = acpi_format_exception(status); | |
101fa037 | 2592 | pr_err("IVRS table error: %s\n", err); |
02f3b3f5 JR |
2593 | return false; |
2594 | } | |
2595 | ||
6b11d1d6 | 2596 | acpi_put_table(ivrs_base); |
02f3b3f5 | 2597 | |
1adb7d31 JR |
2598 | /* Make sure ACS will be enabled during PCI probe */ |
2599 | pci_request_acs(); | |
2600 | ||
02f3b3f5 JR |
2601 | return true; |
2602 | } | |
2603 | ||
2c0ae172 | 2604 | /**************************************************************************** |
8704a1ba | 2605 | * |
2c0ae172 JR |
2606 | * AMD IOMMU Initialization State Machine |
2607 | * | |
2608 | ****************************************************************************/ | |
2609 | ||
2610 | static int __init state_next(void) | |
8704a1ba JR |
2611 | { |
2612 | int ret = 0; | |
2613 | ||
2c0ae172 JR |
2614 | switch (init_state) { |
2615 | case IOMMU_START_STATE: | |
2616 | if (!detect_ivrs()) { | |
2617 | init_state = IOMMU_NOT_FOUND; | |
2618 | ret = -ENODEV; | |
2619 | } else { | |
2620 | init_state = IOMMU_IVRS_DETECTED; | |
2621 | } | |
2622 | break; | |
2623 | case IOMMU_IVRS_DETECTED: | |
2624 | ret = early_amd_iommu_init(); | |
2625 | init_state = ret ? IOMMU_INIT_ERROR : IOMMU_ACPI_FINISHED; | |
7ad820e4 | 2626 | if (init_state == IOMMU_ACPI_FINISHED && amd_iommu_disabled) { |
101fa037 | 2627 | pr_info("AMD IOMMU disabled on kernel command-line\n"); |
7ad820e4 JR |
2628 | free_dma_resources(); |
2629 | free_iommu_resources(); | |
2630 | init_state = IOMMU_CMDLINE_DISABLED; | |
2631 | ret = -EINVAL; | |
2632 | } | |
2c0ae172 JR |
2633 | break; |
2634 | case IOMMU_ACPI_FINISHED: | |
2635 | early_enable_iommus(); | |
2c0ae172 JR |
2636 | x86_platform.iommu_shutdown = disable_iommus; |
2637 | init_state = IOMMU_ENABLED; | |
2638 | break; | |
2639 | case IOMMU_ENABLED: | |
74ddda71 | 2640 | register_syscore_ops(&amd_iommu_syscore_ops); |
2c0ae172 JR |
2641 | ret = amd_iommu_init_pci(); |
2642 | init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT; | |
2643 | enable_iommus_v2(); | |
2644 | break; | |
2645 | case IOMMU_PCI_INIT: | |
2646 | ret = amd_iommu_enable_interrupts(); | |
2647 | init_state = ret ? IOMMU_INIT_ERROR : IOMMU_INTERRUPTS_EN; | |
2648 | break; | |
2649 | case IOMMU_INTERRUPTS_EN: | |
1e6a7b04 | 2650 | ret = amd_iommu_init_dma_ops(); |
2c0ae172 JR |
2651 | init_state = ret ? IOMMU_INIT_ERROR : IOMMU_DMA_OPS; |
2652 | break; | |
2653 | case IOMMU_DMA_OPS: | |
2654 | init_state = IOMMU_INITIALIZED; | |
2655 | break; | |
2656 | case IOMMU_INITIALIZED: | |
2657 | /* Nothing to do */ | |
2658 | break; | |
2659 | case IOMMU_NOT_FOUND: | |
2660 | case IOMMU_INIT_ERROR: | |
1b1e942e | 2661 | case IOMMU_CMDLINE_DISABLED: |
2c0ae172 JR |
2662 | /* Error states => do nothing */ |
2663 | ret = -EINVAL; | |
2664 | break; | |
2665 | default: | |
2666 | /* Unknown state */ | |
2667 | BUG(); | |
2668 | } | |
3d9761e7 | 2669 | |
2c0ae172 JR |
2670 | return ret; |
2671 | } | |
7441e9cb | 2672 | |
2c0ae172 JR |
2673 | static int __init iommu_go_to_state(enum iommu_init_state state) |
2674 | { | |
151b0903 | 2675 | int ret = -EINVAL; |
f5325094 | 2676 | |
2c0ae172 | 2677 | while (init_state != state) { |
1b1e942e JR |
2678 | if (init_state == IOMMU_NOT_FOUND || |
2679 | init_state == IOMMU_INIT_ERROR || | |
2680 | init_state == IOMMU_CMDLINE_DISABLED) | |
2c0ae172 | 2681 | break; |
151b0903 | 2682 | ret = state_next(); |
2c0ae172 | 2683 | } |
f2f12b6f | 2684 | |
fe74c9cf | 2685 | return ret; |
2c0ae172 | 2686 | } |
fe74c9cf | 2687 | |
6b474b82 JR |
2688 | #ifdef CONFIG_IRQ_REMAP |
2689 | int __init amd_iommu_prepare(void) | |
2690 | { | |
3f4cb7c0 TG |
2691 | int ret; |
2692 | ||
7fa1c842 | 2693 | amd_iommu_irq_remap = true; |
84d07793 | 2694 | |
3f4cb7c0 TG |
2695 | ret = iommu_go_to_state(IOMMU_ACPI_FINISHED); |
2696 | if (ret) | |
2697 | return ret; | |
2698 | return amd_iommu_irq_remap ? 0 : -ENODEV; | |
6b474b82 | 2699 | } |
d7f07769 | 2700 | |
6b474b82 JR |
2701 | int __init amd_iommu_enable(void) |
2702 | { | |
2703 | int ret; | |
2704 | ||
2705 | ret = iommu_go_to_state(IOMMU_ENABLED); | |
2706 | if (ret) | |
2707 | return ret; | |
d7f07769 | 2708 | |
6b474b82 | 2709 | irq_remapping_enabled = 1; |
90fcffd9 | 2710 | return amd_iommu_xt_mode; |
6b474b82 JR |
2711 | } |
2712 | ||
2713 | void amd_iommu_disable(void) | |
2714 | { | |
2715 | amd_iommu_suspend(); | |
2716 | } | |
2717 | ||
2718 | int amd_iommu_reenable(int mode) | |
2719 | { | |
2720 | amd_iommu_resume(); | |
2721 | ||
2722 | return 0; | |
2723 | } | |
d7f07769 | 2724 | |
6b474b82 JR |
2725 | int __init amd_iommu_enable_faulting(void) |
2726 | { | |
2727 | /* We enable MSI later when PCI is initialized */ | |
2728 | return 0; | |
2729 | } | |
2730 | #endif | |
d7f07769 | 2731 | |
2c0ae172 JR |
2732 | /* |
2733 | * This is the core init function for AMD IOMMU hardware in the system. | |
2734 | * This function is called from the generic x86 DMA layer initialization | |
2735 | * code. | |
2736 | */ | |
2737 | static int __init amd_iommu_init(void) | |
2738 | { | |
7d0f5fd3 | 2739 | struct amd_iommu *iommu; |
2c0ae172 JR |
2740 | int ret; |
2741 | ||
2742 | ret = iommu_go_to_state(IOMMU_INITIALIZED); | |
2743 | if (ret) { | |
d04e0ba3 JR |
2744 | free_dma_resources(); |
2745 | if (!irq_remapping_enabled) { | |
2746 | disable_iommus(); | |
90b3eb03 | 2747 | free_iommu_resources(); |
d04e0ba3 | 2748 | } else { |
d04e0ba3 JR |
2749 | uninit_device_table_dma(); |
2750 | for_each_iommu(iommu) | |
2751 | iommu_flush_all_caches(iommu); | |
2752 | } | |
2c0ae172 JR |
2753 | } |
2754 | ||
7d0f5fd3 GH |
2755 | for_each_iommu(iommu) |
2756 | amd_iommu_debugfs_setup(iommu); | |
2757 | ||
2c0ae172 | 2758 | return ret; |
fe74c9cf JR |
2759 | } |
2760 | ||
2543a786 TL |
2761 | static bool amd_iommu_sme_check(void) |
2762 | { | |
2763 | if (!sme_active() || (boot_cpu_data.x86 != 0x17)) | |
2764 | return true; | |
2765 | ||
2766 | /* For Fam17h, a specific level of support is required */ | |
2767 | if (boot_cpu_data.microcode >= 0x08001205) | |
2768 | return true; | |
2769 | ||
2770 | if ((boot_cpu_data.microcode >= 0x08001126) && | |
2771 | (boot_cpu_data.microcode <= 0x080011ff)) | |
2772 | return true; | |
2773 | ||
101fa037 | 2774 | pr_notice("IOMMU not currently supported when SME is active\n"); |
2543a786 TL |
2775 | |
2776 | return false; | |
2777 | } | |
2778 | ||
b65233a9 JR |
2779 | /**************************************************************************** |
2780 | * | |
2781 | * Early detect code. This code runs at IOMMU detection time in the DMA | |
2782 | * layer. It just looks if there is an IVRS ACPI table to detect AMD | |
2783 | * IOMMUs | |
2784 | * | |
2785 | ****************************************************************************/ | |
480125ba | 2786 | int __init amd_iommu_detect(void) |
ae7877de | 2787 | { |
2c0ae172 | 2788 | int ret; |
02f3b3f5 | 2789 | |
75f1cdf1 | 2790 | if (no_iommu || (iommu_detected && !gart_iommu_aperture)) |
480125ba | 2791 | return -ENODEV; |
ae7877de | 2792 | |
2543a786 TL |
2793 | if (!amd_iommu_sme_check()) |
2794 | return -ENODEV; | |
2795 | ||
2c0ae172 JR |
2796 | ret = iommu_go_to_state(IOMMU_IVRS_DETECTED); |
2797 | if (ret) | |
2798 | return ret; | |
11bd04f6 | 2799 | |
02f3b3f5 JR |
2800 | amd_iommu_detected = true; |
2801 | iommu_detected = 1; | |
2802 | x86_init.iommu.iommu_init = amd_iommu_init; | |
2803 | ||
4781bc42 | 2804 | return 1; |
ae7877de JR |
2805 | } |
2806 | ||
b65233a9 JR |
2807 | /**************************************************************************** |
2808 | * | |
2809 | * Parsing functions for the AMD IOMMU specific kernel command line | |
2810 | * options. | |
2811 | * | |
2812 | ****************************************************************************/ | |
2813 | ||
fefda117 JR |
2814 | static int __init parse_amd_iommu_dump(char *str) |
2815 | { | |
2816 | amd_iommu_dump = true; | |
2817 | ||
2818 | return 1; | |
2819 | } | |
2820 | ||
3928aa3f SS |
2821 | static int __init parse_amd_iommu_intr(char *str) |
2822 | { | |
2823 | for (; *str; ++str) { | |
2824 | if (strncmp(str, "legacy", 6) == 0) { | |
2825 | amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY; | |
2826 | break; | |
2827 | } | |
2828 | if (strncmp(str, "vapic", 5) == 0) { | |
2829 | amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; | |
2830 | break; | |
2831 | } | |
2832 | } | |
2833 | return 1; | |
2834 | } | |
2835 | ||
918ad6c5 JR |
2836 | static int __init parse_amd_iommu_options(char *str) |
2837 | { | |
2838 | for (; *str; ++str) { | |
695b5676 | 2839 | if (strncmp(str, "fullflush", 9) == 0) |
afa9fdc2 | 2840 | amd_iommu_unmap_flush = true; |
a5235725 JR |
2841 | if (strncmp(str, "off", 3) == 0) |
2842 | amd_iommu_disabled = true; | |
5abcdba4 JR |
2843 | if (strncmp(str, "force_isolation", 15) == 0) |
2844 | amd_iommu_force_isolation = true; | |
918ad6c5 JR |
2845 | } |
2846 | ||
2847 | return 1; | |
2848 | } | |
2849 | ||
440e8998 JR |
2850 | static int __init parse_ivrs_ioapic(char *str) |
2851 | { | |
2852 | unsigned int bus, dev, fn; | |
2853 | int ret, id, i; | |
2854 | u16 devid; | |
2855 | ||
2856 | ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn); | |
2857 | ||
2858 | if (ret != 4) { | |
101fa037 | 2859 | pr_err("Invalid command line: ivrs_ioapic%s\n", str); |
440e8998 JR |
2860 | return 1; |
2861 | } | |
2862 | ||
2863 | if (early_ioapic_map_size == EARLY_MAP_SIZE) { | |
101fa037 | 2864 | pr_err("Early IOAPIC map overflow - ignoring ivrs_ioapic%s\n", |
440e8998 JR |
2865 | str); |
2866 | return 1; | |
2867 | } | |
2868 | ||
2869 | devid = ((bus & 0xff) << 8) | ((dev & 0x1f) << 3) | (fn & 0x7); | |
2870 | ||
dfbb6d47 | 2871 | cmdline_maps = true; |
440e8998 JR |
2872 | i = early_ioapic_map_size++; |
2873 | early_ioapic_map[i].id = id; | |
2874 | early_ioapic_map[i].devid = devid; | |
2875 | early_ioapic_map[i].cmd_line = true; | |
2876 | ||
2877 | return 1; | |
2878 | } | |
2879 | ||
2880 | static int __init parse_ivrs_hpet(char *str) | |
2881 | { | |
2882 | unsigned int bus, dev, fn; | |
2883 | int ret, id, i; | |
2884 | u16 devid; | |
2885 | ||
2886 | ret = sscanf(str, "[%d]=%x:%x.%x", &id, &bus, &dev, &fn); | |
2887 | ||
2888 | if (ret != 4) { | |
101fa037 | 2889 | pr_err("Invalid command line: ivrs_hpet%s\n", str); |
440e8998 JR |
2890 | return 1; |
2891 | } | |
2892 | ||
2893 | if (early_hpet_map_size == EARLY_MAP_SIZE) { | |
101fa037 | 2894 | pr_err("Early HPET map overflow - ignoring ivrs_hpet%s\n", |
440e8998 JR |
2895 | str); |
2896 | return 1; | |
2897 | } | |
2898 | ||
2899 | devid = ((bus & 0xff) << 8) | ((dev & 0x1f) << 3) | (fn & 0x7); | |
2900 | ||
dfbb6d47 | 2901 | cmdline_maps = true; |
440e8998 JR |
2902 | i = early_hpet_map_size++; |
2903 | early_hpet_map[i].id = id; | |
2904 | early_hpet_map[i].devid = devid; | |
2905 | early_hpet_map[i].cmd_line = true; | |
2906 | ||
2907 | return 1; | |
2908 | } | |
2909 | ||
ca3bf5d4 SS |
2910 | static int __init parse_ivrs_acpihid(char *str) |
2911 | { | |
2912 | u32 bus, dev, fn; | |
2913 | char *hid, *uid, *p; | |
2914 | char acpiid[ACPIHID_UID_LEN + ACPIHID_HID_LEN] = {0}; | |
2915 | int ret, i; | |
2916 | ||
2917 | ret = sscanf(str, "[%x:%x.%x]=%s", &bus, &dev, &fn, acpiid); | |
2918 | if (ret != 4) { | |
101fa037 | 2919 | pr_err("Invalid command line: ivrs_acpihid(%s)\n", str); |
ca3bf5d4 SS |
2920 | return 1; |
2921 | } | |
2922 | ||
2923 | p = acpiid; | |
2924 | hid = strsep(&p, ":"); | |
2925 | uid = p; | |
2926 | ||
2927 | if (!hid || !(*hid) || !uid) { | |
101fa037 | 2928 | pr_err("Invalid command line: hid or uid\n"); |
ca3bf5d4 SS |
2929 | return 1; |
2930 | } | |
2931 | ||
2932 | i = early_acpihid_map_size++; | |
2933 | memcpy(early_acpihid_map[i].hid, hid, strlen(hid)); | |
2934 | memcpy(early_acpihid_map[i].uid, uid, strlen(uid)); | |
2935 | early_acpihid_map[i].devid = | |
2936 | ((bus & 0xff) << 8) | ((dev & 0x1f) << 3) | (fn & 0x7); | |
2937 | early_acpihid_map[i].cmd_line = true; | |
2938 | ||
2939 | return 1; | |
2940 | } | |
2941 | ||
440e8998 JR |
2942 | __setup("amd_iommu_dump", parse_amd_iommu_dump); |
2943 | __setup("amd_iommu=", parse_amd_iommu_options); | |
3928aa3f | 2944 | __setup("amd_iommu_intr=", parse_amd_iommu_intr); |
440e8998 JR |
2945 | __setup("ivrs_ioapic", parse_ivrs_ioapic); |
2946 | __setup("ivrs_hpet", parse_ivrs_hpet); | |
ca3bf5d4 | 2947 | __setup("ivrs_acpihid", parse_ivrs_acpihid); |
22e6daf4 KRW |
2948 | |
2949 | IOMMU_INIT_FINISH(amd_iommu_detect, | |
2950 | gart_iommu_hole_init, | |
98f1ad25 JR |
2951 | NULL, |
2952 | NULL); | |
400a28a0 JR |
2953 | |
2954 | bool amd_iommu_v2_supported(void) | |
2955 | { | |
2956 | return amd_iommu_v2_present; | |
2957 | } | |
2958 | EXPORT_SYMBOL(amd_iommu_v2_supported); | |
30861ddc | 2959 | |
f5863a00 SS |
2960 | struct amd_iommu *get_amd_iommu(unsigned int idx) |
2961 | { | |
2962 | unsigned int i = 0; | |
2963 | struct amd_iommu *iommu; | |
2964 | ||
2965 | for_each_iommu(iommu) | |
2966 | if (i++ == idx) | |
2967 | return iommu; | |
2968 | return NULL; | |
2969 | } | |
2970 | EXPORT_SYMBOL(get_amd_iommu); | |
2971 | ||
30861ddc SK |
2972 | /**************************************************************************** |
2973 | * | |
2974 | * IOMMU EFR Performance Counter support functionality. This code allows | |
2975 | * access to the IOMMU PC functionality. | |
2976 | * | |
2977 | ****************************************************************************/ | |
2978 | ||
f5863a00 | 2979 | u8 amd_iommu_pc_get_max_banks(unsigned int idx) |
30861ddc | 2980 | { |
f5863a00 | 2981 | struct amd_iommu *iommu = get_amd_iommu(idx); |
30861ddc | 2982 | |
30861ddc | 2983 | if (iommu) |
f5863a00 | 2984 | return iommu->max_banks; |
30861ddc | 2985 | |
f5863a00 | 2986 | return 0; |
30861ddc SK |
2987 | } |
2988 | EXPORT_SYMBOL(amd_iommu_pc_get_max_banks); | |
2989 | ||
2990 | bool amd_iommu_pc_supported(void) | |
2991 | { | |
2992 | return amd_iommu_pc_present; | |
2993 | } | |
2994 | EXPORT_SYMBOL(amd_iommu_pc_supported); | |
2995 | ||
f5863a00 | 2996 | u8 amd_iommu_pc_get_max_counters(unsigned int idx) |
30861ddc | 2997 | { |
f5863a00 | 2998 | struct amd_iommu *iommu = get_amd_iommu(idx); |
30861ddc | 2999 | |
30861ddc | 3000 | if (iommu) |
f5863a00 | 3001 | return iommu->max_counters; |
30861ddc | 3002 | |
f5863a00 | 3003 | return 0; |
30861ddc SK |
3004 | } |
3005 | EXPORT_SYMBOL(amd_iommu_pc_get_max_counters); | |
3006 | ||
1650dfd1 SS |
3007 | static int iommu_pc_get_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, |
3008 | u8 fxn, u64 *value, bool is_write) | |
30861ddc | 3009 | { |
30861ddc SK |
3010 | u32 offset; |
3011 | u32 max_offset_lim; | |
3012 | ||
1650dfd1 SS |
3013 | /* Make sure the IOMMU PC resource is available */ |
3014 | if (!amd_iommu_pc_present) | |
3015 | return -ENODEV; | |
3016 | ||
30861ddc | 3017 | /* Check for valid iommu and pc register indexing */ |
1650dfd1 | 3018 | if (WARN_ON(!iommu || (fxn > 0x28) || (fxn & 7))) |
30861ddc SK |
3019 | return -ENODEV; |
3020 | ||
0a6d80c7 | 3021 | offset = (u32)(((0x40 | bank) << 12) | (cntr << 8) | fxn); |
30861ddc SK |
3022 | |
3023 | /* Limit the offset to the hw defined mmio region aperture */ | |
0a6d80c7 | 3024 | max_offset_lim = (u32)(((0x40 | iommu->max_banks) << 12) | |
30861ddc SK |
3025 | (iommu->max_counters << 8) | 0x28); |
3026 | if ((offset < MMIO_CNTR_REG_OFFSET) || | |
3027 | (offset > max_offset_lim)) | |
3028 | return -EINVAL; | |
3029 | ||
3030 | if (is_write) { | |
0a6d80c7 SS |
3031 | u64 val = *value & GENMASK_ULL(47, 0); |
3032 | ||
3033 | writel((u32)val, iommu->mmio_base + offset); | |
3034 | writel((val >> 32), iommu->mmio_base + offset + 4); | |
30861ddc SK |
3035 | } else { |
3036 | *value = readl(iommu->mmio_base + offset + 4); | |
3037 | *value <<= 32; | |
0a6d80c7 SS |
3038 | *value |= readl(iommu->mmio_base + offset); |
3039 | *value &= GENMASK_ULL(47, 0); | |
30861ddc SK |
3040 | } |
3041 | ||
3042 | return 0; | |
3043 | } | |
38e45d02 | 3044 | |
1650dfd1 | 3045 | int amd_iommu_pc_get_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value) |
38e45d02 | 3046 | { |
1650dfd1 SS |
3047 | if (!iommu) |
3048 | return -EINVAL; | |
38e45d02 | 3049 | |
1650dfd1 SS |
3050 | return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, false); |
3051 | } | |
3052 | EXPORT_SYMBOL(amd_iommu_pc_get_reg); | |
3053 | ||
3054 | int amd_iommu_pc_set_reg(struct amd_iommu *iommu, u8 bank, u8 cntr, u8 fxn, u64 *value) | |
3055 | { | |
3056 | if (!iommu) | |
3057 | return -EINVAL; | |
38e45d02 | 3058 | |
1650dfd1 | 3059 | return iommu_pc_get_set_reg(iommu, bank, cntr, fxn, value, true); |
38e45d02 | 3060 | } |
1650dfd1 | 3061 | EXPORT_SYMBOL(amd_iommu_pc_set_reg); |