]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - drivers/pci/dmar.c
KVM: x86 emulator: Use DstAcc for 'and'
[mirror_ubuntu-zesty-kernel.git] / drivers / pci / dmar.c
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 *
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
25 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
27 */
28
29 #include <linux/pci.h>
30 #include <linux/dmar.h>
31 #include <linux/timer.h>
32 #include "iova.h"
33 #include "intel-iommu.h"
34
35 #undef PREFIX
36 #define PREFIX "DMAR:"
37
38 /* No locks are needed as DMA remapping hardware unit
39 * list is constructed at boot time and hotplug of
40 * these units are not supported by the architecture.
41 */
42 LIST_HEAD(dmar_drhd_units);
43
44 static struct acpi_table_header * __initdata dmar_tbl;
45
46 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
47 {
48 /*
49 * add INCLUDE_ALL at the tail, so scan the list will find it at
50 * the very end.
51 */
52 if (drhd->include_all)
53 list_add_tail(&drhd->list, &dmar_drhd_units);
54 else
55 list_add(&drhd->list, &dmar_drhd_units);
56 }
57
58 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
59 struct pci_dev **dev, u16 segment)
60 {
61 struct pci_bus *bus;
62 struct pci_dev *pdev = NULL;
63 struct acpi_dmar_pci_path *path;
64 int count;
65
66 bus = pci_find_bus(segment, scope->bus);
67 path = (struct acpi_dmar_pci_path *)(scope + 1);
68 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
69 / sizeof(struct acpi_dmar_pci_path);
70
71 while (count) {
72 if (pdev)
73 pci_dev_put(pdev);
74 /*
75 * Some BIOSes list non-exist devices in DMAR table, just
76 * ignore it
77 */
78 if (!bus) {
79 printk(KERN_WARNING
80 PREFIX "Device scope bus [%d] not found\n",
81 scope->bus);
82 break;
83 }
84 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
85 if (!pdev) {
86 printk(KERN_WARNING PREFIX
87 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
88 segment, bus->number, path->dev, path->fn);
89 break;
90 }
91 path ++;
92 count --;
93 bus = pdev->subordinate;
94 }
95 if (!pdev) {
96 printk(KERN_WARNING PREFIX
97 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
98 segment, scope->bus, path->dev, path->fn);
99 *dev = NULL;
100 return 0;
101 }
102 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
103 pdev->subordinate) || (scope->entry_type == \
104 ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
105 pci_dev_put(pdev);
106 printk(KERN_WARNING PREFIX
107 "Device scope type does not match for %s\n",
108 pci_name(pdev));
109 return -EINVAL;
110 }
111 *dev = pdev;
112 return 0;
113 }
114
115 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
116 struct pci_dev ***devices, u16 segment)
117 {
118 struct acpi_dmar_device_scope *scope;
119 void * tmp = start;
120 int index;
121 int ret;
122
123 *cnt = 0;
124 while (start < end) {
125 scope = start;
126 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
127 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
128 (*cnt)++;
129 else
130 printk(KERN_WARNING PREFIX
131 "Unsupported device scope\n");
132 start += scope->length;
133 }
134 if (*cnt == 0)
135 return 0;
136
137 *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
138 if (!*devices)
139 return -ENOMEM;
140
141 start = tmp;
142 index = 0;
143 while (start < end) {
144 scope = start;
145 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
146 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
147 ret = dmar_parse_one_dev_scope(scope,
148 &(*devices)[index], segment);
149 if (ret) {
150 kfree(*devices);
151 return ret;
152 }
153 index ++;
154 }
155 start += scope->length;
156 }
157
158 return 0;
159 }
160
161 /**
162 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
163 * structure which uniquely represent one DMA remapping hardware unit
164 * present in the platform
165 */
166 static int __init
167 dmar_parse_one_drhd(struct acpi_dmar_header *header)
168 {
169 struct acpi_dmar_hardware_unit *drhd;
170 struct dmar_drhd_unit *dmaru;
171 int ret = 0;
172
173 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
174 if (!dmaru)
175 return -ENOMEM;
176
177 dmaru->hdr = header;
178 drhd = (struct acpi_dmar_hardware_unit *)header;
179 dmaru->reg_base_addr = drhd->address;
180 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
181
182 ret = alloc_iommu(dmaru);
183 if (ret) {
184 kfree(dmaru);
185 return ret;
186 }
187 dmar_register_drhd_unit(dmaru);
188 return 0;
189 }
190
191 static int __init
192 dmar_parse_dev(struct dmar_drhd_unit *dmaru)
193 {
194 struct acpi_dmar_hardware_unit *drhd;
195 static int include_all;
196 int ret;
197
198 drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
199
200 if (!dmaru->include_all)
201 ret = dmar_parse_dev_scope((void *)(drhd + 1),
202 ((void *)drhd) + drhd->header.length,
203 &dmaru->devices_cnt, &dmaru->devices,
204 drhd->segment);
205 else {
206 /* Only allow one INCLUDE_ALL */
207 if (include_all) {
208 printk(KERN_WARNING PREFIX "Only one INCLUDE_ALL "
209 "device scope is allowed\n");
210 ret = -EINVAL;
211 }
212 include_all = 1;
213 }
214
215 if (ret || (dmaru->devices_cnt == 0 && !dmaru->include_all)) {
216 list_del(&dmaru->list);
217 kfree(dmaru);
218 }
219 return ret;
220 }
221
222 #ifdef CONFIG_DMAR
223 LIST_HEAD(dmar_rmrr_units);
224
225 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
226 {
227 list_add(&rmrr->list, &dmar_rmrr_units);
228 }
229
230
231 static int __init
232 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
233 {
234 struct acpi_dmar_reserved_memory *rmrr;
235 struct dmar_rmrr_unit *rmrru;
236
237 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
238 if (!rmrru)
239 return -ENOMEM;
240
241 rmrru->hdr = header;
242 rmrr = (struct acpi_dmar_reserved_memory *)header;
243 rmrru->base_address = rmrr->base_address;
244 rmrru->end_address = rmrr->end_address;
245
246 dmar_register_rmrr_unit(rmrru);
247 return 0;
248 }
249
250 static int __init
251 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
252 {
253 struct acpi_dmar_reserved_memory *rmrr;
254 int ret;
255
256 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
257 ret = dmar_parse_dev_scope((void *)(rmrr + 1),
258 ((void *)rmrr) + rmrr->header.length,
259 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
260
261 if (ret || (rmrru->devices_cnt == 0)) {
262 list_del(&rmrru->list);
263 kfree(rmrru);
264 }
265 return ret;
266 }
267 #endif
268
269 static void __init
270 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
271 {
272 struct acpi_dmar_hardware_unit *drhd;
273 struct acpi_dmar_reserved_memory *rmrr;
274
275 switch (header->type) {
276 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
277 drhd = (struct acpi_dmar_hardware_unit *)header;
278 printk (KERN_INFO PREFIX
279 "DRHD (flags: 0x%08x)base: 0x%016Lx\n",
280 drhd->flags, drhd->address);
281 break;
282 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
283 rmrr = (struct acpi_dmar_reserved_memory *)header;
284
285 printk (KERN_INFO PREFIX
286 "RMRR base: 0x%016Lx end: 0x%016Lx\n",
287 rmrr->base_address, rmrr->end_address);
288 break;
289 }
290 }
291
292
293 /**
294 * parse_dmar_table - parses the DMA reporting table
295 */
296 static int __init
297 parse_dmar_table(void)
298 {
299 struct acpi_table_dmar *dmar;
300 struct acpi_dmar_header *entry_header;
301 int ret = 0;
302
303 dmar = (struct acpi_table_dmar *)dmar_tbl;
304 if (!dmar)
305 return -ENODEV;
306
307 if (dmar->width < PAGE_SHIFT_4K - 1) {
308 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
309 return -EINVAL;
310 }
311
312 printk (KERN_INFO PREFIX "Host address width %d\n",
313 dmar->width + 1);
314
315 entry_header = (struct acpi_dmar_header *)(dmar + 1);
316 while (((unsigned long)entry_header) <
317 (((unsigned long)dmar) + dmar_tbl->length)) {
318 dmar_table_print_dmar_entry(entry_header);
319
320 switch (entry_header->type) {
321 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
322 ret = dmar_parse_one_drhd(entry_header);
323 break;
324 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
325 #ifdef CONFIG_DMAR
326 ret = dmar_parse_one_rmrr(entry_header);
327 #endif
328 break;
329 default:
330 printk(KERN_WARNING PREFIX
331 "Unknown DMAR structure type\n");
332 ret = 0; /* for forward compatibility */
333 break;
334 }
335 if (ret)
336 break;
337
338 entry_header = ((void *)entry_header + entry_header->length);
339 }
340 return ret;
341 }
342
343 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
344 struct pci_dev *dev)
345 {
346 int index;
347
348 while (dev) {
349 for (index = 0; index < cnt; index++)
350 if (dev == devices[index])
351 return 1;
352
353 /* Check our parent */
354 dev = dev->bus->self;
355 }
356
357 return 0;
358 }
359
360 struct dmar_drhd_unit *
361 dmar_find_matched_drhd_unit(struct pci_dev *dev)
362 {
363 struct dmar_drhd_unit *drhd = NULL;
364
365 list_for_each_entry(drhd, &dmar_drhd_units, list) {
366 if (drhd->include_all || dmar_pci_device_match(drhd->devices,
367 drhd->devices_cnt, dev))
368 return drhd;
369 }
370
371 return NULL;
372 }
373
374 int __init dmar_dev_scope_init(void)
375 {
376 struct dmar_drhd_unit *drhd;
377 int ret = -ENODEV;
378
379 for_each_drhd_unit(drhd) {
380 ret = dmar_parse_dev(drhd);
381 if (ret)
382 return ret;
383 }
384
385 #ifdef CONFIG_DMAR
386 {
387 struct dmar_rmrr_unit *rmrr;
388 for_each_rmrr_units(rmrr) {
389 ret = rmrr_parse_dev(rmrr);
390 if (ret)
391 return ret;
392 }
393 }
394 #endif
395
396 return ret;
397 }
398
399
400 int __init dmar_table_init(void)
401 {
402 static int dmar_table_initialized;
403 int ret;
404
405 if (dmar_table_initialized)
406 return 0;
407
408 dmar_table_initialized = 1;
409
410 ret = parse_dmar_table();
411 if (ret) {
412 if (ret != -ENODEV)
413 printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
414 return ret;
415 }
416
417 if (list_empty(&dmar_drhd_units)) {
418 printk(KERN_INFO PREFIX "No DMAR devices found\n");
419 return -ENODEV;
420 }
421
422 #ifdef CONFIG_DMAR
423 if (list_empty(&dmar_rmrr_units))
424 printk(KERN_INFO PREFIX "No RMRR found\n");
425 #endif
426
427 #ifdef CONFIG_INTR_REMAP
428 parse_ioapics_under_ir();
429 #endif
430 return 0;
431 }
432
433 /**
434 * early_dmar_detect - checks to see if the platform supports DMAR devices
435 */
436 int __init early_dmar_detect(void)
437 {
438 acpi_status status = AE_OK;
439
440 /* if we could find DMAR table, then there are DMAR devices */
441 status = acpi_get_table(ACPI_SIG_DMAR, 0,
442 (struct acpi_table_header **)&dmar_tbl);
443
444 if (ACPI_SUCCESS(status) && !dmar_tbl) {
445 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
446 status = AE_NOT_FOUND;
447 }
448
449 return (ACPI_SUCCESS(status) ? 1 : 0);
450 }
451
452 void __init detect_intel_iommu(void)
453 {
454 int ret;
455
456 ret = early_dmar_detect();
457
458 #ifdef CONFIG_DMAR
459 {
460 struct acpi_table_dmar *dmar;
461 /*
462 * for now we will disable dma-remapping when interrupt
463 * remapping is enabled.
464 * When support for queued invalidation for IOTLB invalidation
465 * is added, we will not need this any more.
466 */
467 dmar = (struct acpi_table_dmar *) dmar_tbl;
468 if (ret && cpu_has_x2apic && dmar->flags & 0x1) {
469 printk(KERN_INFO
470 "Queued invalidation will be enabled to support "
471 "x2apic and Intr-remapping.\n");
472 printk(KERN_INFO
473 "Disabling IOMMU detection, because of missing "
474 "queued invalidation support for IOTLB "
475 "invalidation\n");
476 printk(KERN_INFO
477 "Use \"nox2apic\", if you want to use Intel "
478 " IOMMU for DMA-remapping and don't care about "
479 " x2apic support\n");
480
481 dmar_disabled = 1;
482 return;
483 }
484
485 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
486 !dmar_disabled)
487 iommu_detected = 1;
488 }
489 #endif
490 }
491
492
493 int alloc_iommu(struct dmar_drhd_unit *drhd)
494 {
495 struct intel_iommu *iommu;
496 int map_size;
497 u32 ver;
498 static int iommu_allocated = 0;
499
500 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
501 if (!iommu)
502 return -ENOMEM;
503
504 iommu->seq_id = iommu_allocated++;
505
506 iommu->reg = ioremap(drhd->reg_base_addr, PAGE_SIZE_4K);
507 if (!iommu->reg) {
508 printk(KERN_ERR "IOMMU: can't map the region\n");
509 goto error;
510 }
511 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
512 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
513
514 /* the registers might be more than one page */
515 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
516 cap_max_fault_reg_offset(iommu->cap));
517 map_size = PAGE_ALIGN_4K(map_size);
518 if (map_size > PAGE_SIZE_4K) {
519 iounmap(iommu->reg);
520 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
521 if (!iommu->reg) {
522 printk(KERN_ERR "IOMMU: can't map the region\n");
523 goto error;
524 }
525 }
526
527 ver = readl(iommu->reg + DMAR_VER_REG);
528 pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
529 drhd->reg_base_addr, DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
530 iommu->cap, iommu->ecap);
531
532 spin_lock_init(&iommu->register_lock);
533
534 drhd->iommu = iommu;
535 return 0;
536 error:
537 kfree(iommu);
538 return -1;
539 }
540
541 void free_iommu(struct intel_iommu *iommu)
542 {
543 if (!iommu)
544 return;
545
546 #ifdef CONFIG_DMAR
547 free_dmar_iommu(iommu);
548 #endif
549
550 if (iommu->reg)
551 iounmap(iommu->reg);
552 kfree(iommu);
553 }
554
555 /*
556 * Reclaim all the submitted descriptors which have completed its work.
557 */
558 static inline void reclaim_free_desc(struct q_inval *qi)
559 {
560 while (qi->desc_status[qi->free_tail] == QI_DONE) {
561 qi->desc_status[qi->free_tail] = QI_FREE;
562 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
563 qi->free_cnt++;
564 }
565 }
566
567 /*
568 * Submit the queued invalidation descriptor to the remapping
569 * hardware unit and wait for its completion.
570 */
571 void qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
572 {
573 struct q_inval *qi = iommu->qi;
574 struct qi_desc *hw, wait_desc;
575 int wait_index, index;
576 unsigned long flags;
577
578 if (!qi)
579 return;
580
581 hw = qi->desc;
582
583 spin_lock(&qi->q_lock);
584 while (qi->free_cnt < 3) {
585 spin_unlock(&qi->q_lock);
586 cpu_relax();
587 spin_lock(&qi->q_lock);
588 }
589
590 index = qi->free_head;
591 wait_index = (index + 1) % QI_LENGTH;
592
593 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
594
595 hw[index] = *desc;
596
597 wait_desc.low = QI_IWD_STATUS_DATA(2) | QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
598 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
599
600 hw[wait_index] = wait_desc;
601
602 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
603 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
604
605 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
606 qi->free_cnt -= 2;
607
608 spin_lock_irqsave(&iommu->register_lock, flags);
609 /*
610 * update the HW tail register indicating the presence of
611 * new descriptors.
612 */
613 writel(qi->free_head << 4, iommu->reg + DMAR_IQT_REG);
614 spin_unlock_irqrestore(&iommu->register_lock, flags);
615
616 while (qi->desc_status[wait_index] != QI_DONE) {
617 spin_unlock(&qi->q_lock);
618 cpu_relax();
619 spin_lock(&qi->q_lock);
620 }
621
622 qi->desc_status[index] = QI_DONE;
623
624 reclaim_free_desc(qi);
625 spin_unlock(&qi->q_lock);
626 }
627
628 /*
629 * Flush the global interrupt entry cache.
630 */
631 void qi_global_iec(struct intel_iommu *iommu)
632 {
633 struct qi_desc desc;
634
635 desc.low = QI_IEC_TYPE;
636 desc.high = 0;
637
638 qi_submit_sync(&desc, iommu);
639 }
640
641 /*
642 * Enable Queued Invalidation interface. This is a must to support
643 * interrupt-remapping. Also used by DMA-remapping, which replaces
644 * register based IOTLB invalidation.
645 */
646 int dmar_enable_qi(struct intel_iommu *iommu)
647 {
648 u32 cmd, sts;
649 unsigned long flags;
650 struct q_inval *qi;
651
652 if (!ecap_qis(iommu->ecap))
653 return -ENOENT;
654
655 /*
656 * queued invalidation is already setup and enabled.
657 */
658 if (iommu->qi)
659 return 0;
660
661 iommu->qi = kmalloc(sizeof(*qi), GFP_KERNEL);
662 if (!iommu->qi)
663 return -ENOMEM;
664
665 qi = iommu->qi;
666
667 qi->desc = (void *)(get_zeroed_page(GFP_KERNEL));
668 if (!qi->desc) {
669 kfree(qi);
670 iommu->qi = 0;
671 return -ENOMEM;
672 }
673
674 qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_KERNEL);
675 if (!qi->desc_status) {
676 free_page((unsigned long) qi->desc);
677 kfree(qi);
678 iommu->qi = 0;
679 return -ENOMEM;
680 }
681
682 qi->free_head = qi->free_tail = 0;
683 qi->free_cnt = QI_LENGTH;
684
685 spin_lock_init(&qi->q_lock);
686
687 spin_lock_irqsave(&iommu->register_lock, flags);
688 /* write zero to the tail reg */
689 writel(0, iommu->reg + DMAR_IQT_REG);
690
691 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
692
693 cmd = iommu->gcmd | DMA_GCMD_QIE;
694 iommu->gcmd |= DMA_GCMD_QIE;
695 writel(cmd, iommu->reg + DMAR_GCMD_REG);
696
697 /* Make sure hardware complete it */
698 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
699 spin_unlock_irqrestore(&iommu->register_lock, flags);
700
701 return 0;
702 }