arch/x86/kernel/pci-dma.c

   1 #include <linux/dma-mapping.h>
   2 #include <linux/dmar.h>
   3 #include <linux/bootmem.h>
   4 #include <linux/pci.h>
   5
   6 #include <asm/proto.h>
   7 #include <asm/dma.h>
   8 #include <asm/iommu.h>
   9 #include <asm/calgary.h>
  10 #include <asm/amd_iommu.h>
  11
  12 static int forbid_dac __read_mostly;
  13
  14 struct dma_mapping_ops *dma_ops;
  15 EXPORT_SYMBOL(dma_ops);
  16
  17 static int iommu_sac_force __read_mostly;
  18
  19 #ifdef CONFIG_IOMMU_DEBUG
  20 int panic_on_overflow __read_mostly = 1;
  21 int force_iommu __read_mostly = 1;
  22 #else
  23 int panic_on_overflow __read_mostly = 0;
  24 int force_iommu __read_mostly = 0;
  25 #endif
  26
  27 int iommu_merge __read_mostly = 0;
  28
  29 int no_iommu __read_mostly;
  30 /* Set this to 1 if there is a HW IOMMU in the system */
  31 int iommu_detected __read_mostly = 0;
  32
  33 /* This tells the BIO block layer to assume merging. Default to off
  34    because we cannot guarantee merging later. */
  35 int iommu_bio_merge __read_mostly = 0;
  36 EXPORT_SYMBOL(iommu_bio_merge);
  37
  38 dma_addr_t bad_dma_address __read_mostly = 0;
  39 EXPORT_SYMBOL(bad_dma_address);
  40
  41 /* Dummy device used for NULL arguments (normally ISA). Better would
  42    be probably a smaller DMA mask, but this is bug-to-bug compatible
  43    to older i386. */
  44 struct device x86_dma_fallback_dev = {
  45         .bus_id = "fallback device",
  46         .coherent_dma_mask = DMA_32BIT_MASK,
  47         .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
  48 };
  49 EXPORT_SYMBOL(x86_dma_fallback_dev);
  50
  51 int dma_set_mask(struct device *dev, u64 mask)
  52 {
  53         if (!dev->dma_mask || !dma_supported(dev, mask))
  54                 return -EIO;
  55
  56         *dev->dma_mask = mask;
  57
  58         return 0;
  59 }
  60 EXPORT_SYMBOL(dma_set_mask);
  61
  62 #ifdef CONFIG_X86_64
  63 static __initdata void *dma32_bootmem_ptr;
  64 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
  65
  66 static int __init parse_dma32_size_opt(char *p)
  67 {
  68         if (!p)
  69                 return -EINVAL;
  70         dma32_bootmem_size = memparse(p, &p);
  71         return 0;
  72 }
  73 early_param("dma32_size", parse_dma32_size_opt);
  74
  75 void __init dma32_reserve_bootmem(void)
  76 {
  77         unsigned long size, align;
  78         if (max_pfn <= MAX_DMA32_PFN)
  79                 return;
  80
  81         /*
  82          * check aperture_64.c allocate_aperture() for reason about
  83          * using 512M as goal
  84          */
  85         align = 64ULL<<20;
  86         size = round_up(dma32_bootmem_size, align);
  87         dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
  88                                  512ULL<<20);
  89         if (dma32_bootmem_ptr)
  90                 dma32_bootmem_size = size;
  91         else
  92                 dma32_bootmem_size = 0;
  93 }
  94 static void __init dma32_free_bootmem(void)
  95 {
  96
  97         if (max_pfn <= MAX_DMA32_PFN)
  98                 return;
  99
 100         if (!dma32_bootmem_ptr)
 101                 return;
 102
 103         free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
 104
 105         dma32_bootmem_ptr = NULL;
 106         dma32_bootmem_size = 0;
 107 }
 108
 109 void __init pci_iommu_alloc(void)
 110 {
 111         /* free the range so iommu could get some range less than 4G */
 112         dma32_free_bootmem();
 113         /*
 114          * The order of these functions is important for
 115          * fall-back/fail-over reasons
 116          */
 117         gart_iommu_hole_init();
 118
 119         detect_calgary();
 120
 121         detect_intel_iommu();
 122
 123         amd_iommu_detect();
 124
 125         pci_swiotlb_init();
 126 }
 127
 128 unsigned long iommu_num_pages(unsigned long addr, unsigned long len)
 129 {
 130         unsigned long size = roundup((addr & ~PAGE_MASK) + len, PAGE_SIZE);
 131
 132         return size >> PAGE_SHIFT;
 133 }
 134 EXPORT_SYMBOL(iommu_num_pages);
 135 #endif
 136
 137 /*
 138  * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 139  * documentation.
 140  */
 141 static __init int iommu_setup(char *p)
 142 {
 143         iommu_merge = 1;
 144
 145         if (!p)
 146                 return -EINVAL;
 147
 148         while (*p) {
 149                 if (!strncmp(p, "off", 3))
 150                         no_iommu = 1;
 151                 /* gart_parse_options has more force support */
 152                 if (!strncmp(p, "force", 5))
 153                         force_iommu = 1;
 154                 if (!strncmp(p, "noforce", 7)) {
 155                         iommu_merge = 0;
 156                         force_iommu = 0;
 157                 }
 158
 159                 if (!strncmp(p, "biomerge", 8)) {
 160                         iommu_bio_merge = 4096;
 161                         iommu_merge = 1;
 162                         force_iommu = 1;
 163                 }
 164                 if (!strncmp(p, "panic", 5))
 165                         panic_on_overflow = 1;
 166                 if (!strncmp(p, "nopanic", 7))
 167                         panic_on_overflow = 0;
 168                 if (!strncmp(p, "merge", 5)) {
 169                         iommu_merge = 1;
 170                         force_iommu = 1;
 171                 }
 172                 if (!strncmp(p, "nomerge", 7))
 173                         iommu_merge = 0;
 174                 if (!strncmp(p, "forcesac", 8))
 175                         iommu_sac_force = 1;
 176                 if (!strncmp(p, "allowdac", 8))
 177                         forbid_dac = 0;
 178                 if (!strncmp(p, "nodac", 5))
 179                         forbid_dac = -1;
 180                 if (!strncmp(p, "usedac", 6)) {
 181                         forbid_dac = -1;
 182                         return 1;
 183                 }
 184 #ifdef CONFIG_SWIOTLB
 185                 if (!strncmp(p, "soft", 4))
 186                         swiotlb = 1;
 187 #endif
 188
 189                 gart_parse_options(p);
 190
 191 #ifdef CONFIG_CALGARY_IOMMU
 192                 if (!strncmp(p, "calgary", 7))
 193                         use_calgary = 1;
 194 #endif /* CONFIG_CALGARY_IOMMU */
 195
 196                 p += strcspn(p, ",");
 197                 if (*p == ',')
 198                         ++p;
 199         }
 200         return 0;
 201 }
 202 early_param("iommu", iommu_setup);
 203
 204 int dma_supported(struct device *dev, u64 mask)
 205 {
 206         struct dma_mapping_ops *ops = get_dma_ops(dev);
 207
 208 #ifdef CONFIG_PCI
 209         if (mask > 0xffffffff && forbid_dac > 0) {
 210                 dev_info(dev, "PCI: Disallowing DAC for device\n");
 211                 return 0;
 212         }
 213 #endif
 214
 215         if (ops->dma_supported)
 216                 return ops->dma_supported(dev, mask);
 217
 218         /* Copied from i386. Doesn't make much sense, because it will
 219            only work for pci_alloc_coherent.
 220            The caller just has to use GFP_DMA in this case. */
 221         if (mask < DMA_24BIT_MASK)
 222                 return 0;
 223
 224         /* Tell the device to use SAC when IOMMU force is on.  This
 225            allows the driver to use cheaper accesses in some cases.
 226
 227            Problem with this is that if we overflow the IOMMU area and
 228            return DAC as fallback address the device may not handle it
 229            correctly.
 230
 231            As a special case some controllers have a 39bit address
 232            mode that is as efficient as 32bit (aic79xx). Don't force
 233            SAC for these.  Assume all masks <= 40 bits are of this
 234            type. Normally this doesn't make any difference, but gives
 235            more gentle handling of IOMMU overflow. */
 236         if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
 237                 dev_info(dev, "Force SAC with mask %Lx\n", mask);
 238                 return 0;
 239         }
 240
 241         return 1;
 242 }
 243 EXPORT_SYMBOL(dma_supported);
 244
 245 static int __init pci_iommu_init(void)
 246 {
 247         calgary_iommu_init();
 248
 249         intel_iommu_init();
 250
 251         amd_iommu_init();
 252
 253         gart_iommu_init();
 254
 255         no_iommu_init();
 256         return 0;
 257 }
 258
 259 void pci_iommu_shutdown(void)
 260 {
 261         gart_iommu_shutdown();
 262 }
 263 /* Must execute after PCI subsystem */
 264 fs_initcall(pci_iommu_init);
 265
 266 #ifdef CONFIG_PCI
 267 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
 268
 269 static __devinit void via_no_dac(struct pci_dev *dev)
 270 {
 271         if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
 272                 printk(KERN_INFO "PCI: VIA PCI bridge detected."
 273                                  "Disabling DAC.\n");
 274                 forbid_dac = 1;
 275         }
 276 }
 277 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
 278 #endif