arch/x86/kernel/pci-dma.c

   1 #include <linux/dma-mapping.h>
   2 #include <linux/dma-debug.h>
   3 #include <linux/dmar.h>
   4 #include <linux/bootmem.h>
   5 #include <linux/gfp.h>
   6 #include <linux/pci.h>
   7 #include <linux/kmemleak.h>
   8
   9 #include <asm/proto.h>
  10 #include <asm/dma.h>
  11 #include <asm/iommu.h>
  12 #include <asm/gart.h>
  13 #include <asm/calgary.h>
  14 #include <asm/amd_iommu.h>
  15 #include <asm/x86_init.h>
  16
  17 static int forbid_dac __read_mostly;
  18
  19 struct dma_map_ops *dma_ops = &nommu_dma_ops;
  20 EXPORT_SYMBOL(dma_ops);
  21
  22 static int iommu_sac_force __read_mostly;
  23
  24 #ifdef CONFIG_IOMMU_DEBUG
  25 int panic_on_overflow __read_mostly = 1;
  26 int force_iommu __read_mostly = 1;
  27 #else
  28 int panic_on_overflow __read_mostly = 0;
  29 int force_iommu __read_mostly = 0;
  30 #endif
  31
  32 int iommu_merge __read_mostly = 0;
  33
  34 int no_iommu __read_mostly;
  35 /* Set this to 1 if there is a HW IOMMU in the system */
  36 int iommu_detected __read_mostly = 0;
  37
  38 /*
  39  * This variable becomes 1 if iommu=pt is passed on the kernel command line.
  40  * If this variable is 1, IOMMU implementations do no DMA translation for
  41  * devices and allow every device to access to whole physical memory. This is
  42  * useful if a user wants to use an IOMMU only for KVM device assignment to
  43  * guests and not for driver dma translation.
  44  */
  45 int iommu_pass_through __read_mostly;
  46
  47 /* Dummy device used for NULL arguments (normally ISA). */
  48 struct device x86_dma_fallback_dev = {
  49         .init_name = "fallback device",
  50         .coherent_dma_mask = ISA_DMA_BIT_MASK,
  51         .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
  52 };
  53 EXPORT_SYMBOL(x86_dma_fallback_dev);
  54
  55 /* Number of entries preallocated for DMA-API debugging */
  56 #define PREALLOC_DMA_DEBUG_ENTRIES       32768
  57
  58 int dma_set_mask(struct device *dev, u64 mask)
  59 {
  60         if (!dev->dma_mask || !dma_supported(dev, mask))
  61                 return -EIO;
  62
  63         *dev->dma_mask = mask;
  64
  65         return 0;
  66 }
  67 EXPORT_SYMBOL(dma_set_mask);
  68
  69 #if defined(CONFIG_X86_64) && !defined(CONFIG_NUMA)
  70 static __initdata void *dma32_bootmem_ptr;
  71 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
  72
  73 static int __init parse_dma32_size_opt(char *p)
  74 {
  75         if (!p)
  76                 return -EINVAL;
  77         dma32_bootmem_size = memparse(p, &p);
  78         return 0;
  79 }
  80 early_param("dma32_size", parse_dma32_size_opt);
  81
  82 void __init dma32_reserve_bootmem(void)
  83 {
  84         unsigned long size, align;
  85         if (max_pfn <= MAX_DMA32_PFN)
  86                 return;
  87
  88         /*
  89          * check aperture_64.c allocate_aperture() for reason about
  90          * using 512M as goal
  91          */
  92         align = 64ULL<<20;
  93         size = roundup(dma32_bootmem_size, align);
  94         dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
  95                                  512ULL<<20);
  96         /*
  97          * Kmemleak should not scan this block as it may not be mapped via the
  98          * kernel direct mapping.
  99          */
 100         kmemleak_ignore(dma32_bootmem_ptr);
 101         if (dma32_bootmem_ptr)
 102                 dma32_bootmem_size = size;
 103         else
 104                 dma32_bootmem_size = 0;
 105 }
 106 static void __init dma32_free_bootmem(void)
 107 {
 108
 109         if (max_pfn <= MAX_DMA32_PFN)
 110                 return;
 111
 112         if (!dma32_bootmem_ptr)
 113                 return;
 114
 115         free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
 116
 117         dma32_bootmem_ptr = NULL;
 118         dma32_bootmem_size = 0;
 119 }
 120 #else
 121 void __init dma32_reserve_bootmem(void)
 122 {
 123 }
 124 static void __init dma32_free_bootmem(void)
 125 {
 126 }
 127
 128 #endif
 129
 130 void __init pci_iommu_alloc(void)
 131 {
 132         /* free the range so iommu could get some range less than 4G */
 133         dma32_free_bootmem();
 134
 135         if (pci_swiotlb_detect())
 136                 goto out;
 137
 138         gart_iommu_hole_init();
 139
 140         detect_calgary();
 141
 142         detect_intel_iommu();
 143
 144         /* needs to be called after gart_iommu_hole_init */
 145         amd_iommu_detect();
 146 out:
 147         pci_swiotlb_init();
 148 }
 149
 150 void *dma_generic_alloc_coherent(struct device *dev, size_t size,
 151                                  dma_addr_t *dma_addr, gfp_t flag)
 152 {
 153         unsigned long dma_mask;
 154         struct page *page;
 155         dma_addr_t addr;
 156
 157         dma_mask = dma_alloc_coherent_mask(dev, flag);
 158
 159         flag |= __GFP_ZERO;
 160 again:
 161         page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
 162         if (!page)
 163                 return NULL;
 164
 165         addr = page_to_phys(page);
 166         if (addr + size > dma_mask) {
 167                 __free_pages(page, get_order(size));
 168
 169                 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
 170                         flag = (flag & ~GFP_DMA32) | GFP_DMA;
 171                         goto again;
 172                 }
 173
 174                 return NULL;
 175         }
 176
 177         *dma_addr = addr;
 178         return page_address(page);
 179 }
 180
 181 /*
 182  * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 183  * documentation.
 184  */
 185 static __init int iommu_setup(char *p)
 186 {
 187         iommu_merge = 1;
 188
 189         if (!p)
 190                 return -EINVAL;
 191
 192         while (*p) {
 193                 if (!strncmp(p, "off", 3))
 194                         no_iommu = 1;
 195                 /* gart_parse_options has more force support */
 196                 if (!strncmp(p, "force", 5))
 197                         force_iommu = 1;
 198                 if (!strncmp(p, "noforce", 7)) {
 199                         iommu_merge = 0;
 200                         force_iommu = 0;
 201                 }
 202
 203                 if (!strncmp(p, "biomerge", 8)) {
 204                         iommu_merge = 1;
 205                         force_iommu = 1;
 206                 }
 207                 if (!strncmp(p, "panic", 5))
 208                         panic_on_overflow = 1;
 209                 if (!strncmp(p, "nopanic", 7))
 210                         panic_on_overflow = 0;
 211                 if (!strncmp(p, "merge", 5)) {
 212                         iommu_merge = 1;
 213                         force_iommu = 1;
 214                 }
 215                 if (!strncmp(p, "nomerge", 7))
 216                         iommu_merge = 0;
 217                 if (!strncmp(p, "forcesac", 8))
 218                         iommu_sac_force = 1;
 219                 if (!strncmp(p, "allowdac", 8))
 220                         forbid_dac = 0;
 221                 if (!strncmp(p, "nodac", 5))
 222                         forbid_dac = 1;
 223                 if (!strncmp(p, "usedac", 6)) {
 224                         forbid_dac = -1;
 225                         return 1;
 226                 }
 227 #ifdef CONFIG_SWIOTLB
 228                 if (!strncmp(p, "soft", 4))
 229                         swiotlb = 1;
 230 #endif
 231                 if (!strncmp(p, "pt", 2))
 232                         iommu_pass_through = 1;
 233
 234                 gart_parse_options(p);
 235
 236 #ifdef CONFIG_CALGARY_IOMMU
 237                 if (!strncmp(p, "calgary", 7))
 238                         use_calgary = 1;
 239 #endif /* CONFIG_CALGARY_IOMMU */
 240
 241                 p += strcspn(p, ",");
 242                 if (*p == ',')
 243                         ++p;
 244         }
 245         return 0;
 246 }
 247 early_param("iommu", iommu_setup);
 248
 249 int dma_supported(struct device *dev, u64 mask)
 250 {
 251         struct dma_map_ops *ops = get_dma_ops(dev);
 252
 253 #ifdef CONFIG_PCI
 254         if (mask > 0xffffffff && forbid_dac > 0) {
 255                 dev_info(dev, "PCI: Disallowing DAC for device\n");
 256                 return 0;
 257         }
 258 #endif
 259
 260         if (ops->dma_supported)
 261                 return ops->dma_supported(dev, mask);
 262
 263         /* Copied from i386. Doesn't make much sense, because it will
 264            only work for pci_alloc_coherent.
 265            The caller just has to use GFP_DMA in this case. */
 266         if (mask < DMA_BIT_MASK(24))
 267                 return 0;
 268
 269         /* Tell the device to use SAC when IOMMU force is on.  This
 270            allows the driver to use cheaper accesses in some cases.
 271
 272            Problem with this is that if we overflow the IOMMU area and
 273            return DAC as fallback address the device may not handle it
 274            correctly.
 275
 276            As a special case some controllers have a 39bit address
 277            mode that is as efficient as 32bit (aic79xx). Don't force
 278            SAC for these.  Assume all masks <= 40 bits are of this
 279            type. Normally this doesn't make any difference, but gives
 280            more gentle handling of IOMMU overflow. */
 281         if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
 282                 dev_info(dev, "Force SAC with mask %Lx\n", mask);
 283                 return 0;
 284         }
 285
 286         return 1;
 287 }
 288 EXPORT_SYMBOL(dma_supported);
 289
 290 static int __init pci_iommu_init(void)
 291 {
 292         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
 293
 294 #ifdef CONFIG_PCI
 295         dma_debug_add_bus(&pci_bus_type);
 296 #endif
 297         x86_init.iommu.iommu_init();
 298
 299         if (swiotlb) {
 300                 printk(KERN_INFO "PCI-DMA: "
 301                        "Using software bounce buffering for IO (SWIOTLB)\n");
 302                 swiotlb_print_info();
 303         } else
 304                 swiotlb_free();
 305
 306         return 0;
 307 }
 308 /* Must execute after PCI subsystem */
 309 rootfs_initcall(pci_iommu_init);
 310
 311 #ifdef CONFIG_PCI
 312 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
 313
 314 static __devinit void via_no_dac(struct pci_dev *dev)
 315 {
 316         if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
 317                 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
 318                 forbid_dac = 1;
 319         }
 320 }
 321 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
 322 #endif