#include <xen/xen.h>
#include <asm/iommu_table.h>
++
+#include <asm/xen/swiotlb-xen.h>
+ #ifdef CONFIG_X86_64
+ #include <asm/iommu.h>
+ #include <asm/dma.h>
+ #endif
+ #include <linux/export.h>
++
int xen_swiotlb __read_mostly;
static struct dma_map_ops xen_swiotlb_dma_ops = {
int __init pci_xen_swiotlb_detect(void)
{
+ if (!xen_pv_domain())
+ return 0;
+
/* If running as PV guest, either iommu=soft, or swiotlb=force will
* activate this IOMMU. If running as PV privileged, activate it
* irregardless.
*/
- if ((xen_initial_domain() || swiotlb || swiotlb_force) &&
- (xen_pv_domain()))
+ if ((xen_initial_domain() || swiotlb || swiotlb_force))
xen_swiotlb = 1;
/* If we are running under Xen, we MUST disable the native SWIOTLB.
* Don't worry about swiotlb_force flag activating the native, as
* the 'swiotlb' flag is the only one turning it on. */
- if (xen_pv_domain())
- swiotlb = 0;
+ swiotlb = 0;
+ #ifdef CONFIG_X86_64
+ /* pci_swiotlb_detect_4gb turns on native SWIOTLB if no_iommu == 0
+ * (so no iommu=X command line over-writes).
+ * Considering that PV guests do not want the *native SWIOTLB* but
+ * only Xen SWIOTLB it is not useful to us so set no_iommu=1 here.
+ */
+ if (max_pfn > MAX_DMA32_PFN)
+ no_iommu = 1;
+ #endif
return xen_swiotlb;
}
void __init pci_xen_swiotlb_init(void)
{
if (xen_swiotlb) {
- xen_swiotlb_init(1);
+ xen_swiotlb_init(1, true /* early */);
dma_ops = &xen_swiotlb_dma_ops;
/* Make sure ACS will be enabled */
pci_request_acs();
}
}
+
+ int pci_xen_swiotlb_init_late(void)
+ {
+ int rc;
+
+ if (xen_swiotlb)
+ return 0;
+
+ rc = xen_swiotlb_init(1, false /* late */);
+ if (rc)
+ return rc;
+
+ dma_ops = &xen_swiotlb_dma_ops;
+ /* Make sure ACS will be enabled */
+ pci_request_acs();
+
+ return 0;
+ }
+ EXPORT_SYMBOL_GPL(pci_xen_swiotlb_init_late);
+
IOMMU_INIT_FINISH(pci_xen_swiotlb_detect,
NULL,
pci_xen_swiotlb_init,
#include <linux/bitops.h>
#include <linux/time.h>
+ #include <asm/xen/swiotlb-xen.h>
#define INVALID_GRANT_REF (0)
#define INVALID_EVTCHN (-1)
return errno_to_pcibios_err(do_pci_op(pdev, &op));
}
-struct pci_ops pcifront_bus_ops = {
+static struct pci_ops pcifront_bus_ops = {
.read = pcifront_bus_read,
.write = pcifront_bus_write,
};
schedule_pcifront_aer_op(pdev);
return IRQ_HANDLED;
}
- static int pcifront_connect(struct pcifront_device *pdev)
+ static int pcifront_connect_and_init_dma(struct pcifront_device *pdev)
{
int err = 0;
dev_err(&pdev->xdev->dev, "PCI frontend already installed!\n");
err = -EEXIST;
}
-
spin_unlock(&pcifront_dev_lock);
+ if (!err && !swiotlb_nr_tbl()) {
+ err = pci_xen_swiotlb_init_late();
+ if (err)
+ dev_err(&pdev->xdev->dev, "Could not setup SWIOTLB!\n");
+ }
return err;
}
XenbusStateInitialised)
goto out;
- err = pcifront_connect(pdev);
+ err = pcifront_connect_and_init_dma(pdev);
if (err) {
xenbus_dev_fatal(pdev->xdev, err,
- "Error connecting PCI Frontend");
+ "Error setting up PCI Frontend");
goto out;
}
* Quick lookup value of the bus address of the IOTLB.
*/
-u64 start_dma_addr;
+static u64 start_dma_addr;
static dma_addr_t xen_phys_to_bus(phys_addr_t paddr)
{
} while (i < nslabs);
return 0;
}
+ static unsigned long xen_set_nslabs(unsigned long nr_tbl)
+ {
+ if (!nr_tbl) {
+ xen_io_tlb_nslabs = (64 * 1024 * 1024 >> IO_TLB_SHIFT);
+ xen_io_tlb_nslabs = ALIGN(xen_io_tlb_nslabs, IO_TLB_SEGSIZE);
+ } else
+ xen_io_tlb_nslabs = nr_tbl;
+
+ return xen_io_tlb_nslabs << IO_TLB_SHIFT;
+ }
- void __init xen_swiotlb_init(int verbose)
+ enum xen_swiotlb_err {
+ XEN_SWIOTLB_UNKNOWN = 0,
+ XEN_SWIOTLB_ENOMEM,
+ XEN_SWIOTLB_EFIXUP
+ };
+
+ static const char *xen_swiotlb_error(enum xen_swiotlb_err err)
+ {
+ switch (err) {
+ case XEN_SWIOTLB_ENOMEM:
+ return "Cannot allocate Xen-SWIOTLB buffer\n";
+ case XEN_SWIOTLB_EFIXUP:
+ return "Failed to get contiguous memory for DMA from Xen!\n"\
+ "You either: don't have the permissions, do not have"\
+ " enough free memory under 4GB, or the hypervisor memory"\
+ " is too fragmented!";
+ default:
+ break;
+ }
+ return "";
+ }
+ int __ref xen_swiotlb_init(int verbose, bool early)
{
- unsigned long bytes;
+ unsigned long bytes, order;
int rc = -ENOMEM;
- unsigned long nr_tbl;
- char *m = NULL;
+ enum xen_swiotlb_err m_ret = XEN_SWIOTLB_UNKNOWN;
unsigned int repeat = 3;
- nr_tbl = swiotlb_nr_tbl();
- if (nr_tbl)
- xen_io_tlb_nslabs = nr_tbl;
- else {
- xen_io_tlb_nslabs = (64 * 1024 * 1024 >> IO_TLB_SHIFT);
- xen_io_tlb_nslabs = ALIGN(xen_io_tlb_nslabs, IO_TLB_SEGSIZE);
- }
+ xen_io_tlb_nslabs = swiotlb_nr_tbl();
retry:
- bytes = xen_io_tlb_nslabs << IO_TLB_SHIFT;
-
+ bytes = xen_set_nslabs(xen_io_tlb_nslabs);
+ order = get_order(xen_io_tlb_nslabs << IO_TLB_SHIFT);
/*
* Get IO TLB memory from any location.
*/
- xen_io_tlb_start = alloc_bootmem_pages(PAGE_ALIGN(bytes));
+ if (early)
+ xen_io_tlb_start = alloc_bootmem_pages(PAGE_ALIGN(bytes));
+ else {
+ #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
+ #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
+ while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
+ xen_io_tlb_start = (void *)__get_free_pages(__GFP_NOWARN, order);
+ if (xen_io_tlb_start)
+ break;
+ order--;
+ }
+ if (order != get_order(bytes)) {
+ pr_warn("Warning: only able to allocate %ld MB "
+ "for software IO TLB\n", (PAGE_SIZE << order) >> 20);
+ xen_io_tlb_nslabs = SLABS_PER_PAGE << order;
+ bytes = xen_io_tlb_nslabs << IO_TLB_SHIFT;
+ }
+ }
if (!xen_io_tlb_start) {
- m = "Cannot allocate Xen-SWIOTLB buffer!\n";
+ m_ret = XEN_SWIOTLB_ENOMEM;
goto error;
}
xen_io_tlb_end = xen_io_tlb_start + bytes;
bytes,
xen_io_tlb_nslabs);
if (rc) {
- free_bootmem(__pa(xen_io_tlb_start), PAGE_ALIGN(bytes));
- m = "Failed to get contiguous memory for DMA from Xen!\n"\
- "You either: don't have the permissions, do not have"\
- " enough free memory under 4GB, or the hypervisor memory"\
- "is too fragmented!";
+ if (early)
+ free_bootmem(__pa(xen_io_tlb_start), PAGE_ALIGN(bytes));
+ else {
+ free_pages((unsigned long)xen_io_tlb_start, order);
+ xen_io_tlb_start = NULL;
+ }
+ m_ret = XEN_SWIOTLB_EFIXUP;
goto error;
}
start_dma_addr = xen_virt_to_bus(xen_io_tlb_start);
- swiotlb_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs, verbose);
-
- return;
+ if (early) {
+ swiotlb_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs, verbose);
+ rc = 0;
+ } else
+ rc = swiotlb_late_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs);
+ return rc;
error:
if (repeat--) {
xen_io_tlb_nslabs = max(1024UL, /* Min is 2MB */
(xen_io_tlb_nslabs << IO_TLB_SHIFT) >> 20);
goto retry;
}
- xen_raw_printk("%s (rc:%d)", m, rc);
- panic("%s (rc:%d)", m, rc);
+ pr_err("%s (rc:%d)", xen_swiotlb_error(m_ret), rc);
+ if (early)
+ panic("%s (rc:%d)", xen_swiotlb_error(m_ret), rc);
+ else
+ free_pages((unsigned long)xen_io_tlb_start, order);
+ return rc;
}
-
void *
xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
return ret;
if (hwdev && hwdev->coherent_dma_mask)
- dma_mask = hwdev->coherent_dma_mask;
+ dma_mask = dma_alloc_coherent_mask(hwdev, flags);
phys = virt_to_phys(ret);
dev_addr = xen_phys_to_bus(phys);
projects / mirror_ubuntu-bionic-kernel.git / commitdiff