// SPDX-License-Identifier: GPL-2.0
-// Copyright (C) 2013 - 2020 Intel Corporation
+// Copyright (C) 2013 - 2021 Intel Corporation
#include <asm/cacheflush.h>
/* The range of stream ID i in L2 cache is from 0 to 15 */
#define MMUV2_REG_L2_STREAMID(i) (0x4c + ((i) * 4))
-/* ZLW Enable for each stream in L1 MMU AT where i : 0..15 */
-#define MMUV2_AT_REG_L1_ZLW_EN_SID(i) (0x100 + ((i) * 0x20))
-
-/* ZLW 1D mode Enable for each stream in L1 MMU AT where i : 0..15 */
-#define MMUV2_AT_REG_L1_ZLW_1DMODE_SID(i) (0x100 + ((i) * 0x20) + 0x0004)
-
-/* Set ZLW insertion N pages ahead per stream 1D where i : 0..15 */
-#define MMUV2_AT_REG_L1_ZLW_INS_N_AHEAD_SID(i) (0x100 + ((i) * 0x20) + 0x0008)
-
-/* ZLW 2D mode Enable for each stream in L1 MMU AT where i : 0..15 */
-#define MMUV2_AT_REG_L1_ZLW_2DMODE_SID(i) (0x100 + ((i) * 0x20) + 0x0010)
-
-/* ZLW Insertion for each stream in L1 MMU AT where i : 0..15 */
-#define MMUV2_AT_REG_L1_ZLW_INSERTION(i) (0x100 + ((i) * 0x20) + 0x000c)
-
-#define MMUV2_AT_REG_L1_FW_ZLW_FIFO (0x100 + \
- (IPU_MMU_MAX_TLB_L1_STREAMS * 0x20) + 0x003c)
-
-/* FW ZLW has prioty - needed for ZLW invalidations */
-#define MMUV2_AT_REG_L1_FW_ZLW_PRIO (0x100 + \
- (IPU_MMU_MAX_TLB_L1_STREAMS * 0x20))
-
#define TBL_PHYS_ADDR(a) ((phys_addr_t)(a) << ISP_PADDR_SHIFT)
-#define TBL_VIRT_ADDR(a) phys_to_virt(TBL_PHYS_ADDR(a))
-
-static void zlw_invalidate(struct ipu_mmu *mmu, struct ipu_mmu_hw *mmu_hw)
-{
- unsigned int retry = 0;
- unsigned int i, j;
- int ret;
-
- for (i = 0; i < mmu_hw->nr_l1streams; i++) {
- /* We need to invalidate only the zlw enabled stream IDs */
- if (mmu_hw->l1_zlw_en[i]) {
- /*
- * Maximum 16 blocks per L1 stream
- * Write trash buffer iova offset to the FW_ZLW
- * register. This will trigger pre-fetching of next 16
- * pages from the page table. So we need to increment
- * iova address by 16 * 4K to trigger the next 16 pages.
- * Once this loop is completed, the L1 cache will be
- * filled with trash buffer translation.
- *
- * TODO: Instead of maximum 16 blocks, use the allocated
- * block size
- */
- for (j = 0; j < mmu_hw->l1_block_sz[i]; j++)
- writel(mmu->iova_addr_trash +
- j * MMUV2_TRASH_L1_BLOCK_OFFSET,
- mmu_hw->base +
- MMUV2_AT_REG_L1_ZLW_INSERTION(i));
-
- /*
- * Now we need to fill the L2 cache entry. L2 cache
- * entries will be automatically updated, based on the
- * L1 entry. The above loop for L1 will update only one
- * of the two entries in L2 as the L1 is under 4MB
- * range. To force the other entry in L2 to update, we
- * just need to trigger another pre-fetch which is
- * outside the above 4MB range.
- */
- writel(mmu->iova_addr_trash +
- MMUV2_TRASH_L2_BLOCK_OFFSET,
- mmu_hw->base +
- MMUV2_AT_REG_L1_ZLW_INSERTION(0));
- }
- }
-
- /*
- * Wait until AT is ready. FIFO read should return 2 when AT is ready.
- * Retry value of 1000 is just by guess work to avoid the forever loop.
- */
- do {
- if (retry > 1000) {
- dev_err(mmu->dev, "zlw invalidation failed\n");
- return;
- }
- ret = readl(mmu_hw->base + MMUV2_AT_REG_L1_FW_ZLW_FIFO);
- retry++;
- } while (ret != 2);
-}
static void tlb_invalidate(struct ipu_mmu *mmu)
{
* MMUs on successive invalidate calls, we need to first do a
* read to the page table base before writing the invalidate
* register. MMUs which need to implement this WA, will have
- * the insert_read_before_invalidate flasg set as true.
+ * the insert_read_before_invalidate flags set as true.
* Disregard the return value of the read.
*/
if (mmu->mmu_hw[i].insert_read_before_invalidate)
readl(mmu->mmu_hw[i].base + REG_L1_PHYS);
- /* Normal invalidate or zlw invalidate */
- if (mmu->mmu_hw[i].zlw_invalidate) {
- /* trash buffer must be mapped by now, just in case! */
- WARN_ON(!mmu->iova_addr_trash);
-
- zlw_invalidate(mmu, &mmu->mmu_hw[i]);
- } else {
- writel(0xffffffff, mmu->mmu_hw[i].base +
- REG_TLB_INVALIDATE);
- }
+ writel(0xffffffff, mmu->mmu_hw[i].base +
+ REG_TLB_INVALIDATE);
+ /*
+ * The TLB invalidation is a "single cycle" (IOMMU clock cycles)
+ * When the actual MMIO write reaches the IPU TLB Invalidate
+ * register, wmb() will force the TLB invalidate out if the CPU
+ * attempts to update the IOMMU page table (or sooner).
+ */
+ wmb();
}
spin_unlock_irqrestore(&mmu->ready_lock, flags);
}
{
u32 l1_idx;
- pr_debug("begin IOMMU page table dump\n");
+ dev_dbg(mmu_info->dev, "begin IOMMU page table dump\n");
for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) {
u32 l2_idx;
u32 iova = (phys_addr_t)l1_idx << ISP_L1PT_SHIFT;
- if (mmu_info->pgtbl[l1_idx] == mmu_info->dummy_l2_tbl)
+ if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval)
continue;
- pr_debug("l1 entry %u; iovas 0x%8.8x--0x%8.8x, at %p\n",
- l1_idx, iova, iova + ISP_PAGE_SIZE,
- (void *)TBL_PHYS_ADDR(mmu_info->pgtbl[l1_idx]));
+ dev_dbg(mmu_info->dev,
+ "l1 entry %u; iovas 0x%8.8x-0x%8.8x, at %p\n",
+ l1_idx, iova, iova + ISP_PAGE_SIZE,
+ (void *)TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]));
for (l2_idx = 0; l2_idx < ISP_L2PT_PTES; l2_idx++) {
- u32 *l2_pt = TBL_VIRT_ADDR(mmu_info->pgtbl[l1_idx]);
+ u32 *l2_pt = mmu_info->l2_pts[l1_idx];
u32 iova2 = iova + (l2_idx << ISP_L2PT_SHIFT);
- if (l2_pt[l2_idx] == mmu_info->dummy_page)
+ if (l2_pt[l2_idx] == mmu_info->dummy_page_pteval)
continue;
- pr_debug("\tl2 entry %u; iova 0x%8.8x, phys %p\n",
- l2_idx, iova2,
- (void *)TBL_PHYS_ADDR(l2_pt[l2_idx]));
+ dev_dbg(mmu_info->dev,
+ "\tl2 entry %u; iova 0x%8.8x, phys %p\n",
+ l2_idx, iova2,
+ (void *)TBL_PHYS_ADDR(l2_pt[l2_idx]));
}
}
- pr_debug("end IOMMU page table dump\n");
+ dev_dbg(mmu_info->dev, "end IOMMU page table dump\n");
}
#endif /* DEBUG */
-static u32 *alloc_page_table(struct ipu_mmu_info *mmu_info, bool l1)
+static dma_addr_t map_single(struct ipu_mmu_info *mmu_info, void *ptr)
+{
+ dma_addr_t dma;
+
+ dma = dma_map_single(mmu_info->dev, ptr, PAGE_SIZE, DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(mmu_info->dev, dma))
+ return 0;
+
+ return dma;
+}
+
+static int get_dummy_page(struct ipu_mmu_info *mmu_info)
+{
+ dma_addr_t dma;
+ void *pt = (void *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
+
+ if (!pt)
+ return -ENOMEM;
+
+ dev_dbg(mmu_info->dev, "%s get_zeroed_page() == %p\n", __func__, pt);
+
+ dma = map_single(mmu_info, pt);
+ if (!dma) {
+ dev_err(mmu_info->dev, "Failed to map dummy page\n");
+ goto err_free_page;
+ }
+
+ mmu_info->dummy_page = pt;
+ mmu_info->dummy_page_pteval = dma >> ISP_PAGE_SHIFT;
+
+ return 0;
+
+err_free_page:
+ free_page((unsigned long)pt);
+ return -ENOMEM;
+}
+
+static void free_dummy_page(struct ipu_mmu_info *mmu_info)
+{
+ dma_unmap_single(mmu_info->dev,
+ TBL_PHYS_ADDR(mmu_info->dummy_page_pteval),
+ PAGE_SIZE, DMA_BIDIRECTIONAL);
+ free_page((unsigned long)mmu_info->dummy_page);
+}
+
+static int alloc_dummy_l2_pt(struct ipu_mmu_info *mmu_info)
+{
+ dma_addr_t dma;
+ u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
+ int i;
+
+ if (!pt)
+ return -ENOMEM;
+
+ dev_dbg(mmu_info->dev, "%s get_zeroed_page() == %p\n", __func__, pt);
+
+ dma = map_single(mmu_info, pt);
+ if (!dma) {
+ dev_err(mmu_info->dev, "Failed to map l2pt page\n");
+ goto err_free_page;
+ }
+
+ for (i = 0; i < ISP_L2PT_PTES; i++)
+ pt[i] = mmu_info->dummy_page_pteval;
+
+ mmu_info->dummy_l2_pt = pt;
+ mmu_info->dummy_l2_pteval = dma >> ISP_PAGE_SHIFT;
+
+ return 0;
+
+err_free_page:
+ free_page((unsigned long)pt);
+ return -ENOMEM;
+}
+
+static void free_dummy_l2_pt(struct ipu_mmu_info *mmu_info)
+{
+ dma_unmap_single(mmu_info->dev,
+ TBL_PHYS_ADDR(mmu_info->dummy_l2_pteval),
+ PAGE_SIZE, DMA_BIDIRECTIONAL);
+ free_page((unsigned long)mmu_info->dummy_l2_pt);
+}
+
+static u32 *alloc_l1_pt(struct ipu_mmu_info *mmu_info)
{
+ dma_addr_t dma;
u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
int i;
if (!pt)
return NULL;
- pr_debug("get_zeroed_page() == %p\n", pt);
+ dev_dbg(mmu_info->dev, "%s get_zeroed_page() == %p\n", __func__, pt);
for (i = 0; i < ISP_L1PT_PTES; i++)
- pt[i] = l1 ? mmu_info->dummy_l2_tbl : mmu_info->dummy_page;
+ pt[i] = mmu_info->dummy_l2_pteval;
+
+ dma = map_single(mmu_info, pt);
+ if (!dma) {
+ dev_err(mmu_info->dev, "Failed to map l1pt page\n");
+ goto err_free_page;
+ }
+
+ mmu_info->l1_pt_dma = dma >> ISP_PADDR_SHIFT;
+ dev_dbg(mmu_info->dev, "l1 pt %p mapped at %llx\n", pt, dma);
+
+ return pt;
+
+err_free_page:
+ free_page((unsigned long)pt);
+ return NULL;
+}
+
+static u32 *alloc_l2_pt(struct ipu_mmu_info *mmu_info)
+{
+ u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
+ int i;
+
+ if (!pt)
+ return NULL;
+
+ dev_dbg(mmu_info->dev, "%s get_zeroed_page() == %p\n", __func__, pt);
+
+ for (i = 0; i < ISP_L1PT_PTES; i++)
+ pt[i] = mmu_info->dummy_page_pteval;
return pt;
}
phys_addr_t paddr, size_t size)
{
u32 l1_idx = iova >> ISP_L1PT_SHIFT;
- u32 l1_entry = mmu_info->pgtbl[l1_idx];
- u32 *l2_pt;
+ u32 l1_entry;
+ u32 *l2_pt, *l2_virt;
u32 iova_start = iova;
unsigned int l2_idx;
unsigned long flags;
+ dma_addr_t dma;
- pr_debug("mapping l2 page table for l1 index %u (iova %8.8x)\n",
- l1_idx, (u32)iova);
+ dev_dbg(mmu_info->dev,
+ "mapping l2 page table for l1 index %u (iova %8.8x)\n",
+ l1_idx, (u32)iova);
spin_lock_irqsave(&mmu_info->lock, flags);
- if (l1_entry == mmu_info->dummy_l2_tbl) {
- u32 *l2_virt = alloc_page_table(mmu_info, false);
+ l1_entry = mmu_info->l1_pt[l1_idx];
+ if (l1_entry == mmu_info->dummy_l2_pteval) {
+ l2_virt = mmu_info->l2_pts[l1_idx];
+ if (likely(!l2_virt)) {
+ l2_virt = alloc_l2_pt(mmu_info);
+ if (!l2_virt) {
+ spin_unlock_irqrestore(&mmu_info->lock, flags);
+ return -ENOMEM;
+ }
+ }
- if (!l2_virt) {
+ dma = map_single(mmu_info, l2_virt);
+ if (!dma) {
+ dev_err(mmu_info->dev, "Failed to map l2pt page\n");
+ free_page((unsigned long)l2_virt);
spin_unlock_irqrestore(&mmu_info->lock, flags);
- return -ENOMEM;
+ return -EINVAL;
}
- l1_entry = virt_to_phys(l2_virt) >> ISP_PADDR_SHIFT;
- pr_debug("allocated page for l1_idx %u\n", l1_idx);
+ l1_entry = dma >> ISP_PADDR_SHIFT;
- if (mmu_info->pgtbl[l1_idx] == mmu_info->dummy_l2_tbl) {
- mmu_info->pgtbl[l1_idx] = l1_entry;
-#ifdef CONFIG_X86
- clflush_cache_range(&mmu_info->pgtbl[l1_idx],
- sizeof(mmu_info->pgtbl[l1_idx]));
-#endif /* CONFIG_X86 */
- } else {
- free_page((unsigned long)TBL_VIRT_ADDR(l1_entry));
- }
+ dev_dbg(mmu_info->dev, "page for l1_idx %u %p allocated\n",
+ l1_idx, l2_virt);
+ mmu_info->l1_pt[l1_idx] = l1_entry;
+ mmu_info->l2_pts[l1_idx] = l2_virt;
+ clflush_cache_range(&mmu_info->l1_pt[l1_idx],
+ sizeof(mmu_info->l1_pt[l1_idx]));
}
- l2_pt = TBL_VIRT_ADDR(mmu_info->pgtbl[l1_idx]);
+ l2_pt = mmu_info->l2_pts[l1_idx];
- pr_debug("l2_pt at %p\n", l2_pt);
+ dev_dbg(mmu_info->dev, "l2_pt at %p with dma 0x%x\n", l2_pt, l1_entry);
paddr = ALIGN(paddr, ISP_PAGE_SIZE);
l2_idx = (iova_start & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT;
- pr_debug("l2_idx %u, phys 0x%8.8x\n", l2_idx, l2_pt[l2_idx]);
- if (l2_pt[l2_idx] != mmu_info->dummy_page) {
+ dev_dbg(mmu_info->dev, "l2_idx %u, phys 0x%8.8x\n", l2_idx,
+ l2_pt[l2_idx]);
+ if (l2_pt[l2_idx] != mmu_info->dummy_page_pteval) {
spin_unlock_irqrestore(&mmu_info->lock, flags);
- return -EBUSY;
+ return -EINVAL;
}
l2_pt[l2_idx] = paddr >> ISP_PADDR_SHIFT;
-#ifdef CONFIG_X86
clflush_cache_range(&l2_pt[l2_idx], sizeof(l2_pt[l2_idx]));
-#endif /* CONFIG_X86 */
spin_unlock_irqrestore(&mmu_info->lock, flags);
- pr_debug("l2 index %u mapped as 0x%8.8x\n", l2_idx, l2_pt[l2_idx]);
+ dev_dbg(mmu_info->dev, "l2 index %u mapped as 0x%8.8x\n", l2_idx,
+ l2_pt[l2_idx]);
return 0;
}
u32 iova_start = round_down(iova, ISP_PAGE_SIZE);
u32 iova_end = ALIGN(iova + size, ISP_PAGE_SIZE);
- pr_debug
- ("mapping iova 0x%8.8x--0x%8.8x, size %zu at paddr 0x%10.10llx\n",
- iova_start, iova_end, size, paddr);
+ dev_dbg(mmu_info->dev,
+ "mapping iova 0x%8.8x--0x%8.8x, size %zu at paddr 0x%10.10llx\n",
+ iova_start, iova_end, size, paddr);
return l2_map(mmu_info, iova_start, paddr, size);
}
phys_addr_t dummy, size_t size)
{
u32 l1_idx = iova >> ISP_L1PT_SHIFT;
- u32 *l2_pt = TBL_VIRT_ADDR(mmu_info->pgtbl[l1_idx]);
+ u32 *l2_pt;
u32 iova_start = iova;
unsigned int l2_idx;
size_t unmapped = 0;
+ unsigned long flags;
- pr_debug("unmapping l2 page table for l1 index %u (iova 0x%8.8lx)\n",
- l1_idx, iova);
-
- if (mmu_info->pgtbl[l1_idx] == mmu_info->dummy_l2_tbl)
- return -EINVAL;
+ dev_dbg(mmu_info->dev, "unmapping l2 page table for l1 index %u (iova 0x%8.8lx)\n",
+ l1_idx, iova);
- pr_debug("l2_pt at %p\n", l2_pt);
+ spin_lock_irqsave(&mmu_info->lock, flags);
+ if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval) {
+ spin_unlock_irqrestore(&mmu_info->lock, flags);
+ dev_err(mmu_info->dev,
+ "unmap iova 0x%8.8lx l1 idx %u which was not mapped\n",
+ iova, l1_idx);
+ return 0;
+ }
for (l2_idx = (iova_start & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT;
(iova_start & ISP_L1PT_MASK) + (l2_idx << ISP_PAGE_SHIFT)
< iova_start + size && l2_idx < ISP_L2PT_PTES; l2_idx++) {
- unsigned long flags;
+ l2_pt = mmu_info->l2_pts[l1_idx];
+ dev_dbg(mmu_info->dev,
+ "unmap l2 index %u with pteval 0x%10.10llx\n",
+ l2_idx, TBL_PHYS_ADDR(l2_pt[l2_idx]));
+ l2_pt[l2_idx] = mmu_info->dummy_page_pteval;
- pr_debug("l2 index %u unmapped, was 0x%10.10llx\n",
- l2_idx, TBL_PHYS_ADDR(l2_pt[l2_idx]));
- spin_lock_irqsave(&mmu_info->lock, flags);
- l2_pt[l2_idx] = mmu_info->dummy_page;
- spin_unlock_irqrestore(&mmu_info->lock, flags);
-#ifdef CONFIG_X86
clflush_cache_range(&l2_pt[l2_idx], sizeof(l2_pt[l2_idx]));
-#endif /* CONFIG_X86 */
unmapped++;
}
+ spin_unlock_irqrestore(&mmu_info->lock, flags);
return unmapped << ISP_PAGE_SHIFT;
}
struct iova *iova;
u32 iova_addr;
unsigned int i;
+ dma_addr_t dma;
int ret;
/* Allocate 8MB in iova range */
return -ENOMEM;
}
+ dma = dma_map_page(mmu->dmap->mmu_info->dev, mmu->trash_page, 0,
+ PAGE_SIZE, DMA_BIDIRECTIONAL);
+ if (dma_mapping_error(mmu->dmap->mmu_info->dev, dma)) {
+ dev_err(mmu->dmap->mmu_info->dev, "Failed to map trash page\n");
+ ret = -ENOMEM;
+ goto out_free_iova;
+ }
+
+ mmu->pci_trash_page = dma;
+
/*
* Map the 8MB iova address range to the same physical trash page
* mmu->trash_page which is already reserved at the probe
iova_addr = iova->pfn_lo;
for (i = 0; i < n_pages; i++) {
ret = ipu_mmu_map(mmu->dmap->mmu_info, iova_addr << PAGE_SHIFT,
- page_to_phys(mmu->trash_page), PAGE_SIZE);
+ mmu->pci_trash_page, PAGE_SIZE);
if (ret) {
dev_err(mmu->dev,
"mapping trash buffer range failed\n");
iova_addr++;
}
- /* save the address for the ZLW invalidation */
- mmu->iova_addr_trash = iova->pfn_lo << PAGE_SHIFT;
+ mmu->iova_trash_page = iova->pfn_lo << PAGE_SHIFT;
dev_dbg(mmu->dev, "iova trash buffer for MMUID: %d is %u\n",
- mmu->mmid, (unsigned int)mmu->iova_addr_trash);
+ mmu->mmid, (unsigned int)mmu->iova_trash_page);
return 0;
out_unmap:
ipu_mmu_unmap(mmu->dmap->mmu_info, iova->pfn_lo << PAGE_SHIFT,
(iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT);
+ dma_unmap_page(mmu->dmap->mmu_info->dev, mmu->pci_trash_page,
+ PAGE_SIZE, DMA_BIDIRECTIONAL);
+out_free_iova:
__free_iova(&mmu->dmap->iovad, iova);
return ret;
}
u16 block_addr;
/* Write page table address per MMU */
- writel((phys_addr_t)virt_to_phys(mmu_info->pgtbl)
- >> ISP_PADDR_SHIFT,
- mmu->mmu_hw[i].base + REG_L1_PHYS);
+ writel((phys_addr_t)mmu_info->l1_pt_dma,
+ mmu->mmu_hw[i].base + REG_L1_PHYS);
/* Set info bits per MMU */
writel(mmu->mmu_hw[i].info_bits,
}
}
- /*
- * Allocate 1 page of physical memory for the trash buffer.
- */
if (!mmu->trash_page) {
+ int ret;
+
mmu->trash_page = alloc_page(GFP_KERNEL);
if (!mmu->trash_page) {
dev_err(mmu->dev, "insufficient memory for trash buffer\n");
return -ENOMEM;
}
- }
-
- /* Allocate trash buffer, if not allocated. Only once per MMU */
- if (!mmu->iova_addr_trash) {
- int ret;
ret = allocate_trash_buffer(mmu);
if (ret) {
static struct ipu_mmu_info *ipu_mmu_alloc(struct ipu_device *isp)
{
struct ipu_mmu_info *mmu_info;
- void *ptr;
+ int ret;
mmu_info = kzalloc(sizeof(*mmu_info), GFP_KERNEL);
if (!mmu_info)
mmu_info->aperture_start = 0;
mmu_info->aperture_end = DMA_BIT_MASK(isp->secure_mode ?
- IPU_MMU_ADDRESS_BITS :
- IPU_MMU_ADDRESS_BITS_NON_SECURE);
+ IPU_MMU_ADDRESS_BITS :
+ IPU_MMU_ADDRESS_BITS_NON_SECURE);
mmu_info->pgsize_bitmap = SZ_4K;
+ mmu_info->dev = &isp->pdev->dev;
- ptr = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA32);
- if (!ptr)
- goto err_mem;
-
- mmu_info->dummy_page = virt_to_phys(ptr) >> ISP_PAGE_SHIFT;
+ ret = get_dummy_page(mmu_info);
+ if (ret)
+ goto err_free_info;
- ptr = alloc_page_table(mmu_info, false);
- if (!ptr)
- goto err;
+ ret = alloc_dummy_l2_pt(mmu_info);
+ if (ret)
+ goto err_free_dummy_page;
- mmu_info->dummy_l2_tbl = virt_to_phys(ptr) >> ISP_PAGE_SHIFT;
+ mmu_info->l2_pts = vzalloc(ISP_L2PT_PTES * sizeof(*mmu_info->l2_pts));
+ if (!mmu_info->l2_pts)
+ goto err_free_dummy_l2_pt;
/*
* We always map the L1 page table (a single page as well as
* the L2 page tables).
*/
- mmu_info->pgtbl = alloc_page_table(mmu_info, true);
- if (!mmu_info->pgtbl)
- goto err;
+ mmu_info->l1_pt = alloc_l1_pt(mmu_info);
+ if (!mmu_info->l1_pt)
+ goto err_free_l2_pts;
spin_lock_init(&mmu_info->lock);
- pr_debug("domain initialised\n");
+ dev_dbg(mmu_info->dev, "domain initialised\n");
return mmu_info;
-err:
- free_page((unsigned long)TBL_VIRT_ADDR(mmu_info->dummy_page));
- free_page((unsigned long)TBL_VIRT_ADDR(mmu_info->dummy_l2_tbl));
-err_mem:
+err_free_l2_pts:
+ vfree(mmu_info->l2_pts);
+err_free_dummy_l2_pt:
+ free_dummy_l2_pt(mmu_info);
+err_free_dummy_page:
+ free_dummy_page(mmu_info);
+err_free_info:
kfree(mmu_info);
return NULL;
kref_init(&dmap->ref);
- pr_debug("alloc mapping\n");
+ dev_dbg(&isp->pdev->dev, "alloc mapping\n");
iova_cache_get();
phys_addr_t ipu_mmu_iova_to_phys(struct ipu_mmu_info *mmu_info,
dma_addr_t iova)
{
- u32 *l2_pt = TBL_VIRT_ADDR(mmu_info->pgtbl[iova >> ISP_L1PT_SHIFT]);
+ unsigned long flags;
+ u32 *l2_pt;
+ phys_addr_t phy_addr;
- return (phys_addr_t)l2_pt[(iova & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT]
- << ISP_PAGE_SHIFT;
+ spin_lock_irqsave(&mmu_info->lock, flags);
+ l2_pt = mmu_info->l2_pts[iova >> ISP_L1PT_SHIFT];
+ phy_addr = (phys_addr_t)l2_pt[(iova & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT];
+ phy_addr <<= ISP_PAGE_SHIFT;
+ spin_unlock_irqrestore(&mmu_info->lock, flags);
+
+ return phy_addr;
}
/*
* The following four functions are implemented based on iommu.c
- * drivers/iommu/iommu.c/iommu_pgsize().
+ * drivers/iommu/iommu.c:iommu_pgsize().
*/
static size_t ipu_mmu_pgsize(unsigned long pgsize_bitmap,
unsigned long addr_merge, size_t size)
return pgsize;
}
-/* drivers/iommu/iommu.c/iommu_unmap() */
+/* drivers/iommu/iommu.c:iommu_unmap() */
size_t ipu_mmu_unmap(struct ipu_mmu_info *mmu_info, unsigned long iova,
size_t size)
{
if (!unmapped_page)
break;
- dev_dbg(NULL, "unmapped: iova 0x%lx size 0x%zx\n",
+ dev_dbg(mmu_info->dev, "unmapped: iova 0x%lx size 0x%zx\n",
iova, unmapped_page);
iova += unmapped_page;
return unmapped;
}
-/* drivers/iommu/iommu.c/iommu_map() */
+/* drivers/iommu/iommu.c:iommu_map() */
int ipu_mmu_map(struct ipu_mmu_info *mmu_info, unsigned long iova,
phys_addr_t paddr, size_t size)
{
* size of the smallest page supported by the hardware
*/
if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
- pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
- iova, &paddr, size, min_pagesz);
+ dev_err(mmu_info->dev,
+ "unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
+ iova, &paddr, size, min_pagesz);
return -EINVAL;
}
- pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
+ dev_dbg(mmu_info->dev, "map: iova 0x%lx pa %pa size 0x%zx\n",
+ iova, &paddr, size);
while (size) {
size_t pgsize = ipu_mmu_pgsize(mmu_info->pgsize_bitmap,
iova | paddr, size);
- pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",
- iova, &paddr, pgsize);
+ dev_dbg(mmu_info->dev,
+ "mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",
+ iova, &paddr, pgsize);
ret = __ipu_mmu_map(mmu_info, iova, paddr, pgsize);
if (ret)
struct iova *iova;
u32 l1_idx;
- if (mmu->iova_addr_trash) {
+ if (mmu->iova_trash_page) {
iova = find_iova(&dmap->iovad,
- mmu->iova_addr_trash >> PAGE_SHIFT);
+ mmu->iova_trash_page >> PAGE_SHIFT);
if (iova) {
/* unmap and free the trash buffer iova */
ipu_mmu_unmap(mmu_info, iova->pfn_lo << PAGE_SHIFT,
dev_err(mmu->dev, "trash buffer iova not found.\n");
}
- mmu->iova_addr_trash = 0;
- }
-
- if (mmu->trash_page)
+ mmu->iova_trash_page = 0;
+ dma_unmap_page(mmu_info->dev, mmu->pci_trash_page,
+ PAGE_SIZE, DMA_BIDIRECTIONAL);
+ mmu->pci_trash_page = 0;
__free_page(mmu->trash_page);
+ }
- for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++)
- if (mmu_info->pgtbl[l1_idx] != mmu_info->dummy_l2_tbl)
- free_page((unsigned long)
- TBL_VIRT_ADDR(mmu_info->pgtbl[l1_idx]));
+ for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) {
+ if (mmu_info->l1_pt[l1_idx] != mmu_info->dummy_l2_pteval) {
+ dma_unmap_single(mmu_info->dev,
+ TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]),
+ PAGE_SIZE, DMA_BIDIRECTIONAL);
+ free_page((unsigned long)mmu_info->l2_pts[l1_idx]);
+ }
+ }
- free_page((unsigned long)TBL_VIRT_ADDR(mmu_info->dummy_page));
- free_page((unsigned long)TBL_VIRT_ADDR(mmu_info->dummy_l2_tbl));
- free_page((unsigned long)mmu_info->pgtbl);
+ free_dummy_page(mmu_info);
+ dma_unmap_single(mmu_info->dev, mmu_info->l1_pt_dma,
+ PAGE_SIZE, DMA_BIDIRECTIONAL);
+ free_page((unsigned long)mmu_info->dummy_l2_pt);
+ free_page((unsigned long)mmu_info->l1_pt);
kfree(mmu_info);
}
projects / mirror_ubuntu-jammy-kernel.git / blobdiff