X-Git-Url: https://git.proxmox.com/?a=blobdiff_plain;f=drivers%2Fmedia%2Fpci%2Fintel%2Fipu-mmu.c;h=ad84c7b6344143b7129239260ea42bc93aa75181;hb=fad9119b95b10d02fae6bac52d60e60bb8a1d346;hp=baa9826f950092cd50b33a7b23b86e0f96dfa15d;hpb=85301793534e54c882bcf98a5d63d14c7380166e;p=mirror_ubuntu-jammy-kernel.git

diff --git a/drivers/media/pci/intel/ipu-mmu.c b/drivers/media/pci/intel/ipu-mmu.c
index baa9826f9500..ad84c7b63441 100644
--- a/drivers/media/pci/intel/ipu-mmu.c
+++ b/drivers/media/pci/intel/ipu-mmu.c
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0
-// Copyright (C) 2013 - 2020 Intel Corporation
+// Copyright (C) 2013 - 2021 Intel Corporation
 
 #include <asm/cacheflush.h>
 
@@ -40,87 +40,7 @@
 /* The range of stream ID i in L2 cache is from 0 to 15 */
 #define MMUV2_REG_L2_STREAMID(i)	(0x4c + ((i) * 4))
 
-/* ZLW Enable for each stream in L1 MMU AT where i : 0..15 */
-#define MMUV2_AT_REG_L1_ZLW_EN_SID(i)		(0x100 + ((i) * 0x20))
-
-/* ZLW 1D mode Enable for each stream in L1 MMU AT where i : 0..15 */
-#define MMUV2_AT_REG_L1_ZLW_1DMODE_SID(i)	(0x100 + ((i) * 0x20) + 0x0004)
-
-/* Set ZLW insertion N pages ahead per stream 1D where i : 0..15 */
-#define MMUV2_AT_REG_L1_ZLW_INS_N_AHEAD_SID(i)	(0x100 + ((i) * 0x20) + 0x0008)
-
-/* ZLW 2D mode Enable for each stream in L1 MMU AT where i : 0..15 */
-#define MMUV2_AT_REG_L1_ZLW_2DMODE_SID(i)	(0x100 + ((i) * 0x20) + 0x0010)
-
-/* ZLW Insertion for each stream in L1 MMU AT where i : 0..15 */
-#define MMUV2_AT_REG_L1_ZLW_INSERTION(i)	(0x100 + ((i) * 0x20) + 0x000c)
-
-#define MMUV2_AT_REG_L1_FW_ZLW_FIFO		(0x100 + \
-			(IPU_MMU_MAX_TLB_L1_STREAMS * 0x20) + 0x003c)
-
-/* FW ZLW has prioty - needed for ZLW invalidations */
-#define MMUV2_AT_REG_L1_FW_ZLW_PRIO		(0x100 + \
-			(IPU_MMU_MAX_TLB_L1_STREAMS * 0x20))
-
 #define TBL_PHYS_ADDR(a)	((phys_addr_t)(a) << ISP_PADDR_SHIFT)
-#define TBL_VIRT_ADDR(a)	phys_to_virt(TBL_PHYS_ADDR(a))
-
-static void zlw_invalidate(struct ipu_mmu *mmu, struct ipu_mmu_hw *mmu_hw)
-{
-	unsigned int retry = 0;
-	unsigned int i, j;
-	int ret;
-
-	for (i = 0; i < mmu_hw->nr_l1streams; i++) {
-		/* We need to invalidate only the zlw enabled stream IDs */
-		if (mmu_hw->l1_zlw_en[i]) {
-			/*
-			 * Maximum 16 blocks per L1 stream
-			 * Write trash buffer iova offset to the FW_ZLW
-			 * register. This will trigger pre-fetching of next 16
-			 * pages from the page table. So we need to increment
-			 * iova address by 16 * 4K to trigger the next 16 pages.
-			 * Once this loop is completed, the L1 cache will be
-			 * filled with trash buffer translation.
-			 *
-			 * TODO: Instead of maximum 16 blocks, use the allocated
-			 * block size
-			 */
-			for (j = 0; j < mmu_hw->l1_block_sz[i]; j++)
-				writel(mmu->iova_addr_trash +
-					   j * MMUV2_TRASH_L1_BLOCK_OFFSET,
-					   mmu_hw->base +
-					   MMUV2_AT_REG_L1_ZLW_INSERTION(i));
-
-			/*
-			 * Now we need to fill the L2 cache entry. L2 cache
-			 * entries will be automatically updated, based on the
-			 * L1 entry. The above loop for L1 will update only one
-			 * of the two entries in L2 as the L1 is under 4MB
-			 * range. To force the other entry in L2 to update, we
-			 * just need to trigger another pre-fetch which is
-			 * outside the above 4MB range.
-			 */
-			writel(mmu->iova_addr_trash +
-				   MMUV2_TRASH_L2_BLOCK_OFFSET,
-				   mmu_hw->base +
-				   MMUV2_AT_REG_L1_ZLW_INSERTION(0));
-		}
-	}
-
-	/*
-	 * Wait until AT is ready. FIFO read should return 2 when AT is ready.
-	 * Retry value of 1000 is just by guess work to avoid the forever loop.
-	 */
-	do {
-		if (retry > 1000) {
-			dev_err(mmu->dev, "zlw invalidation failed\n");
-			return;
-		}
-		ret = readl(mmu_hw->base + MMUV2_AT_REG_L1_FW_ZLW_FIFO);
-		retry++;
-	} while (ret != 2);
-}
 
 static void tlb_invalidate(struct ipu_mmu *mmu)
 {
@@ -139,22 +59,21 @@ static void tlb_invalidate(struct ipu_mmu *mmu)
 		 * MMUs on successive invalidate calls, we need to first do a
 		 * read to the page table base before writing the invalidate
 		 * register. MMUs which need to implement this WA, will have
-		 * the insert_read_before_invalidate flasg set as true.
+		 * the insert_read_before_invalidate flags set as true.
 		 * Disregard the return value of the read.
 		 */
 		if (mmu->mmu_hw[i].insert_read_before_invalidate)
 			readl(mmu->mmu_hw[i].base + REG_L1_PHYS);
 
-		/* Normal invalidate or zlw invalidate */
-		if (mmu->mmu_hw[i].zlw_invalidate) {
-			/* trash buffer must be mapped by now, just in case! */
-			WARN_ON(!mmu->iova_addr_trash);
-
-			zlw_invalidate(mmu, &mmu->mmu_hw[i]);
-		} else {
-			writel(0xffffffff, mmu->mmu_hw[i].base +
-				   REG_TLB_INVALIDATE);
-		}
+		writel(0xffffffff, mmu->mmu_hw[i].base +
+		       REG_TLB_INVALIDATE);
+		/*
+		 * The TLB invalidation is a "single cycle" (IOMMU clock cycles)
+		 * When the actual MMIO write reaches the IPU TLB Invalidate
+		 * register, wmb() will force the TLB invalidate out if the CPU
+		 * attempts to update the IOMMU page table (or sooner).
+		 */
+		wmb();
 	}
 	spin_unlock_irqrestore(&mmu->ready_lock, flags);
 }
@@ -164,47 +83,162 @@ static void page_table_dump(struct ipu_mmu_info *mmu_info)
 {
 	u32 l1_idx;
 
-	pr_debug("begin IOMMU page table dump\n");
+	dev_dbg(mmu_info->dev, "begin IOMMU page table dump\n");
 
 	for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) {
 		u32 l2_idx;
 		u32 iova = (phys_addr_t)l1_idx << ISP_L1PT_SHIFT;
 
-		if (mmu_info->pgtbl[l1_idx] == mmu_info->dummy_l2_tbl)
+		if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval)
 			continue;
-		pr_debug("l1 entry %u; iovas 0x%8.8x--0x%8.8x, at %p\n",
-			 l1_idx, iova, iova + ISP_PAGE_SIZE,
-			 (void *)TBL_PHYS_ADDR(mmu_info->pgtbl[l1_idx]));
+		dev_dbg(mmu_info->dev,
+			"l1 entry %u; iovas 0x%8.8x-0x%8.8x, at %p\n",
+			l1_idx, iova, iova + ISP_PAGE_SIZE,
+			(void *)TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]));
 
 		for (l2_idx = 0; l2_idx < ISP_L2PT_PTES; l2_idx++) {
-			u32 *l2_pt = TBL_VIRT_ADDR(mmu_info->pgtbl[l1_idx]);
+			u32 *l2_pt = mmu_info->l2_pts[l1_idx];
 			u32 iova2 = iova + (l2_idx << ISP_L2PT_SHIFT);
 
-			if (l2_pt[l2_idx] == mmu_info->dummy_page)
+			if (l2_pt[l2_idx] == mmu_info->dummy_page_pteval)
 				continue;
 
-			pr_debug("\tl2 entry %u; iova 0x%8.8x, phys %p\n",
-				 l2_idx, iova2,
-				 (void *)TBL_PHYS_ADDR(l2_pt[l2_idx]));
+			dev_dbg(mmu_info->dev,
+				"\tl2 entry %u; iova 0x%8.8x, phys %p\n",
+				l2_idx, iova2,
+				(void *)TBL_PHYS_ADDR(l2_pt[l2_idx]));
 		}
 	}
 
-	pr_debug("end IOMMU page table dump\n");
+	dev_dbg(mmu_info->dev, "end IOMMU page table dump\n");
 }
 #endif /* DEBUG */
 
-static u32 *alloc_page_table(struct ipu_mmu_info *mmu_info, bool l1)
+static dma_addr_t map_single(struct ipu_mmu_info *mmu_info, void *ptr)
+{
+	dma_addr_t dma;
+
+	dma = dma_map_single(mmu_info->dev, ptr, PAGE_SIZE, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(mmu_info->dev, dma))
+		return 0;
+
+	return dma;
+}
+
+static int get_dummy_page(struct ipu_mmu_info *mmu_info)
+{
+	dma_addr_t dma;
+	void *pt = (void *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
+
+	if (!pt)
+		return -ENOMEM;
+
+	dev_dbg(mmu_info->dev, "%s get_zeroed_page() == %p\n", __func__, pt);
+
+	dma = map_single(mmu_info, pt);
+	if (!dma) {
+		dev_err(mmu_info->dev, "Failed to map dummy page\n");
+		goto err_free_page;
+	}
+
+	mmu_info->dummy_page = pt;
+	mmu_info->dummy_page_pteval = dma >> ISP_PAGE_SHIFT;
+
+	return 0;
+
+err_free_page:
+	free_page((unsigned long)pt);
+	return -ENOMEM;
+}
+
+static void free_dummy_page(struct ipu_mmu_info *mmu_info)
+{
+	dma_unmap_single(mmu_info->dev,
+			 TBL_PHYS_ADDR(mmu_info->dummy_page_pteval),
+			 PAGE_SIZE, DMA_BIDIRECTIONAL);
+	free_page((unsigned long)mmu_info->dummy_page);
+}
+
+static int alloc_dummy_l2_pt(struct ipu_mmu_info *mmu_info)
+{
+	dma_addr_t dma;
+	u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
+	int i;
+
+	if (!pt)
+		return -ENOMEM;
+
+	dev_dbg(mmu_info->dev, "%s get_zeroed_page() == %p\n", __func__, pt);
+
+	dma = map_single(mmu_info, pt);
+	if (!dma) {
+		dev_err(mmu_info->dev, "Failed to map l2pt page\n");
+		goto err_free_page;
+	}
+
+	for (i = 0; i < ISP_L2PT_PTES; i++)
+		pt[i] = mmu_info->dummy_page_pteval;
+
+	mmu_info->dummy_l2_pt = pt;
+	mmu_info->dummy_l2_pteval = dma >> ISP_PAGE_SHIFT;
+
+	return 0;
+
+err_free_page:
+	free_page((unsigned long)pt);
+	return -ENOMEM;
+}
+
+static void free_dummy_l2_pt(struct ipu_mmu_info *mmu_info)
+{
+	dma_unmap_single(mmu_info->dev,
+			 TBL_PHYS_ADDR(mmu_info->dummy_l2_pteval),
+			 PAGE_SIZE, DMA_BIDIRECTIONAL);
+	free_page((unsigned long)mmu_info->dummy_l2_pt);
+}
+
+static u32 *alloc_l1_pt(struct ipu_mmu_info *mmu_info)
 {
+	dma_addr_t dma;
 	u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
 	int i;
 
 	if (!pt)
 		return NULL;
 
-	pr_debug("get_zeroed_page() == %p\n", pt);
+	dev_dbg(mmu_info->dev, "%s get_zeroed_page() == %p\n", __func__, pt);
 
 	for (i = 0; i < ISP_L1PT_PTES; i++)
-		pt[i] = l1 ? mmu_info->dummy_l2_tbl : mmu_info->dummy_page;
+		pt[i] = mmu_info->dummy_l2_pteval;
+
+	dma = map_single(mmu_info, pt);
+	if (!dma) {
+		dev_err(mmu_info->dev, "Failed to map l1pt page\n");
+		goto err_free_page;
+	}
+
+	mmu_info->l1_pt_dma = dma >> ISP_PADDR_SHIFT;
+	dev_dbg(mmu_info->dev, "l1 pt %p mapped at %llx\n", pt, dma);
+
+	return pt;
+
+err_free_page:
+	free_page((unsigned long)pt);
+	return NULL;
+}
+
+static u32 *alloc_l2_pt(struct ipu_mmu_info *mmu_info)
+{
+	u32 *pt = (u32 *)get_zeroed_page(GFP_ATOMIC | GFP_DMA32);
+	int i;
+
+	if (!pt)
+		return NULL;
+
+	dev_dbg(mmu_info->dev, "%s get_zeroed_page() == %p\n", __func__, pt);
+
+	for (i = 0; i < ISP_L1PT_PTES; i++)
+		pt[i] = mmu_info->dummy_page_pteval;
 
 	return pt;
 }
@@ -213,60 +247,69 @@ static int l2_map(struct ipu_mmu_info *mmu_info, unsigned long iova,
 		  phys_addr_t paddr, size_t size)
 {
 	u32 l1_idx = iova >> ISP_L1PT_SHIFT;
-	u32 l1_entry = mmu_info->pgtbl[l1_idx];
-	u32 *l2_pt;
+	u32 l1_entry;
+	u32 *l2_pt, *l2_virt;
 	u32 iova_start = iova;
 	unsigned int l2_idx;
 	unsigned long flags;
+	dma_addr_t dma;
 
-	pr_debug("mapping l2 page table for l1 index %u (iova %8.8x)\n",
-		 l1_idx, (u32)iova);
+	dev_dbg(mmu_info->dev,
+		"mapping l2 page table for l1 index %u (iova %8.8x)\n",
+		l1_idx, (u32)iova);
 
 	spin_lock_irqsave(&mmu_info->lock, flags);
-	if (l1_entry == mmu_info->dummy_l2_tbl) {
-		u32 *l2_virt = alloc_page_table(mmu_info, false);
+	l1_entry = mmu_info->l1_pt[l1_idx];
+	if (l1_entry == mmu_info->dummy_l2_pteval) {
+		l2_virt = mmu_info->l2_pts[l1_idx];
+		if (likely(!l2_virt)) {
+			l2_virt = alloc_l2_pt(mmu_info);
+			if (!l2_virt) {
+				spin_unlock_irqrestore(&mmu_info->lock, flags);
+				return -ENOMEM;
+			}
+		}
 
-		if (!l2_virt) {
+		dma = map_single(mmu_info, l2_virt);
+		if (!dma) {
+			dev_err(mmu_info->dev, "Failed to map l2pt page\n");
+			free_page((unsigned long)l2_virt);
 			spin_unlock_irqrestore(&mmu_info->lock, flags);
-			return -ENOMEM;
+			return -EINVAL;
 		}
 
-		l1_entry = virt_to_phys(l2_virt) >> ISP_PADDR_SHIFT;
-		pr_debug("allocated page for l1_idx %u\n", l1_idx);
+		l1_entry = dma >> ISP_PADDR_SHIFT;
 
-		if (mmu_info->pgtbl[l1_idx] == mmu_info->dummy_l2_tbl) {
-			mmu_info->pgtbl[l1_idx] = l1_entry;
-#ifdef CONFIG_X86
-			clflush_cache_range(&mmu_info->pgtbl[l1_idx],
-					    sizeof(mmu_info->pgtbl[l1_idx]));
-#endif /* CONFIG_X86 */
-		} else {
-			free_page((unsigned long)TBL_VIRT_ADDR(l1_entry));
-		}
+		dev_dbg(mmu_info->dev, "page for l1_idx %u %p allocated\n",
+			l1_idx, l2_virt);
+		mmu_info->l1_pt[l1_idx] = l1_entry;
+		mmu_info->l2_pts[l1_idx] = l2_virt;
+		clflush_cache_range(&mmu_info->l1_pt[l1_idx],
+				    sizeof(mmu_info->l1_pt[l1_idx]));
 	}
 
-	l2_pt = TBL_VIRT_ADDR(mmu_info->pgtbl[l1_idx]);
+	l2_pt = mmu_info->l2_pts[l1_idx];
 
-	pr_debug("l2_pt at %p\n", l2_pt);
+	dev_dbg(mmu_info->dev, "l2_pt at %p with dma 0x%x\n", l2_pt, l1_entry);
 
 	paddr = ALIGN(paddr, ISP_PAGE_SIZE);
 
 	l2_idx = (iova_start & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT;
 
-	pr_debug("l2_idx %u, phys 0x%8.8x\n", l2_idx, l2_pt[l2_idx]);
-	if (l2_pt[l2_idx] != mmu_info->dummy_page) {
+	dev_dbg(mmu_info->dev, "l2_idx %u, phys 0x%8.8x\n", l2_idx,
+		l2_pt[l2_idx]);
+	if (l2_pt[l2_idx] != mmu_info->dummy_page_pteval) {
 		spin_unlock_irqrestore(&mmu_info->lock, flags);
-		return -EBUSY;
+		return -EINVAL;
 	}
 
 	l2_pt[l2_idx] = paddr >> ISP_PADDR_SHIFT;
 
-#ifdef CONFIG_X86
 	clflush_cache_range(&l2_pt[l2_idx], sizeof(l2_pt[l2_idx]));
-#endif /* CONFIG_X86 */
 	spin_unlock_irqrestore(&mmu_info->lock, flags);
 
-	pr_debug("l2 index %u mapped as 0x%8.8x\n", l2_idx, l2_pt[l2_idx]);
+	dev_dbg(mmu_info->dev, "l2 index %u mapped as 0x%8.8x\n", l2_idx,
+		l2_pt[l2_idx]);
 
 	return 0;
 }
@@ -277,9 +320,9 @@ static int __ipu_mmu_map(struct ipu_mmu_info *mmu_info, unsigned long iova,
 	u32 iova_start = round_down(iova, ISP_PAGE_SIZE);
 	u32 iova_end = ALIGN(iova + size, ISP_PAGE_SIZE);
 
-	pr_debug
-	    ("mapping iova 0x%8.8x--0x%8.8x, size %zu at paddr 0x%10.10llx\n",
-	     iova_start, iova_end, size, paddr);
+	dev_dbg(mmu_info->dev,
+		"mapping iova 0x%8.8x--0x%8.8x, size %zu at paddr 0x%10.10llx\n",
+		iova_start, iova_end, size, paddr);
 
 	return l2_map(mmu_info, iova_start, paddr, size);
 }
@@ -288,34 +331,37 @@ static size_t l2_unmap(struct ipu_mmu_info *mmu_info, unsigned long iova,
 		       phys_addr_t dummy, size_t size)
 {
 	u32 l1_idx = iova >> ISP_L1PT_SHIFT;
-	u32 *l2_pt = TBL_VIRT_ADDR(mmu_info->pgtbl[l1_idx]);
+	u32 *l2_pt;
 	u32 iova_start = iova;
 	unsigned int l2_idx;
 	size_t unmapped = 0;
+	unsigned long flags;
 
-	pr_debug("unmapping l2 page table for l1 index %u (iova 0x%8.8lx)\n",
-		 l1_idx, iova);
-
-	if (mmu_info->pgtbl[l1_idx] == mmu_info->dummy_l2_tbl)
-		return -EINVAL;
+	dev_dbg(mmu_info->dev, "unmapping l2 page table for l1 index %u (iova 0x%8.8lx)\n",
+		l1_idx, iova);
 
-	pr_debug("l2_pt at %p\n", l2_pt);
+	spin_lock_irqsave(&mmu_info->lock, flags);
+	if (mmu_info->l1_pt[l1_idx] == mmu_info->dummy_l2_pteval) {
+		spin_unlock_irqrestore(&mmu_info->lock, flags);
+		dev_err(mmu_info->dev,
+			"unmap iova 0x%8.8lx l1 idx %u which was not mapped\n",
+			iova, l1_idx);
+		return 0;
+	}
 
 	for (l2_idx = (iova_start & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT;
 	     (iova_start & ISP_L1PT_MASK) + (l2_idx << ISP_PAGE_SHIFT)
 	     < iova_start + size && l2_idx < ISP_L2PT_PTES; l2_idx++) {
-		unsigned long flags;
+		l2_pt = mmu_info->l2_pts[l1_idx];
+		dev_dbg(mmu_info->dev,
+			"unmap l2 index %u with pteval 0x%10.10llx\n",
+			l2_idx, TBL_PHYS_ADDR(l2_pt[l2_idx]));
+		l2_pt[l2_idx] = mmu_info->dummy_page_pteval;
 
-		pr_debug("l2 index %u unmapped, was 0x%10.10llx\n",
-			 l2_idx, TBL_PHYS_ADDR(l2_pt[l2_idx]));
-		spin_lock_irqsave(&mmu_info->lock, flags);
-		l2_pt[l2_idx] = mmu_info->dummy_page;
-		spin_unlock_irqrestore(&mmu_info->lock, flags);
-#ifdef CONFIG_X86
 		clflush_cache_range(&l2_pt[l2_idx], sizeof(l2_pt[l2_idx]));
-#endif /* CONFIG_X86 */
 		unmapped++;
 	}
+	spin_unlock_irqrestore(&mmu_info->lock, flags);
 
 	return unmapped << ISP_PAGE_SHIFT;
 }
@@ -332,6 +378,7 @@ static int allocate_trash_buffer(struct ipu_mmu *mmu)
 	struct iova *iova;
 	u32 iova_addr;
 	unsigned int i;
+	dma_addr_t dma;
 	int ret;
 
 	/* Allocate 8MB in iova range */
@@ -342,6 +389,16 @@ static int allocate_trash_buffer(struct ipu_mmu *mmu)
 		return -ENOMEM;
 	}
 
+	dma = dma_map_page(mmu->dmap->mmu_info->dev, mmu->trash_page, 0,
+			   PAGE_SIZE, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(mmu->dmap->mmu_info->dev, dma)) {
+		dev_err(mmu->dmap->mmu_info->dev, "Failed to map trash page\n");
+		ret = -ENOMEM;
+		goto out_free_iova;
+	}
+
+	mmu->pci_trash_page = dma;
+
 	/*
 	 * Map the 8MB iova address range to the same physical trash page
 	 * mmu->trash_page which is already reserved at the probe
@@ -349,7 +406,7 @@ static int allocate_trash_buffer(struct ipu_mmu *mmu)
 	iova_addr = iova->pfn_lo;
 	for (i = 0; i < n_pages; i++) {
 		ret = ipu_mmu_map(mmu->dmap->mmu_info, iova_addr << PAGE_SHIFT,
-				  page_to_phys(mmu->trash_page), PAGE_SIZE);
+				  mmu->pci_trash_page, PAGE_SIZE);
 		if (ret) {
 			dev_err(mmu->dev,
 				"mapping trash buffer range failed\n");
@@ -359,15 +416,17 @@ static int allocate_trash_buffer(struct ipu_mmu *mmu)
 		iova_addr++;
 	}
 
-	/* save the address for the ZLW invalidation */
-	mmu->iova_addr_trash = iova->pfn_lo << PAGE_SHIFT;
+	mmu->iova_trash_page = iova->pfn_lo << PAGE_SHIFT;
 	dev_dbg(mmu->dev, "iova trash buffer for MMUID: %d is %u\n",
-		mmu->mmid, (unsigned int)mmu->iova_addr_trash);
+		mmu->mmid, (unsigned int)mmu->iova_trash_page);
 	return 0;
 
 out_unmap:
 	ipu_mmu_unmap(mmu->dmap->mmu_info, iova->pfn_lo << PAGE_SHIFT,
 		      (iova->pfn_hi - iova->pfn_lo + 1) << PAGE_SHIFT);
+	dma_unmap_page(mmu->dmap->mmu_info->dev, mmu->pci_trash_page,
+		       PAGE_SIZE, DMA_BIDIRECTIONAL);
+out_free_iova:
 	__free_iova(&mmu->dmap->iovad, iova);
 	return ret;
 }
@@ -389,9 +448,8 @@ int ipu_mmu_hw_init(struct ipu_mmu *mmu)
 		u16 block_addr;
 
 		/* Write page table address per MMU */
-		writel((phys_addr_t)virt_to_phys(mmu_info->pgtbl)
-			   >> ISP_PADDR_SHIFT,
-			   mmu->mmu_hw[i].base + REG_L1_PHYS);
+		writel((phys_addr_t)mmu_info->l1_pt_dma,
+		       mmu->mmu_hw[i].base + REG_L1_PHYS);
 
 		/* Set info bits per MMU */
 		writel(mmu->mmu_hw[i].info_bits,
@@ -423,20 +481,14 @@ int ipu_mmu_hw_init(struct ipu_mmu *mmu)
 		}
 	}
 
-	/*
-	 * Allocate 1 page of physical memory for the trash buffer.
-	 */
 	if (!mmu->trash_page) {
+		int ret;
+
 		mmu->trash_page = alloc_page(GFP_KERNEL);
 		if (!mmu->trash_page) {
 			dev_err(mmu->dev, "insufficient memory for trash buffer\n");
 			return -ENOMEM;
 		}
-	}
-
-	/* Allocate trash buffer, if not allocated. Only once per MMU */
-	if (!mmu->iova_addr_trash) {
-		int ret;
 
 		ret = allocate_trash_buffer(mmu);
 		if (ret) {
@@ -458,7 +510,7 @@ EXPORT_SYMBOL(ipu_mmu_hw_init);
 static struct ipu_mmu_info *ipu_mmu_alloc(struct ipu_device *isp)
 {
 	struct ipu_mmu_info *mmu_info;
-	void *ptr;
+	int ret;
 
 	mmu_info = kzalloc(sizeof(*mmu_info), GFP_KERNEL);
 	if (!mmu_info)
@@ -466,40 +518,44 @@ static struct ipu_mmu_info *ipu_mmu_alloc(struct ipu_device *isp)
 
 	mmu_info->aperture_start = 0;
 	mmu_info->aperture_end = DMA_BIT_MASK(isp->secure_mode ?
-				      IPU_MMU_ADDRESS_BITS :
-				      IPU_MMU_ADDRESS_BITS_NON_SECURE);
+					      IPU_MMU_ADDRESS_BITS :
+					      IPU_MMU_ADDRESS_BITS_NON_SECURE);
 	mmu_info->pgsize_bitmap = SZ_4K;
+	mmu_info->dev = &isp->pdev->dev;
 
-	ptr = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA32);
-	if (!ptr)
-		goto err_mem;
-
-	mmu_info->dummy_page = virt_to_phys(ptr) >> ISP_PAGE_SHIFT;
+	ret = get_dummy_page(mmu_info);
+	if (ret)
+		goto err_free_info;
 
-	ptr = alloc_page_table(mmu_info, false);
-	if (!ptr)
-		goto err;
+	ret = alloc_dummy_l2_pt(mmu_info);
+	if (ret)
+		goto err_free_dummy_page;
 
-	mmu_info->dummy_l2_tbl = virt_to_phys(ptr) >> ISP_PAGE_SHIFT;
+	mmu_info->l2_pts = vzalloc(ISP_L2PT_PTES * sizeof(*mmu_info->l2_pts));
+	if (!mmu_info->l2_pts)
+		goto err_free_dummy_l2_pt;
 
 	/*
 	 * We always map the L1 page table (a single page as well as
 	 * the L2 page tables).
 	 */
-	mmu_info->pgtbl = alloc_page_table(mmu_info, true);
-	if (!mmu_info->pgtbl)
-		goto err;
+	mmu_info->l1_pt = alloc_l1_pt(mmu_info);
+	if (!mmu_info->l1_pt)
+		goto err_free_l2_pts;
 
 	spin_lock_init(&mmu_info->lock);
 
-	pr_debug("domain initialised\n");
+	dev_dbg(mmu_info->dev, "domain initialised\n");
 
 	return mmu_info;
 
-err:
-	free_page((unsigned long)TBL_VIRT_ADDR(mmu_info->dummy_page));
-	free_page((unsigned long)TBL_VIRT_ADDR(mmu_info->dummy_l2_tbl));
-err_mem:
+err_free_l2_pts:
+	vfree(mmu_info->l2_pts);
+err_free_dummy_l2_pt:
+	free_dummy_l2_pt(mmu_info);
+err_free_dummy_page:
+	free_dummy_page(mmu_info);
+err_free_info:
 	kfree(mmu_info);
 
 	return NULL;
@@ -535,7 +591,7 @@ static struct ipu_dma_mapping *alloc_dma_mapping(struct ipu_device *isp)
 
 	kref_init(&dmap->ref);
 
-	pr_debug("alloc mapping\n");
+	dev_dbg(&isp->pdev->dev, "alloc mapping\n");
 
 	iova_cache_get();
 
@@ -545,15 +601,22 @@ static struct ipu_dma_mapping *alloc_dma_mapping(struct ipu_device *isp)
 phys_addr_t ipu_mmu_iova_to_phys(struct ipu_mmu_info *mmu_info,
 				 dma_addr_t iova)
 {
-	u32 *l2_pt = TBL_VIRT_ADDR(mmu_info->pgtbl[iova >> ISP_L1PT_SHIFT]);
+	unsigned long flags;
+	u32 *l2_pt;
+	phys_addr_t phy_addr;
 
-	return (phys_addr_t)l2_pt[(iova & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT]
-	    << ISP_PAGE_SHIFT;
+	spin_lock_irqsave(&mmu_info->lock, flags);
+	l2_pt = mmu_info->l2_pts[iova >> ISP_L1PT_SHIFT];
+	phy_addr = (phys_addr_t)l2_pt[(iova & ISP_L2PT_MASK) >> ISP_L2PT_SHIFT];
+	phy_addr <<= ISP_PAGE_SHIFT;
+	spin_unlock_irqrestore(&mmu_info->lock, flags);
+
+	return phy_addr;
 }
 
 /*
  * The following four functions are implemented based on iommu.c
- * drivers/iommu/iommu.c/iommu_pgsize().
+ * drivers/iommu/iommu.c:iommu_pgsize().
  */
 static size_t ipu_mmu_pgsize(unsigned long pgsize_bitmap,
 			     unsigned long addr_merge, size_t size)
@@ -588,7 +651,7 @@ static size_t ipu_mmu_pgsize(unsigned long pgsize_bitmap,
 	return pgsize;
 }
 
-/* drivers/iommu/iommu.c/iommu_unmap() */
+/* drivers/iommu/iommu.c:iommu_unmap() */
 size_t ipu_mmu_unmap(struct ipu_mmu_info *mmu_info, unsigned long iova,
 		     size_t size)
 {
@@ -621,7 +684,7 @@ size_t ipu_mmu_unmap(struct ipu_mmu_info *mmu_info, unsigned long iova,
 		if (!unmapped_page)
 			break;
 
-		dev_dbg(NULL, "unmapped: iova 0x%lx size 0x%zx\n",
+		dev_dbg(mmu_info->dev, "unmapped: iova 0x%lx size 0x%zx\n",
 			iova, unmapped_page);
 
 		iova += unmapped_page;
@@ -631,7 +694,7 @@ size_t ipu_mmu_unmap(struct ipu_mmu_info *mmu_info, unsigned long iova,
 	return unmapped;
 }
 
-/* drivers/iommu/iommu.c/iommu_map() */
+/* drivers/iommu/iommu.c:iommu_map() */
 int ipu_mmu_map(struct ipu_mmu_info *mmu_info, unsigned long iova,
 		phys_addr_t paddr, size_t size)
 {
@@ -652,19 +715,22 @@ int ipu_mmu_map(struct ipu_mmu_info *mmu_info, unsigned long iova,
 	 * size of the smallest page supported by the hardware
 	 */
 	if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) {
-		pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
-		       iova, &paddr, size, min_pagesz);
+		dev_err(mmu_info->dev,
+			"unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n",
+			iova, &paddr, size, min_pagesz);
 		return -EINVAL;
 	}
 
-	pr_debug("map: iova 0x%lx pa %pa size 0x%zx\n", iova, &paddr, size);
+	dev_dbg(mmu_info->dev, "map: iova 0x%lx pa %pa size 0x%zx\n",
+		iova, &paddr, size);
 
 	while (size) {
 		size_t pgsize = ipu_mmu_pgsize(mmu_info->pgsize_bitmap,
 					       iova | paddr, size);
 
-		pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",
-			 iova, &paddr, pgsize);
+		dev_dbg(mmu_info->dev,
+			"mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",
+			iova, &paddr, pgsize);
 
 		ret = __ipu_mmu_map(mmu_info, iova, paddr, pgsize);
 		if (ret)
@@ -689,9 +755,9 @@ static void ipu_mmu_destroy(struct ipu_mmu *mmu)
 	struct iova *iova;
 	u32 l1_idx;
 
-	if (mmu->iova_addr_trash) {
+	if (mmu->iova_trash_page) {
 		iova = find_iova(&dmap->iovad,
-				 mmu->iova_addr_trash >> PAGE_SHIFT);
+				 mmu->iova_trash_page >> PAGE_SHIFT);
 		if (iova) {
 			/* unmap and free the trash buffer iova */
 			ipu_mmu_unmap(mmu_info, iova->pfn_lo << PAGE_SHIFT,
@@ -702,20 +768,27 @@ static void ipu_mmu_destroy(struct ipu_mmu *mmu)
 			dev_err(mmu->dev, "trash buffer iova not found.\n");
 		}
 
-		mmu->iova_addr_trash = 0;
-	}
-
-	if (mmu->trash_page)
+		mmu->iova_trash_page = 0;
+		dma_unmap_page(mmu_info->dev, mmu->pci_trash_page,
+			       PAGE_SIZE, DMA_BIDIRECTIONAL);
+		mmu->pci_trash_page = 0;
 		__free_page(mmu->trash_page);
+	}
 
-	for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++)
-		if (mmu_info->pgtbl[l1_idx] != mmu_info->dummy_l2_tbl)
-			free_page((unsigned long)
-				  TBL_VIRT_ADDR(mmu_info->pgtbl[l1_idx]));
+	for (l1_idx = 0; l1_idx < ISP_L1PT_PTES; l1_idx++) {
+		if (mmu_info->l1_pt[l1_idx] != mmu_info->dummy_l2_pteval) {
+			dma_unmap_single(mmu_info->dev,
+					 TBL_PHYS_ADDR(mmu_info->l1_pt[l1_idx]),
+					 PAGE_SIZE, DMA_BIDIRECTIONAL);
+			free_page((unsigned long)mmu_info->l2_pts[l1_idx]);
+		}
+	}
 
-	free_page((unsigned long)TBL_VIRT_ADDR(mmu_info->dummy_page));
-	free_page((unsigned long)TBL_VIRT_ADDR(mmu_info->dummy_l2_tbl));
-	free_page((unsigned long)mmu_info->pgtbl);
+	free_dummy_page(mmu_info);
+	dma_unmap_single(mmu_info->dev, mmu_info->l1_pt_dma,
+			 PAGE_SIZE, DMA_BIDIRECTIONAL);
+	free_page((unsigned long)mmu_info->dummy_l2_pt);
+	free_page((unsigned long)mmu_info->l1_pt);
 	kfree(mmu_info);
 }