iommu/io-pgtable-arm: Add support for non-strict mode

BugLink: https://bugs.launchpad.net/bugs/1806488
Non-strict mode is simply a case of skipping 'regular' leaf TLBIs, since
the sync is already factored out into ops->iotlb_sync at the core API
level. Non-leaf invalidations where we change the page table structure
itself still have to be issued synchronously in order to maintain walk
caches correctly.

To save having to reason about it too much, make sure the invalidation
in arm_lpae_split_blk_unmap() just performs its own unconditional sync
to minimise the window in which we're technically violating the break-
before-make requirement on a live mapping. This might work out redundant
with an outer-level sync for strict unmaps, but we'll never be splitting
blocks on a DMA fastpath anyway.

Signed-off-by: Zhen Lei <thunder.leizhen@huawei.com>
[rm: tweak comment, commit message, split_blk_unmap logic and barriers]
Signed-off-by: Robin Murphy <robin.murphy@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
(cherry picked from commit b6b65ca20bc93d14319f9b5cf98fd3c19a4244e3)
Signed-off-by: dann frazier <dann.frazier@canonical.com>
Acked-by: Stefan Bader <stefan.bader@canonical.com>
Acked-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>
Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>

diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index cef162772838b747edaee3739b7871d0d1af8788..262e0f35785c143cc956fbb23d398cd9bd3e4444 100644 (file)
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -553,6 +553,7 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
                tablep = iopte_deref(pte, data);
        } else if (unmap_idx >= 0) {
                io_pgtable_tlb_add_flush(&data->iop, iova, size, size, true);
+               io_pgtable_tlb_sync(&data->iop);
                return size;
        }
 
@@ -586,6 +587,13 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
                        io_pgtable_tlb_sync(iop);
                        ptep = iopte_deref(pte, data);
                        __arm_lpae_free_pgtable(data, lvl + 1, ptep);
+               } else if (iop->cfg.quirks & IO_PGTABLE_QUIRK_NON_STRICT) {
+                       /*
+                        * Order the PTE update against queueing the IOVA, to
+                        * guarantee that a flush callback from a different CPU
+                        * has observed it before the TLBIALL can be issued.
+                        */
+                       smp_wmb();
                } else {
                        io_pgtable_tlb_add_flush(iop, iova, size, size, true);
                }
@@ -740,7 +748,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
        u64 reg;
        struct arm_lpae_io_pgtable *data;
 
-       if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_DMA))
+       if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NO_DMA |
+                           IO_PGTABLE_QUIRK_NON_STRICT))
                return NULL;
 
        data = arm_lpae_alloc_pgtable(cfg);
@@ -829,7 +838,8 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie)
        struct arm_lpae_io_pgtable *data;
 
        /* The NS quirk doesn't apply at stage 2 */
-       if (cfg->quirks & ~IO_PGTABLE_QUIRK_NO_DMA)
+       if (cfg->quirks & ~(IO_PGTABLE_QUIRK_NO_DMA |
+                           IO_PGTABLE_QUIRK_NON_STRICT))
                return NULL;
 
        data = arm_lpae_alloc_pgtable(cfg);

diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h
index cd2e1eafffe6883f6b43cb38bc23fb1c31bd4895..edfaead1b236588b8cc0055a96e0042b041c0c8b 100644 (file)
--- a/drivers/iommu/io-pgtable.h
+++ b/drivers/iommu/io-pgtable.h
@@ -71,12 +71,17 @@ struct io_pgtable_cfg {
         *      be accessed by a fully cache-coherent IOMMU or CPU (e.g. for a
         *      software-emulated IOMMU), such that pagetable updates need not
         *      be treated as explicit DMA data.
+        *
+        * IO_PGTABLE_QUIRK_NON_STRICT: Skip issuing synchronous leaf TLBIs
+        *      on unmap, for DMA domains using the flush queue mechanism for
+        *      delayed invalidation.
         */
        #define IO_PGTABLE_QUIRK_ARM_NS         BIT(0)
        #define IO_PGTABLE_QUIRK_NO_PERMS       BIT(1)
        #define IO_PGTABLE_QUIRK_TLBI_ON_MAP    BIT(2)
        #define IO_PGTABLE_QUIRK_ARM_MTK_4GB    BIT(3)
        #define IO_PGTABLE_QUIRK_NO_DMA         BIT(4)
+       #define IO_PGTABLE_QUIRK_NON_STRICT     BIT(5)
        unsigned long                   quirks;
        unsigned long                   pgsize_bitmap;
        unsigned int                    ias;