iommu/vt-d: Invalidate PASID cache when root/context entry changed

BugLink: https://bugs.launchpad.net/bugs/1929455
[ Upstream commit c0474a606ecb9326227b4d68059942f9db88a897 ]

When the Intel IOMMU is operating in the scalable mode, some information
from the root and context table may be used to tag entries in the PASID
cache. Software should invalidate the PASID-cache when changing root or
context table entries.

Suggested-by: Ashok Raj <ashok.raj@intel.com>
Fixes: 7373a8cc38197 ("iommu/vt-d: Setup context and enable RID2PASID support")
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
Link: https://lore.kernel.org/r/20210320025415.641201-4-baolu.lu@linux.intel.com
Signed-off-by: Joerg Roedel <jroedel@suse.de>
Signed-off-by: Sasha Levin <sashal@kernel.org>
Signed-off-by: Kamal Mostafa <kamal@canonical.com>
Signed-off-by: Kelsey Skunberg <kelsey.skunberg@canonical.com>

diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c
index 82300b0d30748d95f600fa212e240ffd4379e012..93f17a8a42e2bbbe4f19681a854c7334b2c56eb6 100644 (file)
--- a/drivers/iommu/intel/iommu.c
+++ b/drivers/iommu/intel/iommu.c
@@ -1336,6 +1336,11 @@ static void iommu_set_root_entry(struct intel_iommu *iommu)
                      readl, (sts & DMA_GSTS_RTPS), sts);
 
        raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
+
+       iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
+       if (sm_supported(iommu))
+               qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0);
+       iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
 }
 
 void iommu_flush_write_buffer(struct intel_iommu *iommu)
@@ -2481,6 +2486,10 @@ static void domain_context_clear_one(struct intel_iommu *iommu, u8 bus, u8 devfn
                                   (((u16)bus) << 8) | devfn,
                                   DMA_CCMD_MASK_NOBIT,
                                   DMA_CCMD_DEVICE_INVL);
+
+       if (sm_supported(iommu))
+               qi_flush_pasid_cache(iommu, did_old, QI_PC_ALL_PASIDS, 0);
+
        iommu->flush.flush_iotlb(iommu,
                                 did_old,
                                 0,
@@ -3325,8 +3334,6 @@ static int __init init_dmars(void)
                register_pasid_allocator(iommu);
 #endif
                iommu_set_root_entry(iommu);
-               iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
-               iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
        }
 
 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
@@ -3516,12 +3523,7 @@ static int init_iommu_hw(void)
                }
 
                iommu_flush_write_buffer(iommu);
-
                iommu_set_root_entry(iommu);
-
-               iommu->flush.flush_context(iommu, 0, 0, 0,
-                                          DMA_CCMD_GLOBAL_INVL);
-               iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
                iommu_enable_translation(iommu);
                iommu_disable_protect_mem_regions(iommu);
        }
@@ -3849,8 +3851,6 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
                goto disable_iommu;
 
        iommu_set_root_entry(iommu);
-       iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
-       iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
        iommu_enable_translation(iommu);
 
        iommu_disable_protect_mem_regions(iommu);

diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index ecb35fdff03eade75874133b7c69b4a49ed3b15b..ce30ea103b8d6a3e2b47ffbc9d0d3fa7d4092718 100644 (file)
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -369,6 +369,7 @@ enum {
 /* PASID cache invalidation granu */
 #define QI_PC_ALL_PASIDS       0
 #define QI_PC_PASID_SEL                1
+#define QI_PC_GLOBAL           3
 
 #define QI_EIOTLB_ADDR(addr)   ((u64)(addr) & VTD_PAGE_MASK)
 #define QI_EIOTLB_IH(ih)       (((u64)ih) << 6)