ArmPkg/ArmMmuLib AARCH64: get rid of needless TLB invalidation

Currently, we always invalidate the TLBs entirely after making
any modification to the page tables. Now that we have introduced
strict memory permissions in quite a number of places, such
modifications occur much more often, and it is better for performance
to flush only those TLB entries that are actually affected by
the changes.

At the same time, relax some system wide data synchronization barriers
to non-shared. When running in UEFI, we don't share virtual address
translations with other masters, unless we are running under virt, but
in that case, the host will upgrade them as appropriate (by setting
an override at EL2)

Contributed-under: TianoCore Contribution Agreement 1.1
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: Leif Lindholm <leif.lindholm@linaro.org>

diff --git a/ArmPkg/Include/Library/ArmMmuLib.h b/ArmPkg/Include/Library/ArmMmuLib.h
index fb7fd006417cb6bc64d7304eeda05da14a2edd17..220039a779e0cac7afcb49a36fdddf5587ee4021 100644 (file)
--- a/ArmPkg/Include/Library/ArmMmuLib.h
+++ b/ArmPkg/Include/Library/ArmMmuLib.h
@@ -59,7 +59,8 @@ VOID
 EFIAPI\r
 ArmReplaceLiveTranslationEntry (\r
   IN  UINT64  *Entry,\r
-  IN  UINT64  Value\r
+  IN  UINT64  Value,\r
+  IN  UINT64  RegionStart\r
   );\r
 \r
 EFI_STATUS\r

diff --git a/ArmPkg/Library/ArmLib/AArch64/ArmLibSupport.S b/ArmPkg/Library/ArmLib/AArch64/ArmLibSupport.S
index b7173e00b039f9c5b59262f8f75dbaa3d9b9dca0..175fb58206b6652a24211bac859845fd9116a672 100644 (file)
--- a/ArmPkg/Library/ArmLib/AArch64/ArmLibSupport.S
+++ b/ArmPkg/Library/ArmLib/AArch64/ArmLibSupport.S
@@ -124,15 +124,15 @@ ASM_FUNC(ArmSetMAIR)
 //  IN VOID  *MVA                    // X1\r
 //  );\r
 ASM_FUNC(ArmUpdateTranslationTableEntry)\r
-   dc      civac, x0             // Clean and invalidate data line\r
-   dsb     sy\r
+   dsb     nshst\r
+   lsr     x1, x1, #12\r
    EL1_OR_EL2_OR_EL3(x0)\r
 1: tlbi    vaae1, x1             // TLB Invalidate VA , EL1\r
    b       4f\r
 2: tlbi    vae2, x1              // TLB Invalidate VA , EL2\r
    b       4f\r
 3: tlbi    vae3, x1              // TLB Invalidate VA , EL3\r
-4: dsb     sy\r
+4: dsb     nsh\r
    isb\r
    ret\r
 \r

diff --git a/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c b/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c
index d66df3e17a02a17c71b4ff947963de3f584320a6..3498f520e39ab7f0a4d26873bd312d52357aad47 100644 (file)
--- a/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c
+++ b/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibCore.c
@@ -129,13 +129,14 @@ STATIC
 VOID\r
 ReplaceLiveEntry (\r
   IN  UINT64  *Entry,\r
-  IN  UINT64  Value\r
+  IN  UINT64  Value,\r
+  IN  UINT64  RegionStart\r
   )\r
 {\r
   if (!ArmMmuEnabled ()) {\r
     *Entry = Value;\r
   } else {\r
-    ArmReplaceLiveTranslationEntry (Entry, Value);\r
+    ArmReplaceLiveTranslationEntry (Entry, Value, RegionStart);\r
   }\r
 }\r
 \r
@@ -296,7 +297,8 @@ GetBlockEntryListFromAddress (
 \r
         // Fill the BlockEntry with the new TranslationTable\r
         ReplaceLiveEntry (BlockEntry,\r
-          ((UINTN)TranslationTable & TT_ADDRESS_MASK_DESCRIPTION_TABLE) | TableAttributes | TT_TYPE_TABLE_ENTRY);\r
+          (UINTN)TranslationTable | TableAttributes | TT_TYPE_TABLE_ENTRY,\r
+          RegionStart);\r
       }\r
     } else {\r
       if (IndexLevel != PageLevel) {\r
@@ -375,6 +377,8 @@ UpdateRegionMapping (
       *BlockEntry &= BlockEntryMask;\r
       *BlockEntry |= (RegionStart & TT_ADDRESS_MASK_BLOCK_ENTRY) | Attributes | Type;\r
 \r
+      ArmUpdateTranslationTableEntry (BlockEntry, (VOID *)RegionStart);\r
+\r
       // Go to the next BlockEntry\r
       RegionStart += BlockEntrySize;\r
       RegionLength -= BlockEntrySize;\r
@@ -487,9 +491,6 @@ ArmSetMemoryAttributes (
     return Status;\r
   }\r
 \r
-  // Invalidate all TLB entries so changes are synced\r
-  ArmInvalidateTlb ();\r
-\r
   return EFI_SUCCESS;\r
 }\r
 \r
@@ -512,9 +513,6 @@ SetMemoryRegionAttribute (
     return Status;\r
   }\r
 \r
-  // Invalidate all TLB entries so changes are synced\r
-  ArmInvalidateTlb ();\r
-\r
   return EFI_SUCCESS;\r
 }\r
 \r

diff --git a/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibReplaceEntry.S b/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibReplaceEntry.S
index 90192df24f55520dbb2bdf9ce74ff64111c2d2f1..8b447e3d666fd36946b83f77218d50ff68df96c0 100644 (file)
--- a/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibReplaceEntry.S
+++ b/ArmPkg/Library/ArmMmuLib/AArch64/ArmMmuLibReplaceEntry.S
@@ -32,13 +32,14 @@
   dmb   sy\r
   dc    ivac, x0\r
 \r
-  // flush the TLBs\r
+  // flush translations for the target address from the TLBs\r
+  lsr   x2, x2, #12\r
   .if   \el == 1\r
-  tlbi  vmalle1\r
+  tlbi  vaae1, x2\r
   .else\r
-  tlbi  alle\el\r
+  tlbi  vae\el, x2\r
   .endif\r
-  dsb   sy\r
+  dsb   nsh\r
 \r
   // re-enable the MMU\r
   msr   sctlr_el\el, x8\r
@@ -48,19 +49,20 @@
 //VOID\r
 //ArmReplaceLiveTranslationEntry (\r
 //  IN  UINT64  *Entry,\r
-//  IN  UINT64  Value\r
+//  IN  UINT64  Value,\r
+//  IN  UINT64  Address\r
 //  )\r
 ASM_FUNC(ArmReplaceLiveTranslationEntry)\r
 \r
   // disable interrupts\r
-  mrs   x2, daif\r
+  mrs   x4, daif\r
   msr   daifset, #0xf\r
   isb\r
 \r
   // clean and invalidate first so that we don't clobber\r
   // adjacent entries that are dirty in the caches\r
   dc    civac, x0\r
-  dsb   ish\r
+  dsb   nsh\r
 \r
   EL1_OR_EL2_OR_EL3(x3)\r
 1:__replace_entry 1\r
@@ -69,7 +71,7 @@ ASM_FUNC(ArmReplaceLiveTranslationEntry)
   b     4f\r
 3:__replace_entry 3\r
 \r
-4:msr   daif, x2\r
+4:msr   daif, x4\r
   ret\r
 \r
 ASM_GLOBAL ASM_PFX(ArmReplaceLiveTranslationEntrySize)\r