ArmPkg/ArmLib: don't invalidate entire I-cache on range operation

Instead of cleaning the data cache to the PoU by virtual address and
subsequently invalidating the entire I-cache, invalidate only the
range that we just cleaned. This way, we don't invalidate other
cachelines unnecessarily.

Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Reviewed-by: Leif Lindholm <leif.lindholm@linaro.org>

diff --git a/ArmPkg/Include/Library/ArmLib.h b/ArmPkg/Include/Library/ArmLib.h
index 1689f0072db640747f0ca0205ffb1a2009f401e4..4608b0cccccc95785736d406c565da4463ef2674 100644 (file)
--- a/ArmPkg/Include/Library/ArmLib.h
+++ b/ArmPkg/Include/Library/ArmLib.h
@@ -183,13 +183,19 @@ ArmInvalidateDataCacheEntryByMVA (
 \r
 VOID\r
 EFIAPI\r
-ArmCleanDataCacheEntryToPoUByMVA(\r
+ArmCleanDataCacheEntryToPoUByMVA (\r
   IN  UINTN   Address\r
   );\r
 \r
 VOID\r
 EFIAPI\r
-ArmCleanDataCacheEntryByMVA(\r
+ArmInvalidateInstructionCacheEntryToPoUByMVA (\r
+  IN  UINTN   Address\r
+  );\r
+\r
+VOID\r
+EFIAPI\r
+ArmCleanDataCacheEntryByMVA (\r
 IN  UINTN   Address\r
 );\r
 \r

diff --git a/ArmPkg/Library/ArmCacheMaintenanceLib/ArmCacheMaintenanceLib.c b/ArmPkg/Library/ArmCacheMaintenanceLib/ArmCacheMaintenanceLib.c
index 1045f9068f4dc0d71d71785d46f6dac4d053004c..0759e38cd4880798449b1d69e6c4eee1363936df 100644 (file)
--- a/ArmPkg/Library/ArmCacheMaintenanceLib/ArmCacheMaintenanceLib.c
+++ b/ArmPkg/Library/ArmCacheMaintenanceLib/ArmCacheMaintenanceLib.c
@@ -17,15 +17,16 @@
 #include <Library/DebugLib.h>\r
 #include <Library/PcdLib.h>\r
 \r
+STATIC\r
 VOID\r
 CacheRangeOperation (\r
   IN  VOID            *Start,\r
   IN  UINTN           Length,\r
-  IN  LINE_OPERATION  LineOperation\r
+  IN  LINE_OPERATION  LineOperation,\r
+  IN  UINTN           LineLength\r
   )\r
 {\r
-  UINTN ArmCacheLineLength         = ArmDataCacheLineLength();\r
-  UINTN ArmCacheLineAlignmentMask  = ArmCacheLineLength - 1;\r
+  UINTN ArmCacheLineAlignmentMask  = LineLength - 1;\r
 \r
   // Align address (rounding down)\r
   UINTN AlignedAddress = (UINTN)Start - ((UINTN)Start & ArmCacheLineAlignmentMask);\r
@@ -34,7 +35,7 @@ CacheRangeOperation (
   // Perform the line operation on an address in each cache line\r
   while (AlignedAddress < EndAddress) {\r
     LineOperation(AlignedAddress);\r
-    AlignedAddress += ArmCacheLineLength;\r
+    AlignedAddress += LineLength;\r
   }\r
   ArmDataSynchronizationBarrier ();\r
 }\r
@@ -64,8 +65,14 @@ InvalidateInstructionCacheRange (
   IN      UINTN                     Length\r
   )\r
 {\r
-  CacheRangeOperation (Address, Length, ArmCleanDataCacheEntryToPoUByMVA);\r
-  ArmInvalidateInstructionCache ();\r
+  CacheRangeOperation (Address, Length, ArmCleanDataCacheEntryToPoUByMVA,\r
+    ArmDataCacheLineLength ());\r
+  CacheRangeOperation (Address, Length,\r
+    ArmInvalidateInstructionCacheEntryToPoUByMVA,\r
+    ArmInstructionCacheLineLength ());\r
+\r
+  ArmInstructionSynchronizationBarrier ();\r
+\r
   return Address;\r
 }\r
 \r
@@ -85,7 +92,8 @@ WriteBackInvalidateDataCacheRange (
   IN      UINTN                     Length\r
   )\r
 {\r
-  CacheRangeOperation(Address, Length, ArmCleanInvalidateDataCacheEntryByMVA);\r
+  CacheRangeOperation(Address, Length, ArmCleanInvalidateDataCacheEntryByMVA,\r
+    ArmDataCacheLineLength ());\r
   return Address;\r
 }\r
 \r
@@ -105,7 +113,8 @@ WriteBackDataCacheRange (
   IN      UINTN                     Length\r
   )\r
 {\r
-  CacheRangeOperation(Address, Length, ArmCleanDataCacheEntryByMVA);\r
+  CacheRangeOperation(Address, Length, ArmCleanDataCacheEntryByMVA,\r
+    ArmDataCacheLineLength ());\r
   return Address;\r
 }\r
 \r
@@ -116,6 +125,7 @@ InvalidateDataCacheRange (
   IN      UINTN                     Length\r
   )\r
 {\r
-  CacheRangeOperation(Address, Length, ArmInvalidateDataCacheEntryByMVA);\r
+  CacheRangeOperation(Address, Length, ArmInvalidateDataCacheEntryByMVA,\r
+    ArmDataCacheLineLength ());\r
   return Address;\r
 }\r

diff --git a/ArmPkg/Library/ArmLib/AArch64/AArch64Support.S b/ArmPkg/Library/ArmLib/AArch64/AArch64Support.S
index 43f7a795acecac5c1d11412cb21cf55d9b1c19a3..9441f47e30bae46332ae4a7c3de171ddac753e0b 100644 (file)
--- a/ArmPkg/Library/ArmLib/AArch64/AArch64Support.S
+++ b/ArmPkg/Library/ArmLib/AArch64/AArch64Support.S
@@ -23,6 +23,7 @@ GCC_ASM_EXPORT (ArmInvalidateInstructionCache)
 GCC_ASM_EXPORT (ArmInvalidateDataCacheEntryByMVA)\r
 GCC_ASM_EXPORT (ArmCleanDataCacheEntryByMVA)\r
 GCC_ASM_EXPORT (ArmCleanDataCacheEntryToPoUByMVA)\r
+GCC_ASM_EXPORT (ArmInvalidateInstructionCacheEntryToPoUByMVA)\r
 GCC_ASM_EXPORT (ArmCleanInvalidateDataCacheEntryByMVA)\r
 GCC_ASM_EXPORT (ArmInvalidateDataCacheEntryBySetWay)\r
 GCC_ASM_EXPORT (ArmCleanDataCacheEntryBySetWay)\r
@@ -80,6 +81,10 @@ ASM_PFX(ArmCleanDataCacheEntryToPoUByMVA):
   dc      cvau, x0    // Clean single data cache line to PoU\r
   ret\r
 \r
+ASM_PFX(ArmInvalidateInstructionCacheEntryToPoUByMVA):\r
+  ic      ivau, x0    // Invalidate single instruction cache line to PoU\r
+  ret\r
+\r
 \r
 ASM_PFX(ArmCleanInvalidateDataCacheEntryByMVA):\r
   dc      civac, x0   // Clean and invalidate single data cache line\r

diff --git a/ArmPkg/Library/ArmLib/ArmV7/ArmV7Support.S b/ArmPkg/Library/ArmLib/ArmV7/ArmV7Support.S
index 50c760f335dec601586b606519e85efd7def79f4..c765032c9e4db66dc874dd6714566c05e70c27a0 100644 (file)
--- a/ArmPkg/Library/ArmLib/ArmV7/ArmV7Support.S
+++ b/ArmPkg/Library/ArmLib/ArmV7/ArmV7Support.S
@@ -18,6 +18,7 @@
 \r
 GCC_ASM_EXPORT (ArmInvalidateInstructionCache)\r
 GCC_ASM_EXPORT (ArmInvalidateDataCacheEntryByMVA)\r
+GCC_ASM_EXPORT (ArmInvalidateInstructionCacheEntryToPoUByMVA)\r
 GCC_ASM_EXPORT (ArmCleanDataCacheEntryByMVA)\r
 GCC_ASM_EXPORT (ArmCleanDataCacheEntryToPoUByMVA)\r
 GCC_ASM_EXPORT (ArmCleanInvalidateDataCacheEntryByMVA)\r
@@ -74,6 +75,10 @@ ASM_PFX(ArmCleanDataCacheEntryToPoUByMVA):
   mcr     p15, 0, r0, c7, c11, 1  @clean single data cache line to PoU\r
   bx      lr\r
 \r
+ASM_PFX(ArmInvalidateInstructionCacheEntryToPoUByMVA):\r
+  mcr     p15, 0, r0, c7, c5, 1  @Invalidate single instruction cache line to PoU\r
+  mcr     p15, 0, r0, c7, c5, 7  @Invalidate branch predictor\r
+  bx      lr\r
 \r
 ASM_PFX(ArmCleanInvalidateDataCacheEntryByMVA):\r
   mcr     p15, 0, r0, c7, c14, 1  @clean and invalidate single data cache line\r

diff --git a/ArmPkg/Library/ArmLib/ArmV7/ArmV7Support.asm b/ArmPkg/Library/ArmLib/ArmV7/ArmV7Support.asm
index a460bd2da7a99dedf4377f6761d8e59597051dfe..2363ee45763299b75ffdc3d3071a0df5e4f7234e 100644 (file)
--- a/ArmPkg/Library/ArmLib/ArmV7/ArmV7Support.asm
+++ b/ArmPkg/Library/ArmLib/ArmV7/ArmV7Support.asm
@@ -34,6 +34,12 @@ CTRL_I_BIT      EQU     (1 << 12)
   bx      lr\r
 \r
 \r
+ RVCT_ASM_EXPORT ArmInvalidateInstructionCacheEntryToPoUByMVA\r
+  mcr     p15, 0, r0, c7, c5, 1   ; invalidate single instruction cache line to PoU\r
+  mcr     p15, 0, r0, c7, c5, 7   ; invalidate branch predictor\r
+  bx      lr\r
+\r
+\r
  RVCT_ASM_EXPORT ArmCleanDataCacheEntryToPoUByMVA\r
   mcr     p15, 0, r0, c7, c11, 1  ; clean single data cache line to PoU\r
   bx      lr\r