powerpc/64s: Add support for a store forwarding barrier at kernel entry/exit

[mirror_ubuntu-bionic-kernel.git] / arch / powerpc / include / asm / exception-64s.h

diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h
index 7197b179c1b150ba819f13a1f7feb6c9a45da1f9..48732b1d2cae09373acf9a91b927f52b384283fd 100644 (file)
--- a/arch/powerpc/include/asm/exception-64s.h
+++ b/arch/powerpc/include/asm/exception-64s.h
@@ -74,6 +74,27 @@
  */
 #define EX_R3          EX_DAR
 
+#define STF_ENTRY_BARRIER_SLOT                                         \
+       STF_ENTRY_BARRIER_FIXUP_SECTION;                                \
+       nop;                                                            \
+       nop;                                                            \
+       nop
+
+#define STF_EXIT_BARRIER_SLOT                                          \
+       STF_EXIT_BARRIER_FIXUP_SECTION;                                 \
+       nop;                                                            \
+       nop;                                                            \
+       nop;                                                            \
+       nop;                                                            \
+       nop;                                                            \
+       nop
+
+/*
+ * r10 must be free to use, r13 must be paca
+ */
+#define INTERRUPT_TO_KERNEL                                            \
+       STF_ENTRY_BARRIER_SLOT
+
 /*
  * Macros for annotating the expected destination of (h)rfid
  *
@@ -90,16 +111,19 @@
        rfid
 
 #define RFI_TO_USER                                                    \
+       STF_EXIT_BARRIER_SLOT;                                          \
        RFI_FLUSH_SLOT;                                                 \
        rfid;                                                           \
        b       rfi_flush_fallback
 
 #define RFI_TO_USER_OR_KERNEL                                          \
+       STF_EXIT_BARRIER_SLOT;                                          \
        RFI_FLUSH_SLOT;                                                 \
        rfid;                                                           \
        b       rfi_flush_fallback
 
 #define RFI_TO_GUEST                                                   \
+       STF_EXIT_BARRIER_SLOT;                                          \
        RFI_FLUSH_SLOT;                                                 \
        rfid;                                                           \
        b       rfi_flush_fallback
@@ -108,21 +132,25 @@
        hrfid
 
 #define HRFI_TO_USER                                                   \
+       STF_EXIT_BARRIER_SLOT;                                          \
        RFI_FLUSH_SLOT;                                                 \
        hrfid;                                                          \
        b       hrfi_flush_fallback
 
 #define HRFI_TO_USER_OR_KERNEL                                         \
+       STF_EXIT_BARRIER_SLOT;                                          \
        RFI_FLUSH_SLOT;                                                 \
        hrfid;                                                          \
        b       hrfi_flush_fallback
 
 #define HRFI_TO_GUEST                                                  \
+       STF_EXIT_BARRIER_SLOT;                                          \
        RFI_FLUSH_SLOT;                                                 \
        hrfid;                                                          \
        b       hrfi_flush_fallback
 
 #define HRFI_TO_UNKNOWN                                                        \
+       STF_EXIT_BARRIER_SLOT;                                          \
        RFI_FLUSH_SLOT;                                                 \
        hrfid;                                                          \
        b       hrfi_flush_fallback
@@ -254,6 +282,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943)
 #define __EXCEPTION_PROLOG_1(area, extra, vec)                         \
        OPT_SAVE_REG_TO_PACA(area+EX_PPR, r9, CPU_FTR_HAS_PPR);         \
        OPT_SAVE_REG_TO_PACA(area+EX_CFAR, r10, CPU_FTR_CFAR);          \
+       INTERRUPT_TO_KERNEL;                                            \
        SAVE_CTR(r10, area);                                            \
        mfcr    r9;                                                     \
        extra(vec);                                                     \