powerpc/64s: Add support for a store forwarding barrier at kernel entry/exit

[mirror_ubuntu-bionic-kernel.git] / arch / powerpc / lib / feature-fixups.c

diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c
index acfc81e3aeb8970d964be8efb1de95ffee7bf18e..29b68f256980c2491eeb7de8c5627198345db017 100644 (file)
--- a/arch/powerpc/lib/feature-fixups.c
+++ b/arch/powerpc/lib/feature-fixups.c
@@ -117,6 +117,111 @@ void do_feature_fixups(unsigned long value, void *fixup_start, void *fixup_end)
 }
 
 #ifdef CONFIG_PPC_BOOK3S_64
+void do_stf_entry_barrier_fixups(enum stf_barrier_type types)
+{
+       unsigned int instrs[3], *dest;
+       long *start, *end;
+       int i;
+
+       start = PTRRELOC(&__start___stf_entry_barrier_fixup),
+       end = PTRRELOC(&__stop___stf_entry_barrier_fixup);
+
+       instrs[0] = 0x60000000; /* nop */
+       instrs[1] = 0x60000000; /* nop */
+       instrs[2] = 0x60000000; /* nop */
+
+       i = 0;
+       if (types & STF_BARRIER_FALLBACK) {
+               instrs[i++] = 0x7d4802a6; /* mflr r10           */
+               instrs[i++] = 0x60000000; /* branch patched below */
+               instrs[i++] = 0x7d4803a6; /* mtlr r10           */
+       } else if (types & STF_BARRIER_EIEIO) {
+               instrs[i++] = 0x7c0006ac; /* eieio              */
+       } else if (types & STF_BARRIER_SYNC_ORI) {
+               instrs[i++] = 0x7c0004ac; /* hwsync             */
+               instrs[i++] = 0xe94d0000; /* ld r10,0(r13)      */
+               instrs[i++] = 0x63ff0000; /* ori 31,31,0 speculation barrier */
+       }
+
+       for (i = 0; start < end; start++, i++) {
+               dest = (void *)start + *start;
+
+               pr_devel("patching dest %lx\n", (unsigned long)dest);
+
+               patch_instruction(dest, instrs[0]);
+
+               if (types & STF_BARRIER_FALLBACK)
+                       patch_branch(dest + 1, (unsigned long)&stf_barrier_fallback,
+                                    BRANCH_SET_LINK);
+               else
+                       patch_instruction(dest + 1, instrs[1]);
+
+               patch_instruction(dest + 2, instrs[2]);
+       }
+
+       printk(KERN_DEBUG "stf-barrier: patched %d entry locations (%s barrier)\n", i,
+               (types == STF_BARRIER_NONE)                  ? "no" :
+               (types == STF_BARRIER_FALLBACK)              ? "fallback" :
+               (types == STF_BARRIER_EIEIO)                 ? "eieio" :
+               (types == (STF_BARRIER_SYNC_ORI))            ? "hwsync"
+                                                          : "unknown");
+}
+
+void do_stf_exit_barrier_fixups(enum stf_barrier_type types)
+{
+       unsigned int instrs[6], *dest;
+       long *start, *end;
+       int i;
+
+       start = PTRRELOC(&__start___stf_exit_barrier_fixup),
+       end = PTRRELOC(&__stop___stf_exit_barrier_fixup);
+
+       instrs[0] = 0x60000000; /* nop */
+       instrs[1] = 0x60000000; /* nop */
+       instrs[2] = 0x60000000; /* nop */
+       instrs[3] = 0x60000000; /* nop */
+       instrs[4] = 0x60000000; /* nop */
+       instrs[5] = 0x60000000; /* nop */
+
+       i = 0;
+       if (types & STF_BARRIER_FALLBACK || types & STF_BARRIER_SYNC_ORI) {
+               instrs[i++] = 0x7db243a6; /* mtsprg 2,r13       */
+               instrs[i++] = 0x7db142a6; /* mfsprg r13,1       */
+               instrs[i++] = 0x7c0004ac; /* hwsync             */
+               instrs[i++] = 0xe9ad0000; /* ld r13,0(r13)      */
+               instrs[i++] = 0x63ff0000; /* ori 31,31,0 speculation barrier */
+               instrs[i++] = 0x7db242a6; /* mfsprg r13,2       */
+       } else if (types & STF_BARRIER_EIEIO) {
+               instrs[i++] = 0x7c0006ac; /* eieio              */
+       }
+
+       for (i = 0; start < end; start++, i++) {
+               dest = (void *)start + *start;
+
+               pr_devel("patching dest %lx\n", (unsigned long)dest);
+
+               patch_instruction(dest, instrs[0]);
+               patch_instruction(dest + 1, instrs[1]);
+               patch_instruction(dest + 2, instrs[2]);
+               patch_instruction(dest + 3, instrs[3]);
+               patch_instruction(dest + 4, instrs[4]);
+               patch_instruction(dest + 5, instrs[5]);
+       }
+       printk(KERN_DEBUG "stf-barrier: patched %d exit locations (%s barrier)\n", i,
+               (types == STF_BARRIER_NONE)                  ? "no" :
+               (types == STF_BARRIER_FALLBACK)              ? "fallback" :
+               (types == STF_BARRIER_EIEIO)                 ? "eieio" :
+               (types == (STF_BARRIER_SYNC_ORI))            ? "hwsync"
+                                                          : "unknown");
+}
+
+
+void do_stf_barrier_fixups(enum stf_barrier_type types)
+{
+       do_stf_entry_barrier_fixups(types);
+       do_stf_exit_barrier_fixups(types);
+}
+
 void do_rfi_flush_fixups(enum l1d_flush_type types)
 {
        unsigned int instrs[3], *dest;