powerpc: Introduce msgsnd/doorbell barrier primitives

BugLink: http://bugs.launchpad.net/bugs/1691973
POWER9 changes requirements and adds new instructions for
synchronization.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
(cherry picked from commit b87ac0218355a83abb899a0022bb2e5252879fc0)
Signed-off-by: Gustavo Walbon <gwalbon@linux.vnet.ibm.com>
Acked-by: Stefan Bader <stefan.bader@canonical.com>
Acked-by: Seth Forshee <seth.forshee@canonical.com>
Signed-off-by: Kleber Sacilotto de Souza <kleber.souza@canonical.com>

diff --git a/arch/powerpc/include/asm/dbell.h b/arch/powerpc/include/asm/dbell.h
index a66eb596940cb7fb5102676e25e4b04ebbf9ffb1..350694a1a6e592761bcdfba93b4c004465698574 100644 (file)
--- a/arch/powerpc/include/asm/dbell.h
+++ b/arch/powerpc/include/asm/dbell.h
@@ -44,6 +44,17 @@ static inline void _ppc_msgsnd(u32 msg)
                __asm__ __volatile__ (PPC_MSGSNDP(%0) : : "r" (msg));
 }
 
+/* sync before sending message */
+static inline void ppc_msgsnd_sync(void)
+{
+       __asm__ __volatile__ ("sync" : : : "memory");
+}
+
+/* sync after taking message interrupt */
+static inline void ppc_msgsync(void)
+{
+}
+
 #else /* CONFIG_PPC_BOOK3S */
 
 #define PPC_DBELL_MSGTYPE              PPC_DBELL
@@ -53,6 +64,17 @@ static inline void _ppc_msgsnd(u32 msg)
        __asm__ __volatile__ (PPC_MSGSND(%0) : : "r" (msg));
 }
 
+/* sync before sending message */
+static inline void ppc_msgsnd_sync(void)
+{
+       __asm__ __volatile__ ("sync" : : : "memory");
+}
+
+/* sync after taking message interrupt */
+static inline void ppc_msgsync(void)
+{
+}
+
 #endif /* CONFIG_PPC_BOOK3S */
 
 extern void doorbell_global_ipi(int cpu);

diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h
index b2e4e9a4e3f8eba3d6623de70d3f4f07082fff5b..edd939e76d528aad0ba27b1e03e47389a2d82756 100644 (file)
--- a/arch/powerpc/include/asm/smp.h
+++ b/arch/powerpc/include/asm/smp.h
@@ -129,6 +129,7 @@ extern const char *smp_ipi_name[];
 extern void smp_muxed_ipi_message_pass(int cpu, int msg);
 extern void smp_muxed_ipi_set_message(int cpu, int msg);
 extern irqreturn_t smp_ipi_demux(void);
+extern irqreturn_t smp_ipi_demux_relaxed(void);
 
 void smp_init_pSeries(void);
 void smp_init_cell(void);

diff --git a/arch/powerpc/kernel/dbell.c b/arch/powerpc/kernel/dbell.c
index 5869c66adfd0bbef77e0f4cc624378e919828668..b6fe883b1016dd395a21c0ec4c64867c2d208b57 100644 (file)
--- a/arch/powerpc/kernel/dbell.c
+++ b/arch/powerpc/kernel/dbell.c
@@ -38,7 +38,7 @@ void doorbell_global_ipi(int cpu)
 
        kvmppc_set_host_ipi(cpu, 1);
        /* Order previous accesses vs. msgsnd, which is treated as a store */
-       mb();
+       ppc_msgsnd_sync();
        ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag);
 }
 
@@ -53,7 +53,7 @@ void doorbell_core_ipi(int cpu)
 
        kvmppc_set_host_ipi(cpu, 1);
        /* Order previous accesses vs. msgsnd, which is treated as a store */
-       mb();
+       ppc_msgsnd_sync();
        ppc_msgsnd(PPC_DBELL_MSGTYPE, 0, tag);
 }
 
@@ -82,12 +82,14 @@ void doorbell_exception(struct pt_regs *regs)
 
        irq_enter();
 
+       ppc_msgsync();
+
        may_hard_irq_enable();
 
        kvmppc_set_host_ipi(smp_processor_id(), 0);
        __this_cpu_inc(irq_stat.doorbell_irqs);
 
-       smp_ipi_demux();
+       smp_ipi_demux_relaxed(); /* already performed the barrier */
 
        irq_exit();
        set_irq_regs(old_regs);

diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c
index f7b38918521019b5e4167726226bc3f7d63434b6..95eefa00abb3eeccdc82fd025d5fd028ca475d55 100644 (file)
--- a/arch/powerpc/kernel/smp.c
+++ b/arch/powerpc/kernel/smp.c
@@ -244,12 +244,18 @@ void smp_muxed_ipi_message_pass(int cpu, int msg)
 #endif
 
 irqreturn_t smp_ipi_demux(void)
+{
+       mb();   /* order any irq clear */
+
+       return smp_ipi_demux_relaxed();
+}
+
+/* sync-free variant. Callers should ensure synchronization */
+irqreturn_t smp_ipi_demux_relaxed(void)
 {
        struct cpu_messages *info;
        unsigned long all;
 
-       mb();   /* order any irq clear */
-
        info = this_cpu_ptr(&ipi_message);
        do {
                all = xchg(&info->messages, 0);