arch/x86/xen/spinlock.c

   1 /*
   2  * Split spinlock implementation out into its own file, so it can be
   3  * compiled in a FTRACE-compatible way.
   4  */
   5 #include <linux/kernel_stat.h>
   6 #include <linux/spinlock.h>
   7 #include <linux/debugfs.h>
   8 #include <linux/log2.h>
   9 #include <linux/gfp.h>
  10 #include <linux/slab.h>
  11
  12 #include <asm/paravirt.h>
  13
  14 #include <xen/interface/xen.h>
  15 #include <xen/events.h>
  16
  17 #include "xen-ops.h"
  18 #include "debugfs.h"
  19
  20 enum xen_contention_stat {
  21         TAKEN_SLOW,
  22         TAKEN_SLOW_PICKUP,
  23         TAKEN_SLOW_SPURIOUS,
  24         RELEASED_SLOW,
  25         RELEASED_SLOW_KICKED,
  26         NR_CONTENTION_STATS
  27 };
  28
  29
  30 #ifdef CONFIG_XEN_DEBUG_FS
  31 #define HISTO_BUCKETS   30
  32 static struct xen_spinlock_stats
  33 {
  34         u32 contention_stats[NR_CONTENTION_STATS];
  35         u32 histo_spin_blocked[HISTO_BUCKETS+1];
  36         u64 time_blocked;
  37 } spinlock_stats;
  38
  39 static u8 zero_stats;
  40
  41 static inline void check_zero(void)
  42 {
  43         u8 ret;
  44         u8 old = ACCESS_ONCE(zero_stats);
  45         if (unlikely(old)) {
  46                 ret = cmpxchg(&zero_stats, old, 0);
  47                 /* This ensures only one fellow resets the stat */
  48                 if (ret == old)
  49                         memset(&spinlock_stats, 0, sizeof(spinlock_stats));
  50         }
  51 }
  52
  53 static inline void add_stats(enum xen_contention_stat var, u32 val)
  54 {
  55         check_zero();
  56         spinlock_stats.contention_stats[var] += val;
  57 }
  58
  59 static inline u64 spin_time_start(void)
  60 {
  61         return xen_clocksource_read();
  62 }
  63
  64 static void __spin_time_accum(u64 delta, u32 *array)
  65 {
  66         unsigned index = ilog2(delta);
  67
  68         check_zero();
  69
  70         if (index < HISTO_BUCKETS)
  71                 array[index]++;
  72         else
  73                 array[HISTO_BUCKETS]++;
  74 }
  75
  76 static inline void spin_time_accum_blocked(u64 start)
  77 {
  78         u32 delta = xen_clocksource_read() - start;
  79
  80         __spin_time_accum(delta, spinlock_stats.histo_spin_blocked);
  81         spinlock_stats.time_blocked += delta;
  82 }
  83 #else  /* !CONFIG_XEN_DEBUG_FS */
  84 #define TIMEOUT                 (1 << 10)
  85 static inline void add_stats(enum xen_contention_stat var, u32 val)
  86 {
  87 }
  88
  89 static inline u64 spin_time_start(void)
  90 {
  91         return 0;
  92 }
  93
  94 static inline void spin_time_accum_blocked(u64 start)
  95 {
  96 }
  97 #endif  /* CONFIG_XEN_DEBUG_FS */
  98
  99 /*
 100  * Size struct xen_spinlock so it's the same as arch_spinlock_t.
 101  */
 102 #if NR_CPUS < 256
 103 typedef u8 xen_spinners_t;
 104 # define inc_spinners(xl) \
 105         asm(LOCK_PREFIX " incb %0" : "+m" ((xl)->spinners) : : "memory");
 106 # define dec_spinners(xl) \
 107         asm(LOCK_PREFIX " decb %0" : "+m" ((xl)->spinners) : : "memory");
 108 #else
 109 typedef u16 xen_spinners_t;
 110 # define inc_spinners(xl) \
 111         asm(LOCK_PREFIX " incw %0" : "+m" ((xl)->spinners) : : "memory");
 112 # define dec_spinners(xl) \
 113         asm(LOCK_PREFIX " decw %0" : "+m" ((xl)->spinners) : : "memory");
 114 #endif
 115
 116 struct xen_lock_waiting {
 117         struct arch_spinlock *lock;
 118         __ticket_t want;
 119 };
 120
 121 static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
 122 static DEFINE_PER_CPU(char *, irq_name);
 123 static DEFINE_PER_CPU(struct xen_lock_waiting, lock_waiting);
 124 static cpumask_t waiting_cpus;
 125
 126 static void xen_lock_spinning(struct arch_spinlock *lock, __ticket_t want)
 127 {
 128         int irq = __this_cpu_read(lock_kicker_irq);
 129         struct xen_lock_waiting *w = &__get_cpu_var(lock_waiting);
 130         int cpu = smp_processor_id();
 131         u64 start;
 132         unsigned long flags;
 133
 134         /* If kicker interrupts not initialized yet, just spin */
 135         if (irq == -1)
 136                 return;
 137
 138         start = spin_time_start();
 139
 140         /*
 141          * Make sure an interrupt handler can't upset things in a
 142          * partially setup state.
 143          */
 144         local_irq_save(flags);
 145         /*
 146          * We don't really care if we're overwriting some other
 147          * (lock,want) pair, as that would mean that we're currently
 148          * in an interrupt context, and the outer context had
 149          * interrupts enabled.  That has already kicked the VCPU out
 150          * of xen_poll_irq(), so it will just return spuriously and
 151          * retry with newly setup (lock,want).
 152          *
 153          * The ordering protocol on this is that the "lock" pointer
 154          * may only be set non-NULL if the "want" ticket is correct.
 155          * If we're updating "want", we must first clear "lock".
 156          */
 157         w->lock = NULL;
 158         smp_wmb();
 159         w->want = want;
 160         smp_wmb();
 161         w->lock = lock;
 162
 163         /* This uses set_bit, which atomic and therefore a barrier */
 164         cpumask_set_cpu(cpu, &waiting_cpus);
 165         add_stats(TAKEN_SLOW, 1);
 166
 167         /* clear pending */
 168         xen_clear_irq_pending(irq);
 169
 170         /* Only check lock once pending cleared */
 171         barrier();
 172
 173         /*
 174          * Mark entry to slowpath before doing the pickup test to make
 175          * sure we don't deadlock with an unlocker.
 176          */
 177         __ticket_enter_slowpath(lock);
 178
 179         /*
 180          * check again make sure it didn't become free while
 181          * we weren't looking
 182          */
 183         if (ACCESS_ONCE(lock->tickets.head) == want) {
 184                 add_stats(TAKEN_SLOW_PICKUP, 1);
 185                 goto out;
 186         }
 187
 188         /* Allow interrupts while blocked */
 189         local_irq_restore(flags);
 190
 191         /*
 192          * If an interrupt happens here, it will leave the wakeup irq
 193          * pending, which will cause xen_poll_irq() to return
 194          * immediately.
 195          */
 196
 197         /* Block until irq becomes pending (or perhaps a spurious wakeup) */
 198         xen_poll_irq(irq);
 199         add_stats(TAKEN_SLOW_SPURIOUS, !xen_test_irq_pending(irq));
 200
 201         local_irq_save(flags);
 202
 203         kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
 204 out:
 205         cpumask_clear_cpu(cpu, &waiting_cpus);
 206         w->lock = NULL;
 207
 208         local_irq_restore(flags);
 209
 210         spin_time_accum_blocked(start);
 211 }
 212 PV_CALLEE_SAVE_REGS_THUNK(xen_lock_spinning);
 213
 214 static void xen_unlock_kick(struct arch_spinlock *lock, __ticket_t next)
 215 {
 216         int cpu;
 217
 218         add_stats(RELEASED_SLOW, 1);
 219
 220         for_each_cpu(cpu, &waiting_cpus) {
 221                 const struct xen_lock_waiting *w = &per_cpu(lock_waiting, cpu);
 222
 223                 /* Make sure we read lock before want */
 224                 if (ACCESS_ONCE(w->lock) == lock &&
 225                     ACCESS_ONCE(w->want) == next) {
 226                         add_stats(RELEASED_SLOW_KICKED, 1);
 227                         xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
 228                         break;
 229                 }
 230         }
 231 }
 232
 233 static irqreturn_t dummy_handler(int irq, void *dev_id)
 234 {
 235         BUG();
 236         return IRQ_HANDLED;
 237 }
 238
 239 void xen_init_lock_cpu(int cpu)
 240 {
 241         int irq;
 242         char *name;
 243
 244         WARN(per_cpu(lock_kicker_irq, cpu) >= 0, "spinlock on CPU%d exists on IRQ%d!\n",
 245              cpu, per_cpu(lock_kicker_irq, cpu));
 246
 247         /*
 248          * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23
 249          * (xen: disable PV spinlocks on HVM)
 250          */
 251         if (xen_hvm_domain())
 252                 return;
 253
 254         name = kasprintf(GFP_KERNEL, "spinlock%d", cpu);
 255         irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR,
 256                                      cpu,
 257                                      dummy_handler,
 258                                      IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
 259                                      name,
 260                                      NULL);
 261
 262         if (irq >= 0) {
 263                 disable_irq(irq); /* make sure it's never delivered */
 264                 per_cpu(lock_kicker_irq, cpu) = irq;
 265                 per_cpu(irq_name, cpu) = name;
 266         }
 267
 268         printk("cpu %d spinlock event irq %d\n", cpu, irq);
 269 }
 270
 271 void xen_uninit_lock_cpu(int cpu)
 272 {
 273         /*
 274          * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23
 275          * (xen: disable PV spinlocks on HVM)
 276          */
 277         if (xen_hvm_domain())
 278                 return;
 279
 280         unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL);
 281         per_cpu(lock_kicker_irq, cpu) = -1;
 282         kfree(per_cpu(irq_name, cpu));
 283         per_cpu(irq_name, cpu) = NULL;
 284 }
 285
 286 static bool xen_pvspin __initdata = true;
 287
 288 void __init xen_init_spinlocks(void)
 289 {
 290         /*
 291          * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23
 292          * (xen: disable PV spinlocks on HVM)
 293          */
 294         if (xen_hvm_domain())
 295                 return;
 296
 297         if (!xen_pvspin) {
 298                 printk(KERN_DEBUG "xen: PV spinlocks disabled\n");
 299                 return;
 300         }
 301
 302         static_key_slow_inc(&paravirt_ticketlocks_enabled);
 303
 304         pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(xen_lock_spinning);
 305         pv_lock_ops.unlock_kick = xen_unlock_kick;
 306 }
 307
 308 static __init int xen_parse_nopvspin(char *arg)
 309 {
 310         xen_pvspin = false;
 311         return 0;
 312 }
 313 early_param("xen_nopvspin", xen_parse_nopvspin);
 314
 315 #ifdef CONFIG_XEN_DEBUG_FS
 316
 317 static struct dentry *d_spin_debug;
 318
 319 static int __init xen_spinlock_debugfs(void)
 320 {
 321         struct dentry *d_xen = xen_init_debugfs();
 322
 323         if (d_xen == NULL)
 324                 return -ENOMEM;
 325
 326         d_spin_debug = debugfs_create_dir("spinlocks", d_xen);
 327
 328         debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats);
 329
 330         debugfs_create_u32("taken_slow", 0444, d_spin_debug,
 331                            &spinlock_stats.contention_stats[TAKEN_SLOW]);
 332         debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug,
 333                            &spinlock_stats.contention_stats[TAKEN_SLOW_PICKUP]);
 334         debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug,
 335                            &spinlock_stats.contention_stats[TAKEN_SLOW_SPURIOUS]);
 336
 337         debugfs_create_u32("released_slow", 0444, d_spin_debug,
 338                            &spinlock_stats.contention_stats[RELEASED_SLOW]);
 339         debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug,
 340                            &spinlock_stats.contention_stats[RELEASED_SLOW_KICKED]);
 341
 342         debugfs_create_u64("time_blocked", 0444, d_spin_debug,
 343                            &spinlock_stats.time_blocked);
 344
 345         debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug,
 346                                 spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1);
 347
 348         return 0;
 349 }
 350 fs_initcall(xen_spinlock_debugfs);
 351
 352 #endif  /* CONFIG_XEN_DEBUG_FS */