arch/x86/xen/spinlock.c

   1 /*
   2  * Split spinlock implementation out into its own file, so it can be
   3  * compiled in a FTRACE-compatible way.
   4  */
   5 #include <linux/kernel_stat.h>
   6 #include <linux/spinlock.h>
   7 #include <linux/debugfs.h>
   8 #include <linux/log2.h>
   9 #include <linux/gfp.h>
  10 #include <linux/slab.h>
  11
  12 #include <asm/paravirt.h>
  13
  14 #include <xen/interface/xen.h>
  15 #include <xen/events.h>
  16
  17 #include "xen-ops.h"
  18 #include "debugfs.h"
  19
  20 enum xen_contention_stat {
  21         TAKEN_SLOW,
  22         TAKEN_SLOW_PICKUP,
  23         TAKEN_SLOW_SPURIOUS,
  24         RELEASED_SLOW,
  25         RELEASED_SLOW_KICKED,
  26         NR_CONTENTION_STATS
  27 };
  28
  29
  30 #ifdef CONFIG_XEN_DEBUG_FS
  31 #define HISTO_BUCKETS   30
  32 static struct xen_spinlock_stats
  33 {
  34         u32 contention_stats[NR_CONTENTION_STATS];
  35         u32 histo_spin_blocked[HISTO_BUCKETS+1];
  36         u64 time_blocked;
  37 } spinlock_stats;
  38
  39 static u8 zero_stats;
  40
  41 static inline void check_zero(void)
  42 {
  43         u8 ret;
  44         u8 old = ACCESS_ONCE(zero_stats);
  45         if (unlikely(old)) {
  46                 ret = cmpxchg(&zero_stats, old, 0);
  47                 /* This ensures only one fellow resets the stat */
  48                 if (ret == old)
  49                         memset(&spinlock_stats, 0, sizeof(spinlock_stats));
  50         }
  51 }
  52
  53 static inline void add_stats(enum xen_contention_stat var, u32 val)
  54 {
  55         check_zero();
  56         spinlock_stats.contention_stats[var] += val;
  57 }
  58
  59 static inline u64 spin_time_start(void)
  60 {
  61         return xen_clocksource_read();
  62 }
  63
  64 static void __spin_time_accum(u64 delta, u32 *array)
  65 {
  66         unsigned index = ilog2(delta);
  67
  68         check_zero();
  69
  70         if (index < HISTO_BUCKETS)
  71                 array[index]++;
  72         else
  73                 array[HISTO_BUCKETS]++;
  74 }
  75
  76 static inline void spin_time_accum_blocked(u64 start)
  77 {
  78         u32 delta = xen_clocksource_read() - start;
  79
  80         __spin_time_accum(delta, spinlock_stats.histo_spin_blocked);
  81         spinlock_stats.time_blocked += delta;
  82 }
  83 #else  /* !CONFIG_XEN_DEBUG_FS */
  84 #define TIMEOUT                 (1 << 10)
  85 static inline void add_stats(enum xen_contention_stat var, u32 val)
  86 {
  87 }
  88
  89 static inline u64 spin_time_start(void)
  90 {
  91         return 0;
  92 }
  93
  94 static inline void spin_time_accum_blocked(u64 start)
  95 {
  96 }
  97 #endif  /* CONFIG_XEN_DEBUG_FS */
  98
  99 /*
 100  * Size struct xen_spinlock so it's the same as arch_spinlock_t.
 101  */
 102 #if NR_CPUS < 256
 103 typedef u8 xen_spinners_t;
 104 # define inc_spinners(xl) \
 105         asm(LOCK_PREFIX " incb %0" : "+m" ((xl)->spinners) : : "memory");
 106 # define dec_spinners(xl) \
 107         asm(LOCK_PREFIX " decb %0" : "+m" ((xl)->spinners) : : "memory");
 108 #else
 109 typedef u16 xen_spinners_t;
 110 # define inc_spinners(xl) \
 111         asm(LOCK_PREFIX " incw %0" : "+m" ((xl)->spinners) : : "memory");
 112 # define dec_spinners(xl) \
 113         asm(LOCK_PREFIX " decw %0" : "+m" ((xl)->spinners) : : "memory");
 114 #endif
 115
 116 struct xen_lock_waiting {
 117         struct arch_spinlock *lock;
 118         __ticket_t want;
 119 };
 120
 121 static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
 122 static DEFINE_PER_CPU(char *, irq_name);
 123 static DEFINE_PER_CPU(struct xen_lock_waiting, lock_waiting);
 124 static cpumask_t waiting_cpus;
 125
 126 static void xen_lock_spinning(struct arch_spinlock *lock, __ticket_t want)
 127 {
 128         int irq = __this_cpu_read(lock_kicker_irq);
 129         struct xen_lock_waiting *w = &__get_cpu_var(lock_waiting);
 130         int cpu = smp_processor_id();
 131         u64 start;
 132         unsigned long flags;
 133
 134         /* If kicker interrupts not initialized yet, just spin */
 135         if (irq == -1)
 136                 return;
 137
 138         start = spin_time_start();
 139
 140         /*
 141          * Make sure an interrupt handler can't upset things in a
 142          * partially setup state.
 143          */
 144         local_irq_save(flags);
 145
 146         w->want = want;
 147         smp_wmb();
 148         w->lock = lock;
 149
 150         /* This uses set_bit, which atomic and therefore a barrier */
 151         cpumask_set_cpu(cpu, &waiting_cpus);
 152         add_stats(TAKEN_SLOW, 1);
 153
 154         /* clear pending */
 155         xen_clear_irq_pending(irq);
 156
 157         /* Only check lock once pending cleared */
 158         barrier();
 159
 160         /* check again make sure it didn't become free while
 161            we weren't looking  */
 162         if (ACCESS_ONCE(lock->tickets.head) == want) {
 163                 add_stats(TAKEN_SLOW_PICKUP, 1);
 164                 goto out;
 165         }
 166         /* Block until irq becomes pending (or perhaps a spurious wakeup) */
 167         xen_poll_irq(irq);
 168         add_stats(TAKEN_SLOW_SPURIOUS, !xen_test_irq_pending(irq));
 169         kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
 170 out:
 171         cpumask_clear_cpu(cpu, &waiting_cpus);
 172         w->lock = NULL;
 173         local_irq_restore(flags);
 174         spin_time_accum_blocked(start);
 175 }
 176 PV_CALLEE_SAVE_REGS_THUNK(xen_lock_spinning);
 177
 178 static void xen_unlock_kick(struct arch_spinlock *lock, __ticket_t next)
 179 {
 180         int cpu;
 181
 182         add_stats(RELEASED_SLOW, 1);
 183
 184         for_each_cpu(cpu, &waiting_cpus) {
 185                 const struct xen_lock_waiting *w = &per_cpu(lock_waiting, cpu);
 186
 187                 if (w->lock == lock && w->want == next) {
 188                         add_stats(RELEASED_SLOW_KICKED, 1);
 189                         xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
 190                         break;
 191                 }
 192         }
 193 }
 194
 195 static irqreturn_t dummy_handler(int irq, void *dev_id)
 196 {
 197         BUG();
 198         return IRQ_HANDLED;
 199 }
 200
 201 void xen_init_lock_cpu(int cpu)
 202 {
 203         int irq;
 204         char *name;
 205
 206         WARN(per_cpu(lock_kicker_irq, cpu) >= 0, "spinlock on CPU%d exists on IRQ%d!\n",
 207              cpu, per_cpu(lock_kicker_irq, cpu));
 208
 209         /*
 210          * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23
 211          * (xen: disable PV spinlocks on HVM)
 212          */
 213         if (xen_hvm_domain())
 214                 return;
 215
 216         name = kasprintf(GFP_KERNEL, "spinlock%d", cpu);
 217         irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR,
 218                                      cpu,
 219                                      dummy_handler,
 220                                      IRQF_DISABLED|IRQF_PERCPU|IRQF_NOBALANCING,
 221                                      name,
 222                                      NULL);
 223
 224         if (irq >= 0) {
 225                 disable_irq(irq); /* make sure it's never delivered */
 226                 per_cpu(lock_kicker_irq, cpu) = irq;
 227                 per_cpu(irq_name, cpu) = name;
 228         }
 229
 230         printk("cpu %d spinlock event irq %d\n", cpu, irq);
 231 }
 232
 233 void xen_uninit_lock_cpu(int cpu)
 234 {
 235         /*
 236          * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23
 237          * (xen: disable PV spinlocks on HVM)
 238          */
 239         if (xen_hvm_domain())
 240                 return;
 241
 242         unbind_from_irqhandler(per_cpu(lock_kicker_irq, cpu), NULL);
 243         per_cpu(lock_kicker_irq, cpu) = -1;
 244         kfree(per_cpu(irq_name, cpu));
 245         per_cpu(irq_name, cpu) = NULL;
 246 }
 247
 248 static bool xen_pvspin __initdata = true;
 249
 250 void __init xen_init_spinlocks(void)
 251 {
 252         /*
 253          * See git commit f10cd522c5fbfec9ae3cc01967868c9c2401ed23
 254          * (xen: disable PV spinlocks on HVM)
 255          */
 256         if (xen_hvm_domain())
 257                 return;
 258
 259         if (!xen_pvspin) {
 260                 printk(KERN_DEBUG "xen: PV spinlocks disabled\n");
 261                 return;
 262         }
 263
 264         pv_lock_ops.lock_spinning = PV_CALLEE_SAVE(xen_lock_spinning);
 265         pv_lock_ops.unlock_kick = xen_unlock_kick;
 266 }
 267
 268 static __init int xen_parse_nopvspin(char *arg)
 269 {
 270         xen_pvspin = false;
 271         return 0;
 272 }
 273 early_param("xen_nopvspin", xen_parse_nopvspin);
 274
 275 #ifdef CONFIG_XEN_DEBUG_FS
 276
 277 static struct dentry *d_spin_debug;
 278
 279 static int __init xen_spinlock_debugfs(void)
 280 {
 281         struct dentry *d_xen = xen_init_debugfs();
 282
 283         if (d_xen == NULL)
 284                 return -ENOMEM;
 285
 286         d_spin_debug = debugfs_create_dir("spinlocks", d_xen);
 287
 288         debugfs_create_u8("zero_stats", 0644, d_spin_debug, &zero_stats);
 289
 290         debugfs_create_u32("taken_slow", 0444, d_spin_debug,
 291                            &spinlock_stats.contention_stats[TAKEN_SLOW]);
 292         debugfs_create_u32("taken_slow_pickup", 0444, d_spin_debug,
 293                            &spinlock_stats.contention_stats[TAKEN_SLOW_PICKUP]);
 294         debugfs_create_u32("taken_slow_spurious", 0444, d_spin_debug,
 295                            &spinlock_stats.contention_stats[TAKEN_SLOW_SPURIOUS]);
 296
 297         debugfs_create_u32("released_slow", 0444, d_spin_debug,
 298                            &spinlock_stats.contention_stats[RELEASED_SLOW]);
 299         debugfs_create_u32("released_slow_kicked", 0444, d_spin_debug,
 300                            &spinlock_stats.contention_stats[RELEASED_SLOW_KICKED]);
 301
 302         debugfs_create_u64("time_blocked", 0444, d_spin_debug,
 303                            &spinlock_stats.time_blocked);
 304
 305         debugfs_create_u32_array("histo_blocked", 0444, d_spin_debug,
 306                                 spinlock_stats.histo_spin_blocked, HISTO_BUCKETS + 1);
 307
 308         return 0;
 309 }
 310 fs_initcall(xen_spinlock_debugfs);
 311
 312 #endif  /* CONFIG_XEN_DEBUG_FS */