]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - arch/x86/lib/delay.c
x86/idle: Disable IBRS entering idle and enable it on wakeup
[mirror_ubuntu-artful-kernel.git] / arch / x86 / lib / delay.c
CommitLineData
1da177e4
LT
1/*
2 * Precise Delay Loops for i386
3 *
4 * Copyright (C) 1993 Linus Torvalds
5 * Copyright (C) 1997 Martin Mares <mj@atrey.karlin.mff.cuni.cz>
e01b70ef 6 * Copyright (C) 2008 Jiri Hladky <hladky _dot_ jiri _at_ gmail _dot_ com>
1da177e4
LT
7 *
8 * The __delay function must _NOT_ be inlined as its execution time
9 * depends wildly on alignment on many x86 processors. The additional
10 * jump magic is needed to get the timing stable on all the CPU's
11 * we have to worry about.
12 */
13
e683014c 14#include <linux/export.h>
1da177e4 15#include <linux/sched.h>
941e492b 16#include <linux/timex.h>
35d5d08a 17#include <linux/preempt.h>
1da177e4 18#include <linux/delay.h>
6f84fa2f 19
1da177e4
LT
20#include <asm/processor.h>
21#include <asm/delay.h>
22#include <asm/timer.h>
b466bdb6 23#include <asm/mwait.h>
1da177e4
LT
24
25#ifdef CONFIG_SMP
6f84fa2f 26# include <asm/smp.h>
1da177e4
LT
27#endif
28
9aff3d50
TC
29#define IBRS_DISABLE_THRESHOLD 1000
30
6f84fa2f
JS
31/* simple loop based delay: */
32static void delay_loop(unsigned long loops)
33{
f0fbf0ab 34 asm volatile(
e01b70ef
JH
35 " test %0,%0 \n"
36 " jz 3f \n"
37 " jmp 1f \n"
38
39 ".align 16 \n"
40 "1: jmp 2f \n"
41
42 ".align 16 \n"
ff1b15b6 43 "2: dec %0 \n"
e01b70ef 44 " jnz 2b \n"
ff1b15b6 45 "3: dec %0 \n"
e01b70ef
JH
46
47 : /* we don't need output */
48 :"a" (loops)
49 );
6f84fa2f
JS
50}
51
52/* TSC based delay: */
a7f4255f 53static void delay_tsc(unsigned long __loops)
6f84fa2f 54{
9cfa1a02 55 u64 bclock, now, loops = __loops;
5c1ea082 56 int cpu;
6f84fa2f 57
5c1ea082
SR
58 preempt_disable();
59 cpu = smp_processor_id();
03b9730b 60 bclock = rdtsc_ordered();
5c1ea082 61 for (;;) {
03b9730b 62 now = rdtsc_ordered();
5c1ea082
SR
63 if ((now - bclock) >= loops)
64 break;
65
66 /* Allow RT tasks to run */
67 preempt_enable();
68 rep_nop();
69 preempt_disable();
70
71 /*
72 * It is possible that we moved to another CPU, and
73 * since TSC's are per-cpu we need to calculate
74 * that. The delay must guarantee that we wait "at
75 * least" the amount of time. Being moved to another
76 * CPU could make the wait longer but we just need to
77 * make sure we waited long enough. Rebalance the
78 * counter for this CPU.
79 */
80 if (unlikely(cpu != smp_processor_id())) {
81 loops -= (now - bclock);
82 cpu = smp_processor_id();
03b9730b 83 bclock = rdtsc_ordered();
5c1ea082
SR
84 }
85 }
35d5d08a 86 preempt_enable();
6f84fa2f
JS
87}
88
b466bdb6
HR
89/*
90 * On some AMD platforms, MWAITX has a configurable 32-bit timer, that
91 * counts with TSC frequency. The input value is the loop of the
92 * counter, it will exit when the timer expires.
93 */
94static void delay_mwaitx(unsigned long __loops)
95{
96 u64 start, end, delay, loops = __loops;
97
88d879d2
JN
98 /*
99 * Timer value of 0 causes MWAITX to wait indefinitely, unless there
100 * is a store on the memory monitored by MONITORX.
101 */
102 if (loops == 0)
103 return;
104
b466bdb6
HR
105 start = rdtsc_ordered();
106
107 for (;;) {
108 delay = min_t(u64, MWAITX_MAX_LOOPS, loops);
109
9aff3d50
TC
110 if (boot_cpu_has(X86_FEATURE_SPEC_CTRL) &&
111 (delay > IBRS_DISABLE_THRESHOLD))
112 native_wrmsrl(MSR_IA32_SPEC_CTRL, 0);
113
b466bdb6 114 /*
785be108 115 * Use cpu_tss_rw as a cacheline-aligned, seldomly
b466bdb6
HR
116 * accessed per-cpu variable as the monitor target.
117 */
785be108 118 __monitorx(raw_cpu_ptr(&cpu_tss_rw), 0, 0);
b466bdb6
HR
119
120 /*
121 * AMD, like Intel, supports the EAX hint and EAX=0xf
122 * means, do not enter any deep C-state and we use it
123 * here in delay() to minimize wakeup latency.
124 */
125 __mwaitx(MWAITX_DISABLE_CSTATES, delay, MWAITX_ECX_TIMER_ENABLE);
126
9aff3d50
TC
127 if (boot_cpu_has(X86_FEATURE_SPEC_CTRL) &&
128 (delay > IBRS_DISABLE_THRESHOLD))
129 native_wrmsrl(MSR_IA32_SPEC_CTRL, FEATURE_ENABLE_IBRS);
130
b466bdb6
HR
131 end = rdtsc_ordered();
132
133 if (loops <= end - start)
134 break;
135
136 loops -= end - start;
137
138 start = end;
139 }
140}
141
6f84fa2f
JS
142/*
143 * Since we calibrate only once at boot, this
144 * function should be set once at boot and not changed
145 */
146static void (*delay_fn)(unsigned long) = delay_loop;
147
148void use_tsc_delay(void)
149{
b466bdb6
HR
150 if (delay_fn == delay_loop)
151 delay_fn = delay_tsc;
152}
153
154void use_mwaitx_delay(void)
155{
156 delay_fn = delay_mwaitx;
6f84fa2f
JS
157}
158
a18e3690 159int read_current_timer(unsigned long *timer_val)
6f84fa2f
JS
160{
161 if (delay_fn == delay_tsc) {
4ea1636b 162 *timer_val = rdtsc();
6f84fa2f
JS
163 return 0;
164 }
165 return -1;
166}
1da177e4
LT
167
168void __delay(unsigned long loops)
169{
6f84fa2f 170 delay_fn(loops);
1da177e4 171}
f0fbf0ab 172EXPORT_SYMBOL(__delay);
1da177e4
LT
173
174inline void __const_udelay(unsigned long xloops)
175{
4c45c516 176 unsigned long lpj = this_cpu_read(cpu_info.loops_per_jiffy) ? : loops_per_jiffy;
1da177e4 177 int d0;
6f84fa2f 178
1da177e4 179 xloops *= 4;
f0fbf0ab 180 asm("mull %%edx"
1da177e4 181 :"=d" (xloops), "=&a" (d0)
4c45c516 182 :"1" (xloops), "0" (lpj * (HZ / 4)));
6f84fa2f
JS
183
184 __delay(++xloops);
1da177e4 185}
f0fbf0ab 186EXPORT_SYMBOL(__const_udelay);
1da177e4
LT
187
188void __udelay(unsigned long usecs)
189{
6f84fa2f 190 __const_udelay(usecs * 0x000010c7); /* 2**32 / 1000000 (rounded up) */
1da177e4 191}
f0fbf0ab 192EXPORT_SYMBOL(__udelay);
1da177e4
LT
193
194void __ndelay(unsigned long nsecs)
195{
6f84fa2f 196 __const_udelay(nsecs * 0x00005); /* 2**32 / 1000000000 (rounded up) */
1da177e4 197}
129f6946 198EXPORT_SYMBOL(__ndelay);