]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * Precise Delay Loops for i386 | |
3 | * | |
4 | * Copyright (C) 1993 Linus Torvalds | |
5 | * Copyright (C) 1997 Martin Mares <mj@atrey.karlin.mff.cuni.cz> | |
e01b70ef | 6 | * Copyright (C) 2008 Jiri Hladky <hladky _dot_ jiri _at_ gmail _dot_ com> |
1da177e4 LT |
7 | * |
8 | * The __delay function must _NOT_ be inlined as its execution time | |
9 | * depends wildly on alignment on many x86 processors. The additional | |
10 | * jump magic is needed to get the timing stable on all the CPU's | |
11 | * we have to worry about. | |
12 | */ | |
13 | ||
e683014c | 14 | #include <linux/export.h> |
1da177e4 | 15 | #include <linux/sched.h> |
941e492b | 16 | #include <linux/timex.h> |
35d5d08a | 17 | #include <linux/preempt.h> |
1da177e4 | 18 | #include <linux/delay.h> |
6f84fa2f | 19 | |
1da177e4 LT |
20 | #include <asm/processor.h> |
21 | #include <asm/delay.h> | |
22 | #include <asm/timer.h> | |
b466bdb6 | 23 | #include <asm/mwait.h> |
1da177e4 LT |
24 | |
25 | #ifdef CONFIG_SMP | |
6f84fa2f | 26 | # include <asm/smp.h> |
1da177e4 LT |
27 | #endif |
28 | ||
9aff3d50 TC |
29 | #define IBRS_DISABLE_THRESHOLD 1000 |
30 | ||
6f84fa2f JS |
31 | /* simple loop based delay: */ |
32 | static void delay_loop(unsigned long loops) | |
33 | { | |
f0fbf0ab | 34 | asm volatile( |
e01b70ef JH |
35 | " test %0,%0 \n" |
36 | " jz 3f \n" | |
37 | " jmp 1f \n" | |
38 | ||
39 | ".align 16 \n" | |
40 | "1: jmp 2f \n" | |
41 | ||
42 | ".align 16 \n" | |
ff1b15b6 | 43 | "2: dec %0 \n" |
e01b70ef | 44 | " jnz 2b \n" |
ff1b15b6 | 45 | "3: dec %0 \n" |
e01b70ef JH |
46 | |
47 | : /* we don't need output */ | |
48 | :"a" (loops) | |
49 | ); | |
6f84fa2f JS |
50 | } |
51 | ||
52 | /* TSC based delay: */ | |
a7f4255f | 53 | static void delay_tsc(unsigned long __loops) |
6f84fa2f | 54 | { |
9cfa1a02 | 55 | u64 bclock, now, loops = __loops; |
5c1ea082 | 56 | int cpu; |
6f84fa2f | 57 | |
5c1ea082 SR |
58 | preempt_disable(); |
59 | cpu = smp_processor_id(); | |
03b9730b | 60 | bclock = rdtsc_ordered(); |
5c1ea082 | 61 | for (;;) { |
03b9730b | 62 | now = rdtsc_ordered(); |
5c1ea082 SR |
63 | if ((now - bclock) >= loops) |
64 | break; | |
65 | ||
66 | /* Allow RT tasks to run */ | |
67 | preempt_enable(); | |
68 | rep_nop(); | |
69 | preempt_disable(); | |
70 | ||
71 | /* | |
72 | * It is possible that we moved to another CPU, and | |
73 | * since TSC's are per-cpu we need to calculate | |
74 | * that. The delay must guarantee that we wait "at | |
75 | * least" the amount of time. Being moved to another | |
76 | * CPU could make the wait longer but we just need to | |
77 | * make sure we waited long enough. Rebalance the | |
78 | * counter for this CPU. | |
79 | */ | |
80 | if (unlikely(cpu != smp_processor_id())) { | |
81 | loops -= (now - bclock); | |
82 | cpu = smp_processor_id(); | |
03b9730b | 83 | bclock = rdtsc_ordered(); |
5c1ea082 SR |
84 | } |
85 | } | |
35d5d08a | 86 | preempt_enable(); |
6f84fa2f JS |
87 | } |
88 | ||
b466bdb6 HR |
89 | /* |
90 | * On some AMD platforms, MWAITX has a configurable 32-bit timer, that | |
91 | * counts with TSC frequency. The input value is the loop of the | |
92 | * counter, it will exit when the timer expires. | |
93 | */ | |
94 | static void delay_mwaitx(unsigned long __loops) | |
95 | { | |
96 | u64 start, end, delay, loops = __loops; | |
97 | ||
88d879d2 JN |
98 | /* |
99 | * Timer value of 0 causes MWAITX to wait indefinitely, unless there | |
100 | * is a store on the memory monitored by MONITORX. | |
101 | */ | |
102 | if (loops == 0) | |
103 | return; | |
104 | ||
b466bdb6 HR |
105 | start = rdtsc_ordered(); |
106 | ||
107 | for (;;) { | |
108 | delay = min_t(u64, MWAITX_MAX_LOOPS, loops); | |
109 | ||
357b57d7 | 110 | if (ibrs_inuse && (delay > IBRS_DISABLE_THRESHOLD)) |
adae8f0d | 111 | native_wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base); |
9aff3d50 | 112 | |
b466bdb6 | 113 | /* |
785be108 | 114 | * Use cpu_tss_rw as a cacheline-aligned, seldomly |
b466bdb6 HR |
115 | * accessed per-cpu variable as the monitor target. |
116 | */ | |
785be108 | 117 | __monitorx(raw_cpu_ptr(&cpu_tss_rw), 0, 0); |
b466bdb6 HR |
118 | |
119 | /* | |
120 | * AMD, like Intel, supports the EAX hint and EAX=0xf | |
121 | * means, do not enter any deep C-state and we use it | |
122 | * here in delay() to minimize wakeup latency. | |
123 | */ | |
124 | __mwaitx(MWAITX_DISABLE_CSTATES, delay, MWAITX_ECX_TIMER_ENABLE); | |
125 | ||
357b57d7 | 126 | if (ibrs_inuse && (delay > IBRS_DISABLE_THRESHOLD)) |
adae8f0d | 127 | native_wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base | SPEC_CTRL_IBRS); |
9aff3d50 | 128 | |
b466bdb6 HR |
129 | end = rdtsc_ordered(); |
130 | ||
131 | if (loops <= end - start) | |
132 | break; | |
133 | ||
134 | loops -= end - start; | |
135 | ||
136 | start = end; | |
137 | } | |
138 | } | |
139 | ||
6f84fa2f JS |
140 | /* |
141 | * Since we calibrate only once at boot, this | |
142 | * function should be set once at boot and not changed | |
143 | */ | |
144 | static void (*delay_fn)(unsigned long) = delay_loop; | |
145 | ||
146 | void use_tsc_delay(void) | |
147 | { | |
b466bdb6 HR |
148 | if (delay_fn == delay_loop) |
149 | delay_fn = delay_tsc; | |
150 | } | |
151 | ||
152 | void use_mwaitx_delay(void) | |
153 | { | |
154 | delay_fn = delay_mwaitx; | |
6f84fa2f JS |
155 | } |
156 | ||
a18e3690 | 157 | int read_current_timer(unsigned long *timer_val) |
6f84fa2f JS |
158 | { |
159 | if (delay_fn == delay_tsc) { | |
4ea1636b | 160 | *timer_val = rdtsc(); |
6f84fa2f JS |
161 | return 0; |
162 | } | |
163 | return -1; | |
164 | } | |
1da177e4 LT |
165 | |
166 | void __delay(unsigned long loops) | |
167 | { | |
6f84fa2f | 168 | delay_fn(loops); |
1da177e4 | 169 | } |
f0fbf0ab | 170 | EXPORT_SYMBOL(__delay); |
1da177e4 LT |
171 | |
172 | inline void __const_udelay(unsigned long xloops) | |
173 | { | |
4c45c516 | 174 | unsigned long lpj = this_cpu_read(cpu_info.loops_per_jiffy) ? : loops_per_jiffy; |
1da177e4 | 175 | int d0; |
6f84fa2f | 176 | |
1da177e4 | 177 | xloops *= 4; |
f0fbf0ab | 178 | asm("mull %%edx" |
1da177e4 | 179 | :"=d" (xloops), "=&a" (d0) |
4c45c516 | 180 | :"1" (xloops), "0" (lpj * (HZ / 4))); |
6f84fa2f JS |
181 | |
182 | __delay(++xloops); | |
1da177e4 | 183 | } |
f0fbf0ab | 184 | EXPORT_SYMBOL(__const_udelay); |
1da177e4 LT |
185 | |
186 | void __udelay(unsigned long usecs) | |
187 | { | |
6f84fa2f | 188 | __const_udelay(usecs * 0x000010c7); /* 2**32 / 1000000 (rounded up) */ |
1da177e4 | 189 | } |
f0fbf0ab | 190 | EXPORT_SYMBOL(__udelay); |
1da177e4 LT |
191 | |
192 | void __ndelay(unsigned long nsecs) | |
193 | { | |
6f84fa2f | 194 | __const_udelay(nsecs * 0x00005); /* 2**32 / 1000000000 (rounded up) */ |
1da177e4 | 195 | } |
129f6946 | 196 | EXPORT_SYMBOL(__ndelay); |