]>
Commit | Line | Data |
---|---|---|
250c2277 | 1 | /* |
835c34a1 | 2 | * check TSC synchronization. |
250c2277 TG |
3 | * |
4 | * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar | |
5 | * | |
6 | * We check whether all boot CPUs have their TSC's synchronized, | |
7 | * print a warning if not and turn off the TSC clock-source. | |
8 | * | |
9 | * The warp-check is point-to-point between two CPUs, the CPU | |
10 | * initiating the bootup is the 'source CPU', the freshly booting | |
11 | * CPU is the 'target CPU'. | |
12 | * | |
13 | * Only two CPUs may participate - they can enter in any order. | |
14 | * ( The serial nature of the boot logic and the CPU hotplug lock | |
15 | * protects against more than 2 CPUs entering this code. ) | |
16 | */ | |
8b223bc7 | 17 | #include <linux/topology.h> |
250c2277 TG |
18 | #include <linux/spinlock.h> |
19 | #include <linux/kernel.h> | |
250c2277 TG |
20 | #include <linux/smp.h> |
21 | #include <linux/nmi.h> | |
22 | #include <asm/tsc.h> | |
23 | ||
8b223bc7 | 24 | struct tsc_adjust { |
1d0095fe TG |
25 | s64 bootval; |
26 | s64 adjusted; | |
27 | unsigned long nextcheck; | |
28 | bool warned; | |
8b223bc7 TG |
29 | }; |
30 | ||
31 | static DEFINE_PER_CPU(struct tsc_adjust, tsc_adjust); | |
32 | ||
6a369583 | 33 | void tsc_verify_tsc_adjust(bool resume) |
1d0095fe TG |
34 | { |
35 | struct tsc_adjust *adj = this_cpu_ptr(&tsc_adjust); | |
36 | s64 curval; | |
37 | ||
38 | if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) | |
39 | return; | |
40 | ||
41 | /* Rate limit the MSR check */ | |
6a369583 | 42 | if (!resume && time_before(jiffies, adj->nextcheck)) |
1d0095fe TG |
43 | return; |
44 | ||
45 | adj->nextcheck = jiffies + HZ; | |
46 | ||
47 | rdmsrl(MSR_IA32_TSC_ADJUST, curval); | |
48 | if (adj->adjusted == curval) | |
49 | return; | |
50 | ||
51 | /* Restore the original value */ | |
52 | wrmsrl(MSR_IA32_TSC_ADJUST, adj->adjusted); | |
53 | ||
6a369583 | 54 | if (!adj->warned || resume) { |
1d0095fe TG |
55 | pr_warn(FW_BUG "TSC ADJUST differs: CPU%u %lld --> %lld. Restoring\n", |
56 | smp_processor_id(), adj->adjusted, curval); | |
57 | adj->warned = true; | |
58 | } | |
59 | } | |
60 | ||
5bae1562 TG |
61 | static void tsc_sanitize_first_cpu(struct tsc_adjust *cur, s64 bootval, |
62 | unsigned int cpu, bool bootcpu) | |
63 | { | |
64 | /* | |
65 | * First online CPU in a package stores the boot value in the | |
66 | * adjustment value. This value might change later via the sync | |
67 | * mechanism. If that fails we still can yell about boot values not | |
68 | * being consistent. | |
69 | * | |
70 | * On the boot cpu we just force set the ADJUST value to 0 if it's | |
71 | * non zero. We don't do that on non boot cpus because physical | |
72 | * hotplug should have set the ADJUST register to a value > 0 so | |
73 | * the TSC is in sync with the already running cpus. | |
74 | * | |
75 | * But we always force positive ADJUST values. Otherwise the TSC | |
8c9b9d87 TG |
76 | * deadline timer creates an interrupt storm. We also have to |
77 | * prevent values > 0x7FFFFFFF as those wreckage the timer as well. | |
5bae1562 | 78 | */ |
8c9b9d87 TG |
79 | if ((bootcpu && bootval != 0) || (!bootcpu && bootval < 0) || |
80 | (bootval > 0x7FFFFFFF)) { | |
16588f65 TG |
81 | pr_warn(FW_BUG "TSC ADJUST: CPU%u: %lld force to 0\n", cpu, |
82 | bootval); | |
5bae1562 TG |
83 | wrmsrl(MSR_IA32_TSC_ADJUST, 0); |
84 | bootval = 0; | |
85 | } | |
86 | cur->adjusted = bootval; | |
87 | } | |
88 | ||
8b223bc7 | 89 | #ifndef CONFIG_SMP |
5bae1562 | 90 | bool __init tsc_store_and_check_tsc_adjust(bool bootcpu) |
8b223bc7 | 91 | { |
b8365543 | 92 | struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust); |
8b223bc7 TG |
93 | s64 bootval; |
94 | ||
95 | if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) | |
a36f5136 | 96 | return false; |
8b223bc7 TG |
97 | |
98 | rdmsrl(MSR_IA32_TSC_ADJUST, bootval); | |
99 | cur->bootval = bootval; | |
1d0095fe | 100 | cur->nextcheck = jiffies + HZ; |
5bae1562 | 101 | tsc_sanitize_first_cpu(cur, bootval, smp_processor_id(), bootcpu); |
a36f5136 | 102 | return false; |
8b223bc7 TG |
103 | } |
104 | ||
105 | #else /* !CONFIG_SMP */ | |
106 | ||
107 | /* | |
108 | * Store and check the TSC ADJUST MSR if available | |
109 | */ | |
5bae1562 | 110 | bool tsc_store_and_check_tsc_adjust(bool bootcpu) |
8b223bc7 TG |
111 | { |
112 | struct tsc_adjust *ref, *cur = this_cpu_ptr(&tsc_adjust); | |
113 | unsigned int refcpu, cpu = smp_processor_id(); | |
31f8a651 | 114 | struct cpumask *mask; |
8b223bc7 TG |
115 | s64 bootval; |
116 | ||
117 | if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) | |
a36f5136 | 118 | return false; |
8b223bc7 TG |
119 | |
120 | rdmsrl(MSR_IA32_TSC_ADJUST, bootval); | |
121 | cur->bootval = bootval; | |
1d0095fe TG |
122 | cur->nextcheck = jiffies + HZ; |
123 | cur->warned = false; | |
8b223bc7 TG |
124 | |
125 | /* | |
126 | * Check whether this CPU is the first in a package to come up. In | |
127 | * this case do not check the boot value against another package | |
5bae1562 TG |
128 | * because the new package might have been physically hotplugged, |
129 | * where TSC_ADJUST is expected to be different. When called on the | |
130 | * boot CPU topology_core_cpumask() might not be available yet. | |
8b223bc7 | 131 | */ |
31f8a651 TG |
132 | mask = topology_core_cpumask(cpu); |
133 | refcpu = mask ? cpumask_any_but(mask, cpu) : nr_cpu_ids; | |
8b223bc7 TG |
134 | |
135 | if (refcpu >= nr_cpu_ids) { | |
5bae1562 TG |
136 | tsc_sanitize_first_cpu(cur, bootval, smp_processor_id(), |
137 | bootcpu); | |
a36f5136 | 138 | return false; |
8b223bc7 TG |
139 | } |
140 | ||
141 | ref = per_cpu_ptr(&tsc_adjust, refcpu); | |
142 | /* | |
143 | * Compare the boot value and complain if it differs in the | |
144 | * package. | |
145 | */ | |
146 | if (bootval != ref->bootval) { | |
16588f65 | 147 | pr_warn(FW_BUG "TSC ADJUST differs: Reference CPU%u: %lld CPU%u: %lld\n", |
8b223bc7 TG |
148 | refcpu, ref->bootval, cpu, bootval); |
149 | } | |
150 | /* | |
151 | * The TSC_ADJUST values in a package must be the same. If the boot | |
152 | * value on this newly upcoming CPU differs from the adjustment | |
153 | * value of the already online CPU in this package, set it to that | |
154 | * adjusted value. | |
155 | */ | |
156 | if (bootval != ref->adjusted) { | |
157 | pr_warn("TSC ADJUST synchronize: Reference CPU%u: %lld CPU%u: %lld\n", | |
158 | refcpu, ref->adjusted, cpu, bootval); | |
159 | cur->adjusted = ref->adjusted; | |
160 | wrmsrl(MSR_IA32_TSC_ADJUST, ref->adjusted); | |
161 | } | |
a36f5136 TG |
162 | /* |
163 | * We have the TSCs forced to be in sync on this package. Skip sync | |
164 | * test: | |
165 | */ | |
166 | return true; | |
8b223bc7 TG |
167 | } |
168 | ||
250c2277 TG |
169 | /* |
170 | * Entry/exit counters that make sure that both CPUs | |
171 | * run the measurement code at once: | |
172 | */ | |
148f9bb8 PG |
173 | static atomic_t start_count; |
174 | static atomic_t stop_count; | |
a36f5136 | 175 | static atomic_t skip_test; |
cc4db268 | 176 | static atomic_t test_runs; |
250c2277 TG |
177 | |
178 | /* | |
179 | * We use a raw spinlock in this exceptional case, because | |
180 | * we want to have the fastest, inlined, non-debug version | |
181 | * of a critical section, to be able to prove TSC time-warps: | |
182 | */ | |
148f9bb8 | 183 | static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED; |
643bec95 | 184 | |
148f9bb8 PG |
185 | static cycles_t last_tsc; |
186 | static cycles_t max_warp; | |
187 | static int nr_warps; | |
bec8520d | 188 | static int random_warps; |
250c2277 TG |
189 | |
190 | /* | |
eee6946e AL |
191 | * TSC-warp measurement loop running on both CPUs. This is not called |
192 | * if there is no TSC. | |
250c2277 | 193 | */ |
76d3b851 | 194 | static cycles_t check_tsc_warp(unsigned int timeout) |
250c2277 | 195 | { |
76d3b851 | 196 | cycles_t start, now, prev, end, cur_max_warp = 0; |
bec8520d | 197 | int i, cur_warps = 0; |
250c2277 | 198 | |
eee6946e | 199 | start = rdtsc_ordered(); |
250c2277 | 200 | /* |
b0e5c779 | 201 | * The measurement runs for 'timeout' msecs: |
250c2277 | 202 | */ |
b0e5c779 | 203 | end = start + (cycles_t) tsc_khz * timeout; |
250c2277 TG |
204 | now = start; |
205 | ||
206 | for (i = 0; ; i++) { | |
207 | /* | |
208 | * We take the global lock, measure TSC, save the | |
209 | * previous TSC that was measured (possibly on | |
210 | * another CPU) and update the previous TSC timestamp. | |
211 | */ | |
0199c4e6 | 212 | arch_spin_lock(&sync_lock); |
250c2277 | 213 | prev = last_tsc; |
eee6946e | 214 | now = rdtsc_ordered(); |
250c2277 | 215 | last_tsc = now; |
0199c4e6 | 216 | arch_spin_unlock(&sync_lock); |
250c2277 TG |
217 | |
218 | /* | |
219 | * Be nice every now and then (and also check whether | |
df43510b | 220 | * measurement is done [we also insert a 10 million |
250c2277 TG |
221 | * loops safety exit, so we dont lock up in case the |
222 | * TSC readout is totally broken]): | |
223 | */ | |
224 | if (unlikely(!(i & 7))) { | |
df43510b | 225 | if (now > end || i > 10000000) |
250c2277 TG |
226 | break; |
227 | cpu_relax(); | |
228 | touch_nmi_watchdog(); | |
229 | } | |
230 | /* | |
231 | * Outside the critical section we can now see whether | |
232 | * we saw a time-warp of the TSC going backwards: | |
233 | */ | |
234 | if (unlikely(prev > now)) { | |
0199c4e6 | 235 | arch_spin_lock(&sync_lock); |
250c2277 | 236 | max_warp = max(max_warp, prev - now); |
76d3b851 | 237 | cur_max_warp = max_warp; |
bec8520d TG |
238 | /* |
239 | * Check whether this bounces back and forth. Only | |
240 | * one CPU should observe time going backwards. | |
241 | */ | |
242 | if (cur_warps != nr_warps) | |
243 | random_warps++; | |
250c2277 | 244 | nr_warps++; |
bec8520d | 245 | cur_warps = nr_warps; |
0199c4e6 | 246 | arch_spin_unlock(&sync_lock); |
250c2277 | 247 | } |
ad8ca495 | 248 | } |
bde78a79 AV |
249 | WARN(!(now-start), |
250 | "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n", | |
ad8ca495 | 251 | now-start, end-start); |
76d3b851 | 252 | return cur_max_warp; |
250c2277 TG |
253 | } |
254 | ||
b0e5c779 SS |
255 | /* |
256 | * If the target CPU coming online doesn't have any of its core-siblings | |
257 | * online, a timeout of 20msec will be used for the TSC-warp measurement | |
258 | * loop. Otherwise a smaller timeout of 2msec will be used, as we have some | |
259 | * information about this socket already (and this information grows as we | |
260 | * have more and more logical-siblings in that socket). | |
261 | * | |
262 | * Ideally we should be able to skip the TSC sync check on the other | |
263 | * core-siblings, if the first logical CPU in a socket passed the sync test. | |
264 | * But as the TSC is per-logical CPU and can potentially be modified wrongly | |
265 | * by the bios, TSC sync test for smaller duration should be able | |
266 | * to catch such errors. Also this will catch the condition where all the | |
267 | * cores in the socket doesn't get reset at the same time. | |
268 | */ | |
269 | static inline unsigned int loop_timeout(int cpu) | |
270 | { | |
7d79a7bd | 271 | return (cpumask_weight(topology_core_cpumask(cpu)) > 1) ? 2 : 20; |
b0e5c779 SS |
272 | } |
273 | ||
250c2277 TG |
274 | /* |
275 | * Source CPU calls into this - it waits for the freshly booted | |
276 | * target CPU to arrive and then starts the measurement: | |
277 | */ | |
148f9bb8 | 278 | void check_tsc_sync_source(int cpu) |
250c2277 TG |
279 | { |
280 | int cpus = 2; | |
281 | ||
282 | /* | |
283 | * No need to check if we already know that the TSC is not | |
eee6946e | 284 | * synchronized or if we have no TSC. |
250c2277 TG |
285 | */ |
286 | if (unsynchronized_tsc()) | |
287 | return; | |
288 | ||
cc4db268 TG |
289 | /* |
290 | * Set the maximum number of test runs to | |
291 | * 1 if the CPU does not provide the TSC_ADJUST MSR | |
292 | * 3 if the MSR is available, so the target can try to adjust | |
293 | */ | |
294 | if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST)) | |
295 | atomic_set(&test_runs, 1); | |
296 | else | |
297 | atomic_set(&test_runs, 3); | |
298 | retry: | |
250c2277 | 299 | /* |
a36f5136 | 300 | * Wait for the target to start or to skip the test: |
250c2277 | 301 | */ |
a36f5136 TG |
302 | while (atomic_read(&start_count) != cpus - 1) { |
303 | if (atomic_read(&skip_test) > 0) { | |
304 | atomic_set(&skip_test, 0); | |
305 | return; | |
306 | } | |
250c2277 | 307 | cpu_relax(); |
a36f5136 TG |
308 | } |
309 | ||
250c2277 TG |
310 | /* |
311 | * Trigger the target to continue into the measurement too: | |
312 | */ | |
313 | atomic_inc(&start_count); | |
314 | ||
b0e5c779 | 315 | check_tsc_warp(loop_timeout(cpu)); |
250c2277 TG |
316 | |
317 | while (atomic_read(&stop_count) != cpus-1) | |
318 | cpu_relax(); | |
319 | ||
cc4db268 TG |
320 | /* |
321 | * If the test was successful set the number of runs to zero and | |
322 | * stop. If not, decrement the number of runs an check if we can | |
323 | * retry. In case of random warps no retry is attempted. | |
324 | */ | |
325 | if (!nr_warps) { | |
326 | atomic_set(&test_runs, 0); | |
327 | ||
328 | pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n", | |
329 | smp_processor_id(), cpu); | |
330 | ||
331 | } else if (atomic_dec_and_test(&test_runs) || random_warps) { | |
332 | /* Force it to 0 if random warps brought us here */ | |
333 | atomic_set(&test_runs, 0); | |
334 | ||
9b3660a5 MT |
335 | pr_warning("TSC synchronization [CPU#%d -> CPU#%d]:\n", |
336 | smp_processor_id(), cpu); | |
643bec95 IM |
337 | pr_warning("Measured %Ld cycles TSC warp between CPUs, " |
338 | "turning off TSC clock.\n", max_warp); | |
bec8520d TG |
339 | if (random_warps) |
340 | pr_warning("TSC warped randomly between CPUs\n"); | |
250c2277 | 341 | mark_tsc_unstable("check_tsc_sync_source failed"); |
250c2277 TG |
342 | } |
343 | ||
4c6b8b4d MG |
344 | /* |
345 | * Reset it - just in case we boot another CPU later: | |
346 | */ | |
347 | atomic_set(&start_count, 0); | |
bec8520d | 348 | random_warps = 0; |
4c6b8b4d MG |
349 | nr_warps = 0; |
350 | max_warp = 0; | |
351 | last_tsc = 0; | |
352 | ||
250c2277 TG |
353 | /* |
354 | * Let the target continue with the bootup: | |
355 | */ | |
356 | atomic_inc(&stop_count); | |
cc4db268 TG |
357 | |
358 | /* | |
359 | * Retry, if there is a chance to do so. | |
360 | */ | |
361 | if (atomic_read(&test_runs) > 0) | |
362 | goto retry; | |
250c2277 TG |
363 | } |
364 | ||
365 | /* | |
366 | * Freshly booted CPUs call into this: | |
367 | */ | |
148f9bb8 | 368 | void check_tsc_sync_target(void) |
250c2277 | 369 | { |
cc4db268 TG |
370 | struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust); |
371 | unsigned int cpu = smp_processor_id(); | |
372 | cycles_t cur_max_warp, gbl_max_warp; | |
250c2277 TG |
373 | int cpus = 2; |
374 | ||
eee6946e | 375 | /* Also aborts if there is no TSC. */ |
5f2e71e7 | 376 | if (unsynchronized_tsc()) |
250c2277 TG |
377 | return; |
378 | ||
a36f5136 TG |
379 | /* |
380 | * Store, verify and sanitize the TSC adjust register. If | |
381 | * successful skip the test. | |
5f2e71e7 TG |
382 | * |
383 | * The test is also skipped when the TSC is marked reliable. This | |
384 | * is true for SoCs which have no fallback clocksource. On these | |
385 | * SoCs the TSC is frequency synchronized, but still the TSC ADJUST | |
386 | * register might have been wreckaged by the BIOS.. | |
a36f5136 | 387 | */ |
5f2e71e7 | 388 | if (tsc_store_and_check_tsc_adjust(false) || tsc_clocksource_reliable) { |
a36f5136 TG |
389 | atomic_inc(&skip_test); |
390 | return; | |
391 | } | |
8b223bc7 | 392 | |
cc4db268 | 393 | retry: |
250c2277 TG |
394 | /* |
395 | * Register this CPU's participation and wait for the | |
396 | * source CPU to start the measurement: | |
397 | */ | |
398 | atomic_inc(&start_count); | |
399 | while (atomic_read(&start_count) != cpus) | |
400 | cpu_relax(); | |
401 | ||
cc4db268 TG |
402 | cur_max_warp = check_tsc_warp(loop_timeout(cpu)); |
403 | ||
404 | /* | |
405 | * Store the maximum observed warp value for a potential retry: | |
406 | */ | |
407 | gbl_max_warp = max_warp; | |
250c2277 TG |
408 | |
409 | /* | |
410 | * Ok, we are done: | |
411 | */ | |
412 | atomic_inc(&stop_count); | |
413 | ||
414 | /* | |
415 | * Wait for the source CPU to print stuff: | |
416 | */ | |
417 | while (atomic_read(&stop_count) != cpus) | |
418 | cpu_relax(); | |
4c5e3c63 TG |
419 | |
420 | /* | |
421 | * Reset it for the next sync test: | |
422 | */ | |
423 | atomic_set(&stop_count, 0); | |
cc4db268 TG |
424 | |
425 | /* | |
426 | * Check the number of remaining test runs. If not zero, the test | |
427 | * failed and a retry with adjusted TSC is possible. If zero the | |
428 | * test was either successful or failed terminally. | |
429 | */ | |
430 | if (!atomic_read(&test_runs)) | |
431 | return; | |
432 | ||
433 | /* | |
434 | * If the warp value of this CPU is 0, then the other CPU | |
435 | * observed time going backwards so this TSC was ahead and | |
436 | * needs to move backwards. | |
437 | */ | |
438 | if (!cur_max_warp) | |
439 | cur_max_warp = -gbl_max_warp; | |
440 | ||
441 | /* | |
442 | * Add the result to the previous adjustment value. | |
443 | * | |
444 | * The adjustement value is slightly off by the overhead of the | |
445 | * sync mechanism (observed values are ~200 TSC cycles), but this | |
446 | * really depends on CPU, node distance and frequency. So | |
447 | * compensating for this is hard to get right. Experiments show | |
448 | * that the warp is not longer detectable when the observed warp | |
449 | * value is used. In the worst case the adjustment needs to go | |
450 | * through a 3rd run for fine tuning. | |
451 | */ | |
452 | cur->adjusted += cur_max_warp; | |
8c9b9d87 TG |
453 | |
454 | /* | |
455 | * TSC deadline timer stops working or creates an interrupt storm | |
456 | * with adjust values < 0 and > x07ffffff. | |
457 | * | |
458 | * To allow adjust values > 0x7FFFFFFF we need to disable the | |
459 | * deadline timer and use the local APIC timer, but that requires | |
460 | * more intrusive changes and we do not have any useful information | |
461 | * from Intel about the underlying HW wreckage yet. | |
462 | */ | |
5bae1562 TG |
463 | if (cur->adjusted < 0) |
464 | cur->adjusted = 0; | |
8c9b9d87 TG |
465 | if (cur->adjusted > 0x7FFFFFFF) |
466 | cur->adjusted = 0x7FFFFFFF; | |
cc4db268 TG |
467 | |
468 | pr_warn("TSC ADJUST compensate: CPU%u observed %lld warp. Adjust: %lld\n", | |
469 | cpu, cur_max_warp, cur->adjusted); | |
470 | ||
471 | wrmsrl(MSR_IA32_TSC_ADJUST, cur->adjusted); | |
472 | goto retry; | |
473 | ||
250c2277 | 474 | } |
8b223bc7 TG |
475 | |
476 | #endif /* CONFIG_SMP */ |