]>
Commit | Line | Data |
---|---|---|
112f38a4 | 1 | /* |
32fea568 IM |
2 | * sched_clock.c: Generic sched_clock() support, to extend low level |
3 | * hardware time counters to full 64-bit ns values. | |
112f38a4 RK |
4 | * |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License version 2 as | |
7 | * published by the Free Software Foundation. | |
8 | */ | |
9 | #include <linux/clocksource.h> | |
10 | #include <linux/init.h> | |
11 | #include <linux/jiffies.h> | |
a08ca5d1 | 12 | #include <linux/ktime.h> |
112f38a4 | 13 | #include <linux/kernel.h> |
a42c3629 | 14 | #include <linux/moduleparam.h> |
112f38a4 | 15 | #include <linux/sched.h> |
e6017571 | 16 | #include <linux/sched/clock.h> |
f153d017 | 17 | #include <linux/syscore_ops.h> |
a08ca5d1 | 18 | #include <linux/hrtimer.h> |
38ff87f7 | 19 | #include <linux/sched_clock.h> |
85c3d2dd | 20 | #include <linux/seqlock.h> |
e7e3ff1b | 21 | #include <linux/bitops.h> |
112f38a4 | 22 | |
cf7c9c17 | 23 | /** |
32fea568 | 24 | * struct clock_read_data - data required to read from sched_clock() |
cf7c9c17 | 25 | * |
32fea568 IM |
26 | * @epoch_ns: sched_clock() value at last update |
27 | * @epoch_cyc: Clock cycle value at last update. | |
cf7c9c17 | 28 | * @sched_clock_mask: Bitmask for two's complement subtraction of non 64bit |
32fea568 IM |
29 | * clocks. |
30 | * @read_sched_clock: Current clock source (or dummy source when suspended). | |
31 | * @mult: Multipler for scaled math conversion. | |
32 | * @shift: Shift value for scaled math conversion. | |
cf7c9c17 DT |
33 | * |
34 | * Care must be taken when updating this structure; it is read by | |
13dbeb38 | 35 | * some very hot code paths. It occupies <=40 bytes and, when combined |
cf7c9c17 DT |
36 | * with the seqcount used to synchronize access, comfortably fits into |
37 | * a 64 byte cache line. | |
38 | */ | |
39 | struct clock_read_data { | |
2f0778af | 40 | u64 epoch_ns; |
e7e3ff1b | 41 | u64 epoch_cyc; |
cf7c9c17 DT |
42 | u64 sched_clock_mask; |
43 | u64 (*read_sched_clock)(void); | |
2f0778af MZ |
44 | u32 mult; |
45 | u32 shift; | |
46 | }; | |
47 | ||
cf7c9c17 | 48 | /** |
32fea568 | 49 | * struct clock_data - all data needed for sched_clock() (including |
cf7c9c17 DT |
50 | * registration of a new clock source) |
51 | * | |
1809bfa4 DT |
52 | * @seq: Sequence counter for protecting updates. The lowest |
53 | * bit is the index for @read_data. | |
cf7c9c17 | 54 | * @read_data: Data required to read from sched_clock. |
32fea568 IM |
55 | * @wrap_kt: Duration for which clock can run before wrapping. |
56 | * @rate: Tick rate of the registered clock. | |
57 | * @actual_read_sched_clock: Registered hardware level clock read function. | |
cf7c9c17 DT |
58 | * |
59 | * The ordering of this structure has been chosen to optimize cache | |
32fea568 IM |
60 | * performance. In particular 'seq' and 'read_data[0]' (combined) should fit |
61 | * into a single 64-byte cache line. | |
cf7c9c17 DT |
62 | */ |
63 | struct clock_data { | |
32fea568 IM |
64 | seqcount_t seq; |
65 | struct clock_read_data read_data[2]; | |
66 | ktime_t wrap_kt; | |
67 | unsigned long rate; | |
68 | ||
13dbeb38 | 69 | u64 (*actual_read_sched_clock)(void); |
cf7c9c17 DT |
70 | }; |
71 | ||
a08ca5d1 | 72 | static struct hrtimer sched_clock_timer; |
a42c3629 RK |
73 | static int irqtime = -1; |
74 | ||
75 | core_param(irqtime, irqtime, int, 0400); | |
2f0778af | 76 | |
e7e3ff1b | 77 | static u64 notrace jiffy_sched_clock_read(void) |
2f0778af | 78 | { |
e7e3ff1b SB |
79 | /* |
80 | * We don't need to use get_jiffies_64 on 32-bit arches here | |
81 | * because we register with BITS_PER_LONG | |
82 | */ | |
83 | return (u64)(jiffies - INITIAL_JIFFIES); | |
84 | } | |
85 | ||
cf7c9c17 | 86 | static struct clock_data cd ____cacheline_aligned = { |
1809bfa4 DT |
87 | .read_data[0] = { .mult = NSEC_PER_SEC / HZ, |
88 | .read_sched_clock = jiffy_sched_clock_read, }, | |
13dbeb38 | 89 | .actual_read_sched_clock = jiffy_sched_clock_read, |
cf7c9c17 | 90 | }; |
2f0778af | 91 | |
cea15092 | 92 | static inline u64 notrace cyc_to_ns(u64 cyc, u32 mult, u32 shift) |
2f0778af MZ |
93 | { |
94 | return (cyc * mult) >> shift; | |
95 | } | |
96 | ||
b4042cea | 97 | unsigned long long notrace sched_clock(void) |
2f0778af | 98 | { |
8710e914 | 99 | u64 cyc, res; |
85c3d2dd | 100 | unsigned long seq; |
1809bfa4 | 101 | struct clock_read_data *rd; |
336ae118 | 102 | |
2f0778af | 103 | do { |
1809bfa4 DT |
104 | seq = raw_read_seqcount(&cd.seq); |
105 | rd = cd.read_data + (seq & 1); | |
8710e914 | 106 | |
13dbeb38 DT |
107 | cyc = (rd->read_sched_clock() - rd->epoch_cyc) & |
108 | rd->sched_clock_mask; | |
109 | res = rd->epoch_ns + cyc_to_ns(cyc, rd->mult, rd->shift); | |
85c3d2dd | 110 | } while (read_seqcount_retry(&cd.seq, seq)); |
2f0778af | 111 | |
8710e914 | 112 | return res; |
2f0778af MZ |
113 | } |
114 | ||
1809bfa4 DT |
115 | /* |
116 | * Updating the data required to read the clock. | |
117 | * | |
32fea568 | 118 | * sched_clock() will never observe mis-matched data even if called from |
1809bfa4 | 119 | * an NMI. We do this by maintaining an odd/even copy of the data and |
32fea568 IM |
120 | * steering sched_clock() to one or the other using a sequence counter. |
121 | * In order to preserve the data cache profile of sched_clock() as much | |
1809bfa4 DT |
122 | * as possible the system reverts back to the even copy when the update |
123 | * completes; the odd copy is used *only* during an update. | |
124 | */ | |
125 | static void update_clock_read_data(struct clock_read_data *rd) | |
126 | { | |
127 | /* update the backup (odd) copy with the new data */ | |
128 | cd.read_data[1] = *rd; | |
129 | ||
130 | /* steer readers towards the odd copy */ | |
131 | raw_write_seqcount_latch(&cd.seq); | |
132 | ||
133 | /* now its safe for us to update the normal (even) copy */ | |
134 | cd.read_data[0] = *rd; | |
135 | ||
136 | /* switch readers back to the even copy */ | |
137 | raw_write_seqcount_latch(&cd.seq); | |
138 | } | |
139 | ||
2f0778af | 140 | /* |
32fea568 | 141 | * Atomically update the sched_clock() epoch. |
2f0778af | 142 | */ |
9fee69a8 | 143 | static void update_sched_clock(void) |
2f0778af | 144 | { |
e7e3ff1b | 145 | u64 cyc; |
2f0778af | 146 | u64 ns; |
1809bfa4 DT |
147 | struct clock_read_data rd; |
148 | ||
149 | rd = cd.read_data[0]; | |
2f0778af | 150 | |
13dbeb38 | 151 | cyc = cd.actual_read_sched_clock(); |
32fea568 | 152 | ns = rd.epoch_ns + cyc_to_ns((cyc - rd.epoch_cyc) & rd.sched_clock_mask, rd.mult, rd.shift); |
1809bfa4 DT |
153 | |
154 | rd.epoch_ns = ns; | |
155 | rd.epoch_cyc = cyc; | |
156 | ||
157 | update_clock_read_data(&rd); | |
2f0778af | 158 | } |
112f38a4 | 159 | |
a08ca5d1 | 160 | static enum hrtimer_restart sched_clock_poll(struct hrtimer *hrt) |
112f38a4 | 161 | { |
2f0778af | 162 | update_sched_clock(); |
a08ca5d1 | 163 | hrtimer_forward_now(hrt, cd.wrap_kt); |
32fea568 | 164 | |
a08ca5d1 | 165 | return HRTIMER_RESTART; |
112f38a4 RK |
166 | } |
167 | ||
32fea568 IM |
168 | void __init |
169 | sched_clock_register(u64 (*read)(void), int bits, unsigned long rate) | |
112f38a4 | 170 | { |
5ae8aabe SB |
171 | u64 res, wrap, new_mask, new_epoch, cyc, ns; |
172 | u32 new_mult, new_shift; | |
a08ca5d1 | 173 | unsigned long r; |
112f38a4 | 174 | char r_unit; |
1809bfa4 | 175 | struct clock_read_data rd; |
112f38a4 | 176 | |
c115739d RH |
177 | if (cd.rate > rate) |
178 | return; | |
179 | ||
2f0778af | 180 | WARN_ON(!irqs_disabled()); |
112f38a4 | 181 | |
32fea568 | 182 | /* Calculate the mult/shift to convert counter ticks to ns. */ |
5ae8aabe SB |
183 | clocks_calc_mult_shift(&new_mult, &new_shift, rate, NSEC_PER_SEC, 3600); |
184 | ||
185 | new_mask = CLOCKSOURCE_MASK(bits); | |
8710e914 | 186 | cd.rate = rate; |
5ae8aabe | 187 | |
32fea568 | 188 | /* Calculate how many nanosecs until we risk wrapping */ |
fb82fe2f | 189 | wrap = clocks_calc_max_nsecs(new_mult, new_shift, 0, new_mask, NULL); |
8710e914 | 190 | cd.wrap_kt = ns_to_ktime(wrap); |
5ae8aabe | 191 | |
1809bfa4 DT |
192 | rd = cd.read_data[0]; |
193 | ||
32fea568 | 194 | /* Update epoch for new counter and update 'epoch_ns' from old counter*/ |
5ae8aabe | 195 | new_epoch = read(); |
13dbeb38 | 196 | cyc = cd.actual_read_sched_clock(); |
32fea568 | 197 | ns = rd.epoch_ns + cyc_to_ns((cyc - rd.epoch_cyc) & rd.sched_clock_mask, rd.mult, rd.shift); |
13dbeb38 | 198 | cd.actual_read_sched_clock = read; |
5ae8aabe | 199 | |
32fea568 IM |
200 | rd.read_sched_clock = read; |
201 | rd.sched_clock_mask = new_mask; | |
202 | rd.mult = new_mult; | |
203 | rd.shift = new_shift; | |
204 | rd.epoch_cyc = new_epoch; | |
205 | rd.epoch_ns = ns; | |
206 | ||
1809bfa4 | 207 | update_clock_read_data(&rd); |
112f38a4 | 208 | |
1b8955bc DE |
209 | if (sched_clock_timer.function != NULL) { |
210 | /* update timeout for clock wrap */ | |
211 | hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); | |
212 | } | |
213 | ||
112f38a4 RK |
214 | r = rate; |
215 | if (r >= 4000000) { | |
216 | r /= 1000000; | |
217 | r_unit = 'M'; | |
32fea568 IM |
218 | } else { |
219 | if (r >= 1000) { | |
220 | r /= 1000; | |
221 | r_unit = 'k'; | |
222 | } else { | |
223 | r_unit = ' '; | |
224 | } | |
225 | } | |
226 | ||
227 | /* Calculate the ns resolution of this counter */ | |
5ae8aabe SB |
228 | res = cyc_to_ns(1ULL, new_mult, new_shift); |
229 | ||
a08ca5d1 SB |
230 | pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lluns\n", |
231 | bits, r, r_unit, res, wrap); | |
112f38a4 | 232 | |
32fea568 | 233 | /* Enable IRQ time accounting if we have a fast enough sched_clock() */ |
a42c3629 RK |
234 | if (irqtime > 0 || (irqtime == -1 && rate >= 1000000)) |
235 | enable_sched_clock_irqtime(); | |
236 | ||
2f0778af MZ |
237 | pr_debug("Registered %pF as sched_clock source\n", read); |
238 | } | |
239 | ||
211baa70 RK |
240 | void __init sched_clock_postinit(void) |
241 | { | |
2f0778af | 242 | /* |
32fea568 | 243 | * If no sched_clock() function has been provided at that point, |
2f0778af MZ |
244 | * make it the final one one. |
245 | */ | |
13dbeb38 | 246 | if (cd.actual_read_sched_clock == jiffy_sched_clock_read) |
e7e3ff1b | 247 | sched_clock_register(jiffy_sched_clock_read, BITS_PER_LONG, HZ); |
2f0778af | 248 | |
a08ca5d1 SB |
249 | update_sched_clock(); |
250 | ||
251 | /* | |
252 | * Start the timer to keep sched_clock() properly updated and | |
253 | * sets the initial epoch. | |
254 | */ | |
255 | hrtimer_init(&sched_clock_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | |
256 | sched_clock_timer.function = sched_clock_poll; | |
257 | hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); | |
211baa70 | 258 | } |
f153d017 | 259 | |
13dbeb38 DT |
260 | /* |
261 | * Clock read function for use when the clock is suspended. | |
262 | * | |
263 | * This function makes it appear to sched_clock() as if the clock | |
264 | * stopped counting at its last update. | |
1809bfa4 DT |
265 | * |
266 | * This function must only be called from the critical | |
267 | * section in sched_clock(). It relies on the read_seqcount_retry() | |
268 | * at the end of the critical section to be sure we observe the | |
32fea568 | 269 | * correct copy of 'epoch_cyc'. |
13dbeb38 DT |
270 | */ |
271 | static u64 notrace suspended_sched_clock_read(void) | |
272 | { | |
1809bfa4 DT |
273 | unsigned long seq = raw_read_seqcount(&cd.seq); |
274 | ||
275 | return cd.read_data[seq & 1].epoch_cyc; | |
13dbeb38 DT |
276 | } |
277 | ||
f153d017 RK |
278 | static int sched_clock_suspend(void) |
279 | { | |
1809bfa4 | 280 | struct clock_read_data *rd = &cd.read_data[0]; |
cf7c9c17 | 281 | |
f723aa18 SB |
282 | update_sched_clock(); |
283 | hrtimer_cancel(&sched_clock_timer); | |
13dbeb38 | 284 | rd->read_sched_clock = suspended_sched_clock_read; |
32fea568 | 285 | |
f153d017 RK |
286 | return 0; |
287 | } | |
288 | ||
237ec6f2 CC |
289 | static void sched_clock_resume(void) |
290 | { | |
1809bfa4 | 291 | struct clock_read_data *rd = &cd.read_data[0]; |
cf7c9c17 | 292 | |
13dbeb38 | 293 | rd->epoch_cyc = cd.actual_read_sched_clock(); |
f723aa18 | 294 | hrtimer_start(&sched_clock_timer, cd.wrap_kt, HRTIMER_MODE_REL); |
13dbeb38 | 295 | rd->read_sched_clock = cd.actual_read_sched_clock; |
237ec6f2 CC |
296 | } |
297 | ||
f153d017 | 298 | static struct syscore_ops sched_clock_ops = { |
32fea568 IM |
299 | .suspend = sched_clock_suspend, |
300 | .resume = sched_clock_resume, | |
f153d017 RK |
301 | }; |
302 | ||
303 | static int __init sched_clock_syscore_init(void) | |
304 | { | |
305 | register_syscore_ops(&sched_clock_ops); | |
32fea568 | 306 | |
f153d017 RK |
307 | return 0; |
308 | } | |
309 | device_initcall(sched_clock_syscore_init); |