]>
Commit | Line | Data |
---|---|---|
653ed64b JFG |
1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | // | |
8e4ec3d0 | 3 | // Scalability test comparing RCU vs other mechanisms |
653ed64b JFG |
4 | // for acquiring references on objects. |
5 | // | |
6 | // Copyright (C) Google, 2020. | |
7 | // | |
8 | // Author: Joel Fernandes <joel@joelfernandes.org> | |
9 | ||
10 | #define pr_fmt(fmt) fmt | |
11 | ||
12 | #include <linux/atomic.h> | |
13 | #include <linux/bitops.h> | |
14 | #include <linux/completion.h> | |
15 | #include <linux/cpu.h> | |
16 | #include <linux/delay.h> | |
17 | #include <linux/err.h> | |
18 | #include <linux/init.h> | |
19 | #include <linux/interrupt.h> | |
20 | #include <linux/kthread.h> | |
21 | #include <linux/kernel.h> | |
22 | #include <linux/mm.h> | |
23 | #include <linux/module.h> | |
24 | #include <linux/moduleparam.h> | |
25 | #include <linux/notifier.h> | |
26 | #include <linux/percpu.h> | |
27 | #include <linux/rcupdate.h> | |
72bb749e | 28 | #include <linux/rcupdate_trace.h> |
653ed64b JFG |
29 | #include <linux/reboot.h> |
30 | #include <linux/sched.h> | |
31 | #include <linux/spinlock.h> | |
32 | #include <linux/smp.h> | |
33 | #include <linux/stat.h> | |
34 | #include <linux/srcu.h> | |
35 | #include <linux/slab.h> | |
36 | #include <linux/torture.h> | |
37 | #include <linux/types.h> | |
38 | ||
39 | #include "rcu.h" | |
40 | ||
1fbeb3a8 | 41 | #define SCALE_FLAG "-ref-scale: " |
653ed64b | 42 | |
1fbeb3a8 PM |
43 | #define SCALEOUT(s, x...) \ |
44 | pr_alert("%s" SCALE_FLAG s, scale_type, ## x) | |
653ed64b | 45 | |
1fbeb3a8 | 46 | #define VERBOSE_SCALEOUT(s, x...) \ |
f71f22b6 LZ |
47 | do { \ |
48 | if (verbose) \ | |
49 | pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \ | |
50 | } while (0) | |
653ed64b | 51 | |
e76506f0 PM |
52 | static atomic_t verbose_batch_ctr; |
53 | ||
54 | #define VERBOSE_SCALEOUT_BATCH(s, x...) \ | |
55 | do { \ | |
56 | if (verbose && \ | |
57 | (verbose_batched <= 0 || \ | |
414c116e PM |
58 | !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) { \ |
59 | schedule_timeout_uninterruptible(1); \ | |
f71f22b6 | 60 | pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \ |
414c116e | 61 | } \ |
e76506f0 PM |
62 | } while (0) |
63 | ||
f71f22b6 | 64 | #define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x) |
653ed64b JFG |
65 | |
66 | MODULE_LICENSE("GPL"); | |
67 | MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>"); | |
68 | ||
1fbeb3a8 PM |
69 | static char *scale_type = "rcu"; |
70 | module_param(scale_type, charp, 0444); | |
71 | MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock."); | |
653ed64b JFG |
72 | |
73 | torture_param(int, verbose, 0, "Enable verbose debugging printk()s"); | |
e76506f0 | 74 | torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s"); |
653ed64b | 75 | |
777a54c9 | 76 | // Wait until there are multiple CPUs before starting test. |
8e4ec3d0 | 77 | torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0, |
777a54c9 PM |
78 | "Holdoff time before test start (s)"); |
79 | // Number of loops per experiment, all readers execute operations concurrently. | |
4dd72a33 | 80 | torture_param(long, loops, 10000, "Number of loops per experiment."); |
8fc28783 PM |
81 | // Number of readers, with -1 defaulting to about 75% of the CPUs. |
82 | torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs."); | |
83 | // Number of runs. | |
84 | torture_param(int, nruns, 30, "Number of experiments to run."); | |
918b351d PM |
85 | // Reader delay in nanoseconds, 0 for no delay. |
86 | torture_param(int, readdelay, 0, "Read-side delay in nanoseconds."); | |
653ed64b JFG |
87 | |
88 | #ifdef MODULE | |
1fbeb3a8 | 89 | # define REFSCALE_SHUTDOWN 0 |
653ed64b | 90 | #else |
1fbeb3a8 | 91 | # define REFSCALE_SHUTDOWN 1 |
653ed64b JFG |
92 | #endif |
93 | ||
1fbeb3a8 PM |
94 | torture_param(bool, shutdown, REFSCALE_SHUTDOWN, |
95 | "Shutdown at end of scalability tests."); | |
653ed64b JFG |
96 | |
97 | struct reader_task { | |
98 | struct task_struct *task; | |
af2789db | 99 | int start_reader; |
653ed64b JFG |
100 | wait_queue_head_t wq; |
101 | u64 last_duration_ns; | |
653ed64b JFG |
102 | }; |
103 | ||
104 | static struct task_struct *shutdown_task; | |
105 | static wait_queue_head_t shutdown_wq; | |
106 | ||
107 | static struct task_struct *main_task; | |
108 | static wait_queue_head_t main_wq; | |
109 | static int shutdown_start; | |
110 | ||
111 | static struct reader_task *reader_tasks; | |
653ed64b JFG |
112 | |
113 | // Number of readers that are part of the current experiment. | |
114 | static atomic_t nreaders_exp; | |
115 | ||
116 | // Use to wait for all threads to start. | |
117 | static atomic_t n_init; | |
86e0da2b | 118 | static atomic_t n_started; |
2db0bda3 PM |
119 | static atomic_t n_warmedup; |
120 | static atomic_t n_cooleddown; | |
653ed64b JFG |
121 | |
122 | // Track which experiment is currently running. | |
123 | static int exp_idx; | |
124 | ||
125 | // Operations vector for selecting different types of tests. | |
1fbeb3a8 | 126 | struct ref_scale_ops { |
653ed64b JFG |
127 | void (*init)(void); |
128 | void (*cleanup)(void); | |
75dd8efe | 129 | void (*readsection)(const int nloops); |
918b351d | 130 | void (*delaysection)(const int nloops, const int udl, const int ndl); |
653ed64b JFG |
131 | const char *name; |
132 | }; | |
133 | ||
1fbeb3a8 | 134 | static struct ref_scale_ops *cur_ops; |
653ed64b | 135 | |
918b351d PM |
136 | static void un_delay(const int udl, const int ndl) |
137 | { | |
138 | if (udl) | |
139 | udelay(udl); | |
140 | if (ndl) | |
141 | ndelay(ndl); | |
142 | } | |
143 | ||
75dd8efe | 144 | static void ref_rcu_read_section(const int nloops) |
653ed64b | 145 | { |
75dd8efe | 146 | int i; |
653ed64b | 147 | |
75dd8efe PM |
148 | for (i = nloops; i >= 0; i--) { |
149 | rcu_read_lock(); | |
150 | rcu_read_unlock(); | |
151 | } | |
653ed64b JFG |
152 | } |
153 | ||
918b351d | 154 | static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl) |
b4d1e34f PM |
155 | { |
156 | int i; | |
157 | ||
158 | for (i = nloops; i >= 0; i--) { | |
159 | rcu_read_lock(); | |
918b351d | 160 | un_delay(udl, ndl); |
b4d1e34f PM |
161 | rcu_read_unlock(); |
162 | } | |
163 | } | |
164 | ||
1fbeb3a8 | 165 | static void rcu_sync_scale_init(void) |
653ed64b JFG |
166 | { |
167 | } | |
168 | ||
1fbeb3a8 PM |
169 | static struct ref_scale_ops rcu_ops = { |
170 | .init = rcu_sync_scale_init, | |
75dd8efe | 171 | .readsection = ref_rcu_read_section, |
b4d1e34f | 172 | .delaysection = ref_rcu_delay_section, |
653ed64b JFG |
173 | .name = "rcu" |
174 | }; | |
175 | ||
1fbeb3a8 PM |
176 | // Definitions for SRCU ref scale testing. |
177 | DEFINE_STATIC_SRCU(srcu_refctl_scale); | |
178 | static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale; | |
653ed64b | 179 | |
1fbeb3a8 | 180 | static void srcu_ref_scale_read_section(const int nloops) |
653ed64b | 181 | { |
75dd8efe PM |
182 | int i; |
183 | int idx; | |
653ed64b | 184 | |
75dd8efe PM |
185 | for (i = nloops; i >= 0; i--) { |
186 | idx = srcu_read_lock(srcu_ctlp); | |
187 | srcu_read_unlock(srcu_ctlp, idx); | |
188 | } | |
653ed64b JFG |
189 | } |
190 | ||
1fbeb3a8 | 191 | static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl) |
b4d1e34f PM |
192 | { |
193 | int i; | |
194 | int idx; | |
195 | ||
196 | for (i = nloops; i >= 0; i--) { | |
197 | idx = srcu_read_lock(srcu_ctlp); | |
918b351d | 198 | un_delay(udl, ndl); |
b4d1e34f PM |
199 | srcu_read_unlock(srcu_ctlp, idx); |
200 | } | |
201 | } | |
202 | ||
1fbeb3a8 PM |
203 | static struct ref_scale_ops srcu_ops = { |
204 | .init = rcu_sync_scale_init, | |
205 | .readsection = srcu_ref_scale_read_section, | |
206 | .delaysection = srcu_ref_scale_delay_section, | |
653ed64b JFG |
207 | .name = "srcu" |
208 | }; | |
209 | ||
5f654af1 PM |
210 | #ifdef CONFIG_TASKS_RCU |
211 | ||
1fbeb3a8 | 212 | // Definitions for RCU Tasks ref scale testing: Empty read markers. |
e13ef442 | 213 | // These definitions also work for RCU Rude readers. |
1fbeb3a8 | 214 | static void rcu_tasks_ref_scale_read_section(const int nloops) |
e13ef442 PM |
215 | { |
216 | int i; | |
217 | ||
218 | for (i = nloops; i >= 0; i--) | |
219 | continue; | |
220 | } | |
221 | ||
1fbeb3a8 | 222 | static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl) |
e13ef442 PM |
223 | { |
224 | int i; | |
225 | ||
226 | for (i = nloops; i >= 0; i--) | |
227 | un_delay(udl, ndl); | |
228 | } | |
229 | ||
1fbeb3a8 PM |
230 | static struct ref_scale_ops rcu_tasks_ops = { |
231 | .init = rcu_sync_scale_init, | |
232 | .readsection = rcu_tasks_ref_scale_read_section, | |
233 | .delaysection = rcu_tasks_ref_scale_delay_section, | |
e13ef442 PM |
234 | .name = "rcu-tasks" |
235 | }; | |
236 | ||
5f654af1 PM |
237 | #define RCU_TASKS_OPS &rcu_tasks_ops, |
238 | ||
239 | #else // #ifdef CONFIG_TASKS_RCU | |
240 | ||
241 | #define RCU_TASKS_OPS | |
242 | ||
243 | #endif // #else // #ifdef CONFIG_TASKS_RCU | |
244 | ||
dec86781 PM |
245 | #ifdef CONFIG_TASKS_TRACE_RCU |
246 | ||
1fbeb3a8 PM |
247 | // Definitions for RCU Tasks Trace ref scale testing. |
248 | static void rcu_trace_ref_scale_read_section(const int nloops) | |
72bb749e PM |
249 | { |
250 | int i; | |
251 | ||
252 | for (i = nloops; i >= 0; i--) { | |
253 | rcu_read_lock_trace(); | |
254 | rcu_read_unlock_trace(); | |
255 | } | |
256 | } | |
257 | ||
1fbeb3a8 | 258 | static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl) |
72bb749e PM |
259 | { |
260 | int i; | |
261 | ||
262 | for (i = nloops; i >= 0; i--) { | |
263 | rcu_read_lock_trace(); | |
264 | un_delay(udl, ndl); | |
265 | rcu_read_unlock_trace(); | |
266 | } | |
267 | } | |
268 | ||
1fbeb3a8 PM |
269 | static struct ref_scale_ops rcu_trace_ops = { |
270 | .init = rcu_sync_scale_init, | |
271 | .readsection = rcu_trace_ref_scale_read_section, | |
272 | .delaysection = rcu_trace_ref_scale_delay_section, | |
72bb749e PM |
273 | .name = "rcu-trace" |
274 | }; | |
275 | ||
dec86781 PM |
276 | #define RCU_TRACE_OPS &rcu_trace_ops, |
277 | ||
278 | #else // #ifdef CONFIG_TASKS_TRACE_RCU | |
279 | ||
280 | #define RCU_TRACE_OPS | |
281 | ||
282 | #endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU | |
283 | ||
653ed64b JFG |
284 | // Definitions for reference count |
285 | static atomic_t refcnt; | |
286 | ||
b4d1e34f | 287 | static void ref_refcnt_section(const int nloops) |
653ed64b | 288 | { |
75dd8efe | 289 | int i; |
653ed64b | 290 | |
75dd8efe PM |
291 | for (i = nloops; i >= 0; i--) { |
292 | atomic_inc(&refcnt); | |
293 | atomic_dec(&refcnt); | |
294 | } | |
653ed64b JFG |
295 | } |
296 | ||
918b351d | 297 | static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl) |
b4d1e34f PM |
298 | { |
299 | int i; | |
300 | ||
301 | for (i = nloops; i >= 0; i--) { | |
302 | atomic_inc(&refcnt); | |
918b351d | 303 | un_delay(udl, ndl); |
b4d1e34f PM |
304 | atomic_dec(&refcnt); |
305 | } | |
306 | } | |
307 | ||
1fbeb3a8 PM |
308 | static struct ref_scale_ops refcnt_ops = { |
309 | .init = rcu_sync_scale_init, | |
b4d1e34f PM |
310 | .readsection = ref_refcnt_section, |
311 | .delaysection = ref_refcnt_delay_section, | |
653ed64b JFG |
312 | .name = "refcnt" |
313 | }; | |
314 | ||
315 | // Definitions for rwlock | |
316 | static rwlock_t test_rwlock; | |
317 | ||
b4d1e34f | 318 | static void ref_rwlock_init(void) |
653ed64b JFG |
319 | { |
320 | rwlock_init(&test_rwlock); | |
321 | } | |
322 | ||
b4d1e34f PM |
323 | static void ref_rwlock_section(const int nloops) |
324 | { | |
325 | int i; | |
326 | ||
327 | for (i = nloops; i >= 0; i--) { | |
328 | read_lock(&test_rwlock); | |
329 | read_unlock(&test_rwlock); | |
330 | } | |
331 | } | |
332 | ||
918b351d | 333 | static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl) |
653ed64b | 334 | { |
75dd8efe | 335 | int i; |
653ed64b | 336 | |
75dd8efe PM |
337 | for (i = nloops; i >= 0; i--) { |
338 | read_lock(&test_rwlock); | |
918b351d | 339 | un_delay(udl, ndl); |
75dd8efe PM |
340 | read_unlock(&test_rwlock); |
341 | } | |
653ed64b JFG |
342 | } |
343 | ||
1fbeb3a8 | 344 | static struct ref_scale_ops rwlock_ops = { |
b4d1e34f PM |
345 | .init = ref_rwlock_init, |
346 | .readsection = ref_rwlock_section, | |
347 | .delaysection = ref_rwlock_delay_section, | |
653ed64b JFG |
348 | .name = "rwlock" |
349 | }; | |
350 | ||
351 | // Definitions for rwsem | |
352 | static struct rw_semaphore test_rwsem; | |
353 | ||
b4d1e34f | 354 | static void ref_rwsem_init(void) |
653ed64b JFG |
355 | { |
356 | init_rwsem(&test_rwsem); | |
357 | } | |
358 | ||
b4d1e34f | 359 | static void ref_rwsem_section(const int nloops) |
653ed64b | 360 | { |
75dd8efe | 361 | int i; |
653ed64b | 362 | |
75dd8efe PM |
363 | for (i = nloops; i >= 0; i--) { |
364 | down_read(&test_rwsem); | |
365 | up_read(&test_rwsem); | |
366 | } | |
653ed64b JFG |
367 | } |
368 | ||
918b351d | 369 | static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl) |
b4d1e34f PM |
370 | { |
371 | int i; | |
372 | ||
373 | for (i = nloops; i >= 0; i--) { | |
374 | down_read(&test_rwsem); | |
918b351d | 375 | un_delay(udl, ndl); |
b4d1e34f PM |
376 | up_read(&test_rwsem); |
377 | } | |
378 | } | |
379 | ||
1fbeb3a8 | 380 | static struct ref_scale_ops rwsem_ops = { |
b4d1e34f PM |
381 | .init = ref_rwsem_init, |
382 | .readsection = ref_rwsem_section, | |
383 | .delaysection = ref_rwsem_delay_section, | |
653ed64b JFG |
384 | .name = "rwsem" |
385 | }; | |
386 | ||
e9b800db | 387 | // Definitions for global spinlock |
7bf336fb | 388 | static DEFINE_RAW_SPINLOCK(test_lock); |
e9b800db PM |
389 | |
390 | static void ref_lock_section(const int nloops) | |
391 | { | |
392 | int i; | |
393 | ||
394 | preempt_disable(); | |
395 | for (i = nloops; i >= 0; i--) { | |
7bf336fb Z |
396 | raw_spin_lock(&test_lock); |
397 | raw_spin_unlock(&test_lock); | |
e9b800db PM |
398 | } |
399 | preempt_enable(); | |
400 | } | |
401 | ||
402 | static void ref_lock_delay_section(const int nloops, const int udl, const int ndl) | |
403 | { | |
404 | int i; | |
405 | ||
406 | preempt_disable(); | |
407 | for (i = nloops; i >= 0; i--) { | |
7bf336fb | 408 | raw_spin_lock(&test_lock); |
e9b800db | 409 | un_delay(udl, ndl); |
7bf336fb | 410 | raw_spin_unlock(&test_lock); |
e9b800db PM |
411 | } |
412 | preempt_enable(); | |
413 | } | |
414 | ||
415 | static struct ref_scale_ops lock_ops = { | |
416 | .readsection = ref_lock_section, | |
417 | .delaysection = ref_lock_delay_section, | |
418 | .name = "lock" | |
419 | }; | |
420 | ||
421 | // Definitions for global irq-save spinlock | |
422 | ||
423 | static void ref_lock_irq_section(const int nloops) | |
424 | { | |
425 | unsigned long flags; | |
426 | int i; | |
427 | ||
428 | preempt_disable(); | |
429 | for (i = nloops; i >= 0; i--) { | |
7bf336fb Z |
430 | raw_spin_lock_irqsave(&test_lock, flags); |
431 | raw_spin_unlock_irqrestore(&test_lock, flags); | |
e9b800db PM |
432 | } |
433 | preempt_enable(); | |
434 | } | |
435 | ||
436 | static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl) | |
437 | { | |
438 | unsigned long flags; | |
439 | int i; | |
440 | ||
441 | preempt_disable(); | |
442 | for (i = nloops; i >= 0; i--) { | |
7bf336fb | 443 | raw_spin_lock_irqsave(&test_lock, flags); |
e9b800db | 444 | un_delay(udl, ndl); |
7bf336fb | 445 | raw_spin_unlock_irqrestore(&test_lock, flags); |
e9b800db PM |
446 | } |
447 | preempt_enable(); | |
448 | } | |
449 | ||
450 | static struct ref_scale_ops lock_irq_ops = { | |
451 | .readsection = ref_lock_irq_section, | |
452 | .delaysection = ref_lock_irq_delay_section, | |
453 | .name = "lock-irq" | |
454 | }; | |
455 | ||
456 | // Definitions acquire-release. | |
457 | static DEFINE_PER_CPU(unsigned long, test_acqrel); | |
458 | ||
459 | static void ref_acqrel_section(const int nloops) | |
460 | { | |
461 | unsigned long x; | |
462 | int i; | |
463 | ||
464 | preempt_disable(); | |
465 | for (i = nloops; i >= 0; i--) { | |
466 | x = smp_load_acquire(this_cpu_ptr(&test_acqrel)); | |
467 | smp_store_release(this_cpu_ptr(&test_acqrel), x + 1); | |
468 | } | |
469 | preempt_enable(); | |
470 | } | |
471 | ||
472 | static void ref_acqrel_delay_section(const int nloops, const int udl, const int ndl) | |
473 | { | |
474 | unsigned long x; | |
475 | int i; | |
476 | ||
477 | preempt_disable(); | |
478 | for (i = nloops; i >= 0; i--) { | |
479 | x = smp_load_acquire(this_cpu_ptr(&test_acqrel)); | |
480 | un_delay(udl, ndl); | |
481 | smp_store_release(this_cpu_ptr(&test_acqrel), x + 1); | |
482 | } | |
483 | preempt_enable(); | |
484 | } | |
485 | ||
486 | static struct ref_scale_ops acqrel_ops = { | |
487 | .readsection = ref_acqrel_section, | |
488 | .delaysection = ref_acqrel_delay_section, | |
489 | .name = "acqrel" | |
490 | }; | |
491 | ||
25f6fa53 PM |
492 | static volatile u64 stopopts; |
493 | ||
494 | static void ref_clock_section(const int nloops) | |
495 | { | |
496 | u64 x = 0; | |
497 | int i; | |
498 | ||
499 | preempt_disable(); | |
500 | for (i = nloops; i >= 0; i--) | |
501 | x += ktime_get_real_fast_ns(); | |
502 | preempt_enable(); | |
503 | stopopts = x; | |
504 | } | |
505 | ||
506 | static void ref_clock_delay_section(const int nloops, const int udl, const int ndl) | |
507 | { | |
508 | u64 x = 0; | |
509 | int i; | |
510 | ||
511 | preempt_disable(); | |
512 | for (i = nloops; i >= 0; i--) { | |
513 | x += ktime_get_real_fast_ns(); | |
514 | un_delay(udl, ndl); | |
515 | } | |
516 | preempt_enable(); | |
517 | stopopts = x; | |
518 | } | |
519 | ||
520 | static struct ref_scale_ops clock_ops = { | |
521 | .readsection = ref_clock_section, | |
522 | .delaysection = ref_clock_delay_section, | |
523 | .name = "clock" | |
524 | }; | |
525 | ||
1fbeb3a8 | 526 | static void rcu_scale_one_reader(void) |
b4d1e34f PM |
527 | { |
528 | if (readdelay <= 0) | |
529 | cur_ops->readsection(loops); | |
530 | else | |
918b351d | 531 | cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000); |
b4d1e34f PM |
532 | } |
533 | ||
653ed64b JFG |
534 | // Reader kthread. Repeatedly does empty RCU read-side |
535 | // critical section, minimizing update-side interference. | |
536 | static int | |
1fbeb3a8 | 537 | ref_scale_reader(void *arg) |
653ed64b JFG |
538 | { |
539 | unsigned long flags; | |
540 | long me = (long)arg; | |
541 | struct reader_task *rt = &(reader_tasks[me]); | |
653ed64b JFG |
542 | u64 start; |
543 | s64 duration; | |
544 | ||
e76506f0 | 545 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me); |
05bc276c | 546 | WARN_ON_ONCE(set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids))); |
653ed64b JFG |
547 | set_user_nice(current, MAX_NICE); |
548 | atomic_inc(&n_init); | |
777a54c9 PM |
549 | if (holdoff) |
550 | schedule_timeout_interruptible(holdoff * HZ); | |
653ed64b | 551 | repeat: |
05bc276c | 552 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, raw_smp_processor_id()); |
653ed64b JFG |
553 | |
554 | // Wait for signal that this reader can start. | |
af2789db | 555 | wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) || |
653ed64b JFG |
556 | torture_must_stop()); |
557 | ||
558 | if (torture_must_stop()) | |
559 | goto end; | |
560 | ||
561 | // Make sure that the CPU is affinitized appropriately during testing. | |
05bc276c | 562 | WARN_ON_ONCE(raw_smp_processor_id() != me); |
653ed64b | 563 | |
af2789db | 564 | WRITE_ONCE(rt->start_reader, 0); |
86e0da2b PM |
565 | if (!atomic_dec_return(&n_started)) |
566 | while (atomic_read_acquire(&n_started)) | |
567 | cpu_relax(); | |
653ed64b | 568 | |
e76506f0 | 569 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx); |
b864f89f | 570 | |
2db0bda3 PM |
571 | |
572 | // To reduce noise, do an initial cache-warming invocation, check | |
573 | // in, and then keep warming until everyone has checked in. | |
1fbeb3a8 | 574 | rcu_scale_one_reader(); |
2db0bda3 PM |
575 | if (!atomic_dec_return(&n_warmedup)) |
576 | while (atomic_read_acquire(&n_warmedup)) | |
1fbeb3a8 | 577 | rcu_scale_one_reader(); |
2db0bda3 PM |
578 | // Also keep interrupts disabled. This also has the effect |
579 | // of preventing entries into slow path for rcu_read_unlock(). | |
653ed64b JFG |
580 | local_irq_save(flags); |
581 | start = ktime_get_mono_fast_ns(); | |
582 | ||
1fbeb3a8 | 583 | rcu_scale_one_reader(); |
653ed64b JFG |
584 | |
585 | duration = ktime_get_mono_fast_ns() - start; | |
586 | local_irq_restore(flags); | |
587 | ||
588 | rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration; | |
2db0bda3 PM |
589 | // To reduce runtime-skew noise, do maintain-load invocations until |
590 | // everyone is done. | |
591 | if (!atomic_dec_return(&n_cooleddown)) | |
592 | while (atomic_read_acquire(&n_cooleddown)) | |
1fbeb3a8 | 593 | rcu_scale_one_reader(); |
653ed64b | 594 | |
b864f89f PM |
595 | if (atomic_dec_and_test(&nreaders_exp)) |
596 | wake_up(&main_wq); | |
653ed64b | 597 | |
e76506f0 PM |
598 | VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)", |
599 | me, exp_idx, atomic_read(&nreaders_exp)); | |
653ed64b | 600 | |
653ed64b JFG |
601 | if (!torture_must_stop()) |
602 | goto repeat; | |
603 | end: | |
1fbeb3a8 | 604 | torture_kthread_stopping("ref_scale_reader"); |
653ed64b JFG |
605 | return 0; |
606 | } | |
607 | ||
2990750b | 608 | static void reset_readers(void) |
653ed64b JFG |
609 | { |
610 | int i; | |
611 | struct reader_task *rt; | |
612 | ||
dbf28efd | 613 | for (i = 0; i < nreaders; i++) { |
653ed64b JFG |
614 | rt = &(reader_tasks[i]); |
615 | ||
616 | rt->last_duration_ns = 0; | |
617 | } | |
618 | } | |
619 | ||
620 | // Print the results of each reader and return the sum of all their durations. | |
2990750b | 621 | static u64 process_durations(int n) |
653ed64b JFG |
622 | { |
623 | int i; | |
624 | struct reader_task *rt; | |
625 | char buf1[64]; | |
2e90de76 | 626 | char *buf; |
653ed64b JFG |
627 | u64 sum = 0; |
628 | ||
9880eb87 | 629 | buf = kmalloc(800 + 64, GFP_KERNEL); |
2e90de76 PM |
630 | if (!buf) |
631 | return 0; | |
653ed64b JFG |
632 | buf[0] = 0; |
633 | sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)", | |
634 | exp_idx); | |
635 | ||
dbf28efd | 636 | for (i = 0; i < n && !torture_must_stop(); i++) { |
653ed64b JFG |
637 | rt = &(reader_tasks[i]); |
638 | sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns); | |
639 | ||
640 | if (i % 5 == 0) | |
641 | strcat(buf, "\n"); | |
9880eb87 LZ |
642 | if (strlen(buf) >= 800) { |
643 | pr_alert("%s", buf); | |
644 | buf[0] = 0; | |
645 | } | |
653ed64b JFG |
646 | strcat(buf, buf1); |
647 | ||
648 | sum += rt->last_duration_ns; | |
649 | } | |
9880eb87 | 650 | pr_alert("%s\n", buf); |
653ed64b | 651 | |
2e90de76 | 652 | kfree(buf); |
653ed64b JFG |
653 | return sum; |
654 | } | |
655 | ||
656 | // The main_func is the main orchestrator, it performs a bunch of | |
657 | // experiments. For every experiment, it orders all the readers | |
658 | // involved to start and waits for them to finish the experiment. It | |
659 | // then reads their timestamps and starts the next experiment. Each | |
660 | // experiment progresses from 1 concurrent reader to N of them at which | |
661 | // point all the timestamps are printed. | |
662 | static int main_func(void *arg) | |
663 | { | |
664 | int exp, r; | |
665 | char buf1[64]; | |
f518f154 | 666 | char *buf; |
dbf28efd | 667 | u64 *result_avg; |
653ed64b JFG |
668 | |
669 | set_cpus_allowed_ptr(current, cpumask_of(nreaders % nr_cpu_ids)); | |
670 | set_user_nice(current, MAX_NICE); | |
671 | ||
1fbeb3a8 | 672 | VERBOSE_SCALEOUT("main_func task started"); |
dbf28efd | 673 | result_avg = kzalloc(nruns * sizeof(*result_avg), GFP_KERNEL); |
9880eb87 | 674 | buf = kzalloc(800 + 64, GFP_KERNEL); |
f518f154 | 675 | if (!result_avg || !buf) { |
4feeb9d5 | 676 | SCALEOUT_ERRSTRING("out of memory"); |
c30c8763 | 677 | goto oom_exit; |
f518f154 | 678 | } |
777a54c9 PM |
679 | if (holdoff) |
680 | schedule_timeout_interruptible(holdoff * HZ); | |
653ed64b | 681 | |
96af8669 PM |
682 | // Wait for all threads to start. |
683 | atomic_inc(&n_init); | |
684 | while (atomic_read(&n_init) < nreaders + 1) | |
685 | schedule_timeout_uninterruptible(1); | |
686 | ||
653ed64b | 687 | // Start exp readers up per experiment |
dbf28efd | 688 | for (exp = 0; exp < nruns && !torture_must_stop(); exp++) { |
653ed64b JFG |
689 | if (torture_must_stop()) |
690 | goto end; | |
691 | ||
dbf28efd PM |
692 | reset_readers(); |
693 | atomic_set(&nreaders_exp, nreaders); | |
86e0da2b | 694 | atomic_set(&n_started, nreaders); |
2db0bda3 PM |
695 | atomic_set(&n_warmedup, nreaders); |
696 | atomic_set(&n_cooleddown, nreaders); | |
653ed64b JFG |
697 | |
698 | exp_idx = exp; | |
699 | ||
dbf28efd | 700 | for (r = 0; r < nreaders; r++) { |
af2789db | 701 | smp_store_release(&reader_tasks[r].start_reader, 1); |
653ed64b JFG |
702 | wake_up(&reader_tasks[r].wq); |
703 | } | |
704 | ||
1fbeb3a8 | 705 | VERBOSE_SCALEOUT("main_func: experiment started, waiting for %d readers", |
dbf28efd | 706 | nreaders); |
653ed64b JFG |
707 | |
708 | wait_event(main_wq, | |
709 | !atomic_read(&nreaders_exp) || torture_must_stop()); | |
710 | ||
1fbeb3a8 | 711 | VERBOSE_SCALEOUT("main_func: experiment ended"); |
653ed64b JFG |
712 | |
713 | if (torture_must_stop()) | |
714 | goto end; | |
715 | ||
7c944d7c | 716 | result_avg[exp] = div_u64(1000 * process_durations(nreaders), nreaders * loops); |
653ed64b JFG |
717 | } |
718 | ||
719 | // Print the average of all experiments | |
1fbeb3a8 | 720 | SCALEOUT("END OF TEST. Calculating average duration per loop (nanoseconds)...\n"); |
653ed64b | 721 | |
9880eb87 | 722 | pr_alert("Runs\tTime(ns)\n"); |
dbf28efd | 723 | for (exp = 0; exp < nruns; exp++) { |
7c944d7c AB |
724 | u64 avg; |
725 | u32 rem; | |
726 | ||
7c944d7c AB |
727 | avg = div_u64_rem(result_avg[exp], 1000, &rem); |
728 | sprintf(buf1, "%d\t%llu.%03u\n", exp + 1, avg, rem); | |
653ed64b | 729 | strcat(buf, buf1); |
9880eb87 LZ |
730 | if (strlen(buf) >= 800) { |
731 | pr_alert("%s", buf); | |
732 | buf[0] = 0; | |
733 | } | |
653ed64b JFG |
734 | } |
735 | ||
9880eb87 | 736 | pr_alert("%s", buf); |
653ed64b | 737 | |
c30c8763 | 738 | oom_exit: |
653ed64b JFG |
739 | // This will shutdown everything including us. |
740 | if (shutdown) { | |
741 | shutdown_start = 1; | |
742 | wake_up(&shutdown_wq); | |
743 | } | |
744 | ||
745 | // Wait for torture to stop us | |
746 | while (!torture_must_stop()) | |
747 | schedule_timeout_uninterruptible(1); | |
748 | ||
749 | end: | |
750 | torture_kthread_stopping("main_func"); | |
f518f154 PM |
751 | kfree(result_avg); |
752 | kfree(buf); | |
653ed64b JFG |
753 | return 0; |
754 | } | |
755 | ||
756 | static void | |
1fbeb3a8 | 757 | ref_scale_print_module_parms(struct ref_scale_ops *cur_ops, const char *tag) |
653ed64b | 758 | { |
1fbeb3a8 PM |
759 | pr_alert("%s" SCALE_FLAG |
760 | "--- %s: verbose=%d shutdown=%d holdoff=%d loops=%ld nreaders=%d nruns=%d readdelay=%d\n", scale_type, tag, | |
b4d1e34f | 761 | verbose, shutdown, holdoff, loops, nreaders, nruns, readdelay); |
653ed64b JFG |
762 | } |
763 | ||
764 | static void | |
1fbeb3a8 | 765 | ref_scale_cleanup(void) |
653ed64b JFG |
766 | { |
767 | int i; | |
768 | ||
769 | if (torture_cleanup_begin()) | |
770 | return; | |
771 | ||
772 | if (!cur_ops) { | |
773 | torture_cleanup_end(); | |
774 | return; | |
775 | } | |
776 | ||
777 | if (reader_tasks) { | |
778 | for (i = 0; i < nreaders; i++) | |
1fbeb3a8 | 779 | torture_stop_kthread("ref_scale_reader", |
653ed64b JFG |
780 | reader_tasks[i].task); |
781 | } | |
782 | kfree(reader_tasks); | |
783 | ||
784 | torture_stop_kthread("main_task", main_task); | |
785 | kfree(main_task); | |
786 | ||
1fbeb3a8 | 787 | // Do scale-type-specific cleanup operations. |
653ed64b JFG |
788 | if (cur_ops->cleanup != NULL) |
789 | cur_ops->cleanup(); | |
790 | ||
791 | torture_cleanup_end(); | |
792 | } | |
793 | ||
794 | // Shutdown kthread. Just waits to be awakened, then shuts down system. | |
795 | static int | |
1fbeb3a8 | 796 | ref_scale_shutdown(void *arg) |
653ed64b JFG |
797 | { |
798 | wait_event(shutdown_wq, shutdown_start); | |
799 | ||
800 | smp_mb(); // Wake before output. | |
1fbeb3a8 | 801 | ref_scale_cleanup(); |
653ed64b JFG |
802 | kernel_power_off(); |
803 | ||
804 | return -EINVAL; | |
805 | } | |
806 | ||
807 | static int __init | |
1fbeb3a8 | 808 | ref_scale_init(void) |
653ed64b JFG |
809 | { |
810 | long i; | |
811 | int firsterr = 0; | |
1fbeb3a8 | 812 | static struct ref_scale_ops *scale_ops[] = { |
dec86781 | 813 | &rcu_ops, &srcu_ops, RCU_TRACE_OPS RCU_TASKS_OPS &refcnt_ops, &rwlock_ops, |
25f6fa53 | 814 | &rwsem_ops, &lock_ops, &lock_irq_ops, &acqrel_ops, &clock_ops, |
653ed64b JFG |
815 | }; |
816 | ||
1fbeb3a8 | 817 | if (!torture_init_begin(scale_type, verbose)) |
653ed64b JFG |
818 | return -EBUSY; |
819 | ||
1fbeb3a8 PM |
820 | for (i = 0; i < ARRAY_SIZE(scale_ops); i++) { |
821 | cur_ops = scale_ops[i]; | |
822 | if (strcmp(scale_type, cur_ops->name) == 0) | |
653ed64b JFG |
823 | break; |
824 | } | |
1fbeb3a8 PM |
825 | if (i == ARRAY_SIZE(scale_ops)) { |
826 | pr_alert("rcu-scale: invalid scale type: \"%s\"\n", scale_type); | |
827 | pr_alert("rcu-scale types:"); | |
828 | for (i = 0; i < ARRAY_SIZE(scale_ops); i++) | |
829 | pr_cont(" %s", scale_ops[i]->name); | |
653ed64b | 830 | pr_cont("\n"); |
653ed64b JFG |
831 | firsterr = -EINVAL; |
832 | cur_ops = NULL; | |
833 | goto unwind; | |
834 | } | |
835 | if (cur_ops->init) | |
836 | cur_ops->init(); | |
837 | ||
1fbeb3a8 | 838 | ref_scale_print_module_parms(cur_ops, "Start of test"); |
653ed64b JFG |
839 | |
840 | // Shutdown task | |
841 | if (shutdown) { | |
842 | init_waitqueue_head(&shutdown_wq); | |
1fbeb3a8 | 843 | firsterr = torture_create_kthread(ref_scale_shutdown, NULL, |
653ed64b | 844 | shutdown_task); |
ed60ad73 | 845 | if (torture_init_error(firsterr)) |
653ed64b JFG |
846 | goto unwind; |
847 | schedule_timeout_uninterruptible(1); | |
848 | } | |
849 | ||
8fc28783 PM |
850 | // Reader tasks (default to ~75% of online CPUs). |
851 | if (nreaders < 0) | |
852 | nreaders = (num_online_cpus() >> 1) + (num_online_cpus() >> 2); | |
0c6d18d8 PM |
853 | if (WARN_ONCE(loops <= 0, "%s: loops = %ld, adjusted to 1\n", __func__, loops)) |
854 | loops = 1; | |
855 | if (WARN_ONCE(nreaders <= 0, "%s: nreaders = %d, adjusted to 1\n", __func__, nreaders)) | |
856 | nreaders = 1; | |
857 | if (WARN_ONCE(nruns <= 0, "%s: nruns = %d, adjusted to 1\n", __func__, nruns)) | |
858 | nruns = 1; | |
653ed64b JFG |
859 | reader_tasks = kcalloc(nreaders, sizeof(reader_tasks[0]), |
860 | GFP_KERNEL); | |
861 | if (!reader_tasks) { | |
4feeb9d5 | 862 | SCALEOUT_ERRSTRING("out of memory"); |
653ed64b JFG |
863 | firsterr = -ENOMEM; |
864 | goto unwind; | |
865 | } | |
866 | ||
f71f22b6 | 867 | VERBOSE_SCALEOUT("Starting %d reader threads", nreaders); |
653ed64b JFG |
868 | |
869 | for (i = 0; i < nreaders; i++) { | |
1fbeb3a8 | 870 | firsterr = torture_create_kthread(ref_scale_reader, (void *)i, |
653ed64b | 871 | reader_tasks[i].task); |
ed60ad73 | 872 | if (torture_init_error(firsterr)) |
653ed64b JFG |
873 | goto unwind; |
874 | ||
875 | init_waitqueue_head(&(reader_tasks[i].wq)); | |
876 | } | |
877 | ||
878 | // Main Task | |
879 | init_waitqueue_head(&main_wq); | |
880 | firsterr = torture_create_kthread(main_func, NULL, main_task); | |
ed60ad73 | 881 | if (torture_init_error(firsterr)) |
653ed64b | 882 | goto unwind; |
653ed64b JFG |
883 | |
884 | torture_init_end(); | |
885 | return 0; | |
886 | ||
887 | unwind: | |
888 | torture_init_end(); | |
1fbeb3a8 | 889 | ref_scale_cleanup(); |
bc80d353 PM |
890 | if (shutdown) { |
891 | WARN_ON(!IS_MODULE(CONFIG_RCU_REF_SCALE_TEST)); | |
892 | kernel_power_off(); | |
893 | } | |
653ed64b JFG |
894 | return firsterr; |
895 | } | |
896 | ||
1fbeb3a8 PM |
897 | module_init(ref_scale_init); |
898 | module_exit(ref_scale_cleanup); |