2 * Read-Copy Update module-based torture test facility
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, you can access it online at
16 * http://www.gnu.org/licenses/gpl-2.0.html.
18 * Copyright (C) IBM Corporation, 2005, 2006
20 * Authors: Paul E. McKenney <paulmck@us.ibm.com>
21 * Josh Triplett <josh@joshtriplett.org>
23 * See also: Documentation/RCU/torture.txt
25 #include <linux/types.h>
26 #include <linux/kernel.h>
27 #include <linux/init.h>
28 #include <linux/module.h>
29 #include <linux/kthread.h>
30 #include <linux/err.h>
31 #include <linux/spinlock.h>
32 #include <linux/smp.h>
33 #include <linux/rcupdate.h>
34 #include <linux/interrupt.h>
35 #include <linux/sched.h>
36 #include <linux/atomic.h>
37 #include <linux/bitops.h>
38 #include <linux/completion.h>
39 #include <linux/moduleparam.h>
40 #include <linux/percpu.h>
41 #include <linux/notifier.h>
42 #include <linux/reboot.h>
43 #include <linux/freezer.h>
44 #include <linux/cpu.h>
45 #include <linux/delay.h>
46 #include <linux/stat.h>
47 #include <linux/srcu.h>
48 #include <linux/slab.h>
49 #include <linux/trace_clock.h>
50 #include <asm/byteorder.h>
51 #include <linux/torture.h>
52 #include <linux/vmalloc.h>
54 MODULE_LICENSE("GPL");
55 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>");
58 torture_param(int, cbflood_inter_holdoff
, HZ
,
59 "Holdoff between floods (jiffies)");
60 torture_param(int, cbflood_intra_holdoff
, 1,
61 "Holdoff between bursts (jiffies)");
62 torture_param(int, cbflood_n_burst
, 3, "# bursts in flood, zero to disable");
63 torture_param(int, cbflood_n_per_burst
, 20000,
64 "# callbacks per burst in flood");
65 torture_param(int, fqs_duration
, 0,
66 "Duration of fqs bursts (us), 0 to disable");
67 torture_param(int, fqs_holdoff
, 0, "Holdoff time within fqs bursts (us)");
68 torture_param(int, fqs_stutter
, 3, "Wait time between fqs bursts (s)");
69 torture_param(bool, gp_cond
, false, "Use conditional/async GP wait primitives");
70 torture_param(bool, gp_exp
, false, "Use expedited GP wait primitives");
71 torture_param(bool, gp_normal
, false,
72 "Use normal (non-expedited) GP wait primitives");
73 torture_param(bool, gp_sync
, false, "Use synchronous GP wait primitives");
74 torture_param(int, irqreader
, 1, "Allow RCU readers from irq handlers");
75 torture_param(int, n_barrier_cbs
, 0,
76 "# of callbacks/kthreads for barrier testing");
77 torture_param(int, nfakewriters
, 4, "Number of RCU fake writer threads");
78 torture_param(int, nreaders
, -1, "Number of RCU reader threads");
79 torture_param(int, object_debug
, 0,
80 "Enable debug-object double call_rcu() testing");
81 torture_param(int, onoff_holdoff
, 0, "Time after boot before CPU hotplugs (s)");
82 torture_param(int, onoff_interval
, 0,
83 "Time between CPU hotplugs (s), 0=disable");
84 torture_param(int, shuffle_interval
, 3, "Number of seconds between shuffles");
85 torture_param(int, shutdown_secs
, 0, "Shutdown time (s), <= zero to disable.");
86 torture_param(int, stall_cpu
, 0, "Stall duration (s), zero to disable.");
87 torture_param(int, stall_cpu_holdoff
, 10,
88 "Time to wait before starting stall (s).");
89 torture_param(int, stat_interval
, 60,
90 "Number of seconds between stats printk()s");
91 torture_param(int, stutter
, 5, "Number of seconds to run/halt test");
92 torture_param(int, test_boost
, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
93 torture_param(int, test_boost_duration
, 4,
94 "Duration of each boost test, seconds.");
95 torture_param(int, test_boost_interval
, 7,
96 "Interval between boost tests, seconds.");
97 torture_param(bool, test_no_idle_hz
, true,
98 "Test support for tickless idle CPUs");
99 torture_param(bool, verbose
, true,
100 "Enable verbose debugging printk()s");
102 static char *torture_type
= "rcu";
103 module_param(torture_type
, charp
, 0444);
104 MODULE_PARM_DESC(torture_type
, "Type of RCU to torture (rcu, rcu_bh, ...)");
106 static int nrealreaders
;
107 static int ncbflooders
;
108 static struct task_struct
*writer_task
;
109 static struct task_struct
**fakewriter_tasks
;
110 static struct task_struct
**reader_tasks
;
111 static struct task_struct
*stats_task
;
112 static struct task_struct
**cbflood_task
;
113 static struct task_struct
*fqs_task
;
114 static struct task_struct
*boost_tasks
[NR_CPUS
];
115 static struct task_struct
*stall_task
;
116 static struct task_struct
**barrier_cbs_tasks
;
117 static struct task_struct
*barrier_task
;
119 #define RCU_TORTURE_PIPE_LEN 10
122 struct rcu_head rtort_rcu
;
123 int rtort_pipe_count
;
124 struct list_head rtort_free
;
128 static LIST_HEAD(rcu_torture_freelist
);
129 static struct rcu_torture __rcu
*rcu_torture_current
;
130 static unsigned long rcu_torture_current_version
;
131 static struct rcu_torture rcu_tortures
[10 * RCU_TORTURE_PIPE_LEN
];
132 static DEFINE_SPINLOCK(rcu_torture_lock
);
133 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN
+ 1], rcu_torture_count
);
134 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN
+ 1], rcu_torture_batch
);
135 static atomic_t rcu_torture_wcount
[RCU_TORTURE_PIPE_LEN
+ 1];
136 static atomic_t n_rcu_torture_alloc
;
137 static atomic_t n_rcu_torture_alloc_fail
;
138 static atomic_t n_rcu_torture_free
;
139 static atomic_t n_rcu_torture_mberror
;
140 static atomic_t n_rcu_torture_error
;
141 static long n_rcu_torture_barrier_error
;
142 static long n_rcu_torture_boost_ktrerror
;
143 static long n_rcu_torture_boost_rterror
;
144 static long n_rcu_torture_boost_failure
;
145 static long n_rcu_torture_boosts
;
146 static long n_rcu_torture_timers
;
147 static long n_barrier_attempts
;
148 static long n_barrier_successes
;
149 static atomic_long_t n_cbfloods
;
150 static struct list_head rcu_torture_removed
;
152 static int rcu_torture_writer_state
;
153 #define RTWS_FIXED_DELAY 0
155 #define RTWS_REPLACE 2
156 #define RTWS_DEF_FREE 3
157 #define RTWS_EXP_SYNC 4
158 #define RTWS_COND_GET 5
159 #define RTWS_COND_SYNC 6
161 #define RTWS_STUTTER 8
162 #define RTWS_STOPPING 9
163 static const char * const rcu_torture_writer_state_names
[] = {
176 static const char *rcu_torture_writer_state_getname(void)
178 unsigned int i
= READ_ONCE(rcu_torture_writer_state
);
180 if (i
>= ARRAY_SIZE(rcu_torture_writer_state_names
))
182 return rcu_torture_writer_state_names
[i
];
185 static int torture_runnable
= IS_ENABLED(MODULE
);
186 module_param(torture_runnable
, int, 0444);
187 MODULE_PARM_DESC(torture_runnable
, "Start rcutorture at boot");
189 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU)
190 #define rcu_can_boost() 1
191 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
192 #define rcu_can_boost() 0
193 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
195 #ifdef CONFIG_RCU_TRACE
196 static u64 notrace
rcu_trace_clock_local(void)
198 u64 ts
= trace_clock_local();
199 unsigned long __maybe_unused ts_rem
= do_div(ts
, NSEC_PER_USEC
);
202 #else /* #ifdef CONFIG_RCU_TRACE */
203 static u64 notrace
rcu_trace_clock_local(void)
207 #endif /* #else #ifdef CONFIG_RCU_TRACE */
209 static unsigned long boost_starttime
; /* jiffies of next boost test start. */
210 static DEFINE_MUTEX(boost_mutex
); /* protect setting boost_starttime */
211 /* and boost task create/destroy. */
212 static atomic_t barrier_cbs_count
; /* Barrier callbacks registered. */
213 static bool barrier_phase
; /* Test phase. */
214 static atomic_t barrier_cbs_invoked
; /* Barrier callbacks invoked. */
215 static wait_queue_head_t
*barrier_cbs_wq
; /* Coordinate barrier testing. */
216 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq
);
219 * Allocate an element from the rcu_tortures pool.
221 static struct rcu_torture
*
222 rcu_torture_alloc(void)
226 spin_lock_bh(&rcu_torture_lock
);
227 if (list_empty(&rcu_torture_freelist
)) {
228 atomic_inc(&n_rcu_torture_alloc_fail
);
229 spin_unlock_bh(&rcu_torture_lock
);
232 atomic_inc(&n_rcu_torture_alloc
);
233 p
= rcu_torture_freelist
.next
;
235 spin_unlock_bh(&rcu_torture_lock
);
236 return container_of(p
, struct rcu_torture
, rtort_free
);
240 * Free an element to the rcu_tortures pool.
243 rcu_torture_free(struct rcu_torture
*p
)
245 atomic_inc(&n_rcu_torture_free
);
246 spin_lock_bh(&rcu_torture_lock
);
247 list_add_tail(&p
->rtort_free
, &rcu_torture_freelist
);
248 spin_unlock_bh(&rcu_torture_lock
);
252 * Operations vector for selecting different types of tests.
255 struct rcu_torture_ops
{
258 void (*cleanup
)(void);
259 int (*readlock
)(void);
260 void (*read_delay
)(struct torture_random_state
*rrsp
);
261 void (*readunlock
)(int idx
);
262 unsigned long (*started
)(void);
263 unsigned long (*completed
)(void);
264 void (*deferred_free
)(struct rcu_torture
*p
);
266 void (*exp_sync
)(void);
267 unsigned long (*get_state
)(void);
268 void (*cond_sync
)(unsigned long oldstate
);
269 call_rcu_func_t call
;
270 void (*cb_barrier
)(void);
278 static struct rcu_torture_ops
*cur_ops
;
281 * Definitions for rcu torture testing.
284 static int rcu_torture_read_lock(void) __acquires(RCU
)
290 static void rcu_read_delay(struct torture_random_state
*rrsp
)
292 unsigned long started
;
293 unsigned long completed
;
294 const unsigned long shortdelay_us
= 200;
295 const unsigned long longdelay_ms
= 50;
296 unsigned long long ts
;
298 /* We want a short delay sometimes to make a reader delay the grace
299 * period, and we want a long delay occasionally to trigger
300 * force_quiescent_state. */
302 if (!(torture_random(rrsp
) % (nrealreaders
* 2000 * longdelay_ms
))) {
303 started
= cur_ops
->completed();
304 ts
= rcu_trace_clock_local();
305 mdelay(longdelay_ms
);
306 completed
= cur_ops
->completed();
307 do_trace_rcu_torture_read(cur_ops
->name
, NULL
, ts
,
310 if (!(torture_random(rrsp
) % (nrealreaders
* 2 * shortdelay_us
)))
311 udelay(shortdelay_us
);
312 #ifdef CONFIG_PREEMPT
313 if (!preempt_count() &&
314 !(torture_random(rrsp
) % (nrealreaders
* 20000)))
315 preempt_schedule(); /* No QS if preempt_disable() in effect */
319 static void rcu_torture_read_unlock(int idx
) __releases(RCU
)
325 * Update callback in the pipe. This should be invoked after a grace period.
328 rcu_torture_pipe_update_one(struct rcu_torture
*rp
)
332 i
= rp
->rtort_pipe_count
;
333 if (i
> RCU_TORTURE_PIPE_LEN
)
334 i
= RCU_TORTURE_PIPE_LEN
;
335 atomic_inc(&rcu_torture_wcount
[i
]);
336 if (++rp
->rtort_pipe_count
>= RCU_TORTURE_PIPE_LEN
) {
337 rp
->rtort_mbtest
= 0;
344 * Update all callbacks in the pipe. Suitable for synchronous grace-period
348 rcu_torture_pipe_update(struct rcu_torture
*old_rp
)
350 struct rcu_torture
*rp
;
351 struct rcu_torture
*rp1
;
354 list_add(&old_rp
->rtort_free
, &rcu_torture_removed
);
355 list_for_each_entry_safe(rp
, rp1
, &rcu_torture_removed
, rtort_free
) {
356 if (rcu_torture_pipe_update_one(rp
)) {
357 list_del(&rp
->rtort_free
);
358 rcu_torture_free(rp
);
364 rcu_torture_cb(struct rcu_head
*p
)
366 struct rcu_torture
*rp
= container_of(p
, struct rcu_torture
, rtort_rcu
);
368 if (torture_must_stop_irq()) {
369 /* Test is ending, just drop callbacks on the floor. */
370 /* The next initialization will pick up the pieces. */
373 if (rcu_torture_pipe_update_one(rp
))
374 rcu_torture_free(rp
);
376 cur_ops
->deferred_free(rp
);
379 static unsigned long rcu_no_completed(void)
384 static void rcu_torture_deferred_free(struct rcu_torture
*p
)
386 call_rcu(&p
->rtort_rcu
, rcu_torture_cb
);
389 static void rcu_sync_torture_init(void)
391 INIT_LIST_HEAD(&rcu_torture_removed
);
394 static struct rcu_torture_ops rcu_ops
= {
396 .init
= rcu_sync_torture_init
,
397 .readlock
= rcu_torture_read_lock
,
398 .read_delay
= rcu_read_delay
,
399 .readunlock
= rcu_torture_read_unlock
,
400 .started
= rcu_batches_started
,
401 .completed
= rcu_batches_completed
,
402 .deferred_free
= rcu_torture_deferred_free
,
403 .sync
= synchronize_rcu
,
404 .exp_sync
= synchronize_rcu_expedited
,
405 .get_state
= get_state_synchronize_rcu
,
406 .cond_sync
= cond_synchronize_rcu
,
408 .cb_barrier
= rcu_barrier
,
409 .fqs
= rcu_force_quiescent_state
,
412 .can_boost
= rcu_can_boost(),
417 * Definitions for rcu_bh torture testing.
420 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH
)
426 static void rcu_bh_torture_read_unlock(int idx
) __releases(RCU_BH
)
428 rcu_read_unlock_bh();
431 static void rcu_bh_torture_deferred_free(struct rcu_torture
*p
)
433 call_rcu_bh(&p
->rtort_rcu
, rcu_torture_cb
);
436 static struct rcu_torture_ops rcu_bh_ops
= {
437 .ttype
= RCU_BH_FLAVOR
,
438 .init
= rcu_sync_torture_init
,
439 .readlock
= rcu_bh_torture_read_lock
,
440 .read_delay
= rcu_read_delay
, /* just reuse rcu's version. */
441 .readunlock
= rcu_bh_torture_read_unlock
,
442 .started
= rcu_batches_started_bh
,
443 .completed
= rcu_batches_completed_bh
,
444 .deferred_free
= rcu_bh_torture_deferred_free
,
445 .sync
= synchronize_rcu_bh
,
446 .exp_sync
= synchronize_rcu_bh_expedited
,
448 .cb_barrier
= rcu_barrier_bh
,
449 .fqs
= rcu_bh_force_quiescent_state
,
456 * Don't even think about trying any of these in real life!!!
457 * The names includes "busted", and they really means it!
458 * The only purpose of these functions is to provide a buggy RCU
459 * implementation to make sure that rcutorture correctly emits
460 * buggy-RCU error messages.
462 static void rcu_busted_torture_deferred_free(struct rcu_torture
*p
)
464 /* This is a deliberate bug for testing purposes only! */
465 rcu_torture_cb(&p
->rtort_rcu
);
468 static void synchronize_rcu_busted(void)
470 /* This is a deliberate bug for testing purposes only! */
474 call_rcu_busted(struct rcu_head
*head
, rcu_callback_t func
)
476 /* This is a deliberate bug for testing purposes only! */
480 static struct rcu_torture_ops rcu_busted_ops
= {
481 .ttype
= INVALID_RCU_FLAVOR
,
482 .init
= rcu_sync_torture_init
,
483 .readlock
= rcu_torture_read_lock
,
484 .read_delay
= rcu_read_delay
, /* just reuse rcu's version. */
485 .readunlock
= rcu_torture_read_unlock
,
486 .started
= rcu_no_completed
,
487 .completed
= rcu_no_completed
,
488 .deferred_free
= rcu_busted_torture_deferred_free
,
489 .sync
= synchronize_rcu_busted
,
490 .exp_sync
= synchronize_rcu_busted
,
491 .call
= call_rcu_busted
,
500 * Definitions for srcu torture testing.
503 DEFINE_STATIC_SRCU(srcu_ctl
);
504 static struct srcu_struct srcu_ctld
;
505 static struct srcu_struct
*srcu_ctlp
= &srcu_ctl
;
507 static int srcu_torture_read_lock(void) __acquires(srcu_ctlp
)
509 return srcu_read_lock(srcu_ctlp
);
512 static void srcu_read_delay(struct torture_random_state
*rrsp
)
515 const long uspertick
= 1000000 / HZ
;
516 const long longdelay
= 10;
518 /* We want there to be long-running readers, but not all the time. */
520 delay
= torture_random(rrsp
) %
521 (nrealreaders
* 2 * longdelay
* uspertick
);
523 schedule_timeout_interruptible(longdelay
);
525 rcu_read_delay(rrsp
);
528 static void srcu_torture_read_unlock(int idx
) __releases(srcu_ctlp
)
530 srcu_read_unlock(srcu_ctlp
, idx
);
533 static unsigned long srcu_torture_completed(void)
535 return srcu_batches_completed(srcu_ctlp
);
538 static void srcu_torture_deferred_free(struct rcu_torture
*rp
)
540 call_srcu(srcu_ctlp
, &rp
->rtort_rcu
, rcu_torture_cb
);
543 static void srcu_torture_synchronize(void)
545 synchronize_srcu(srcu_ctlp
);
548 static void srcu_torture_call(struct rcu_head
*head
,
551 call_srcu(srcu_ctlp
, head
, func
);
554 static void srcu_torture_barrier(void)
556 srcu_barrier(srcu_ctlp
);
559 static void srcu_torture_stats(void)
562 int idx
= srcu_ctlp
->completed
& 0x1;
564 pr_alert("%s%s per-CPU(idx=%d):",
565 torture_type
, TORTURE_FLAG
, idx
);
566 for_each_possible_cpu(cpu
) {
567 unsigned long l0
, l1
;
568 unsigned long u0
, u1
;
570 struct srcu_array
*counts
= per_cpu_ptr(srcu_ctlp
->per_cpu_ref
, cpu
);
572 u0
= counts
->unlock_count
[!idx
];
573 u1
= counts
->unlock_count
[idx
];
576 * Make sure that a lock is always counted if the corresponding
581 l0
= counts
->lock_count
[!idx
];
582 l1
= counts
->lock_count
[idx
];
586 pr_cont(" %d(%ld,%ld)", cpu
, c0
, c1
);
591 static void srcu_torture_synchronize_expedited(void)
593 synchronize_srcu_expedited(srcu_ctlp
);
596 static struct rcu_torture_ops srcu_ops
= {
597 .ttype
= SRCU_FLAVOR
,
598 .init
= rcu_sync_torture_init
,
599 .readlock
= srcu_torture_read_lock
,
600 .read_delay
= srcu_read_delay
,
601 .readunlock
= srcu_torture_read_unlock
,
603 .completed
= srcu_torture_completed
,
604 .deferred_free
= srcu_torture_deferred_free
,
605 .sync
= srcu_torture_synchronize
,
606 .exp_sync
= srcu_torture_synchronize_expedited
,
607 .call
= srcu_torture_call
,
608 .cb_barrier
= srcu_torture_barrier
,
609 .stats
= srcu_torture_stats
,
613 static void srcu_torture_init(void)
615 rcu_sync_torture_init();
616 WARN_ON(init_srcu_struct(&srcu_ctld
));
617 srcu_ctlp
= &srcu_ctld
;
620 static void srcu_torture_cleanup(void)
622 cleanup_srcu_struct(&srcu_ctld
);
623 srcu_ctlp
= &srcu_ctl
; /* In case of a later rcutorture run. */
626 /* As above, but dynamically allocated. */
627 static struct rcu_torture_ops srcud_ops
= {
628 .ttype
= SRCU_FLAVOR
,
629 .init
= srcu_torture_init
,
630 .cleanup
= srcu_torture_cleanup
,
631 .readlock
= srcu_torture_read_lock
,
632 .read_delay
= srcu_read_delay
,
633 .readunlock
= srcu_torture_read_unlock
,
635 .completed
= srcu_torture_completed
,
636 .deferred_free
= srcu_torture_deferred_free
,
637 .sync
= srcu_torture_synchronize
,
638 .exp_sync
= srcu_torture_synchronize_expedited
,
639 .call
= srcu_torture_call
,
640 .cb_barrier
= srcu_torture_barrier
,
641 .stats
= srcu_torture_stats
,
646 * Definitions for sched torture testing.
649 static int sched_torture_read_lock(void)
655 static void sched_torture_read_unlock(int idx
)
660 static void rcu_sched_torture_deferred_free(struct rcu_torture
*p
)
662 call_rcu_sched(&p
->rtort_rcu
, rcu_torture_cb
);
665 static struct rcu_torture_ops sched_ops
= {
666 .ttype
= RCU_SCHED_FLAVOR
,
667 .init
= rcu_sync_torture_init
,
668 .readlock
= sched_torture_read_lock
,
669 .read_delay
= rcu_read_delay
, /* just reuse rcu's version. */
670 .readunlock
= sched_torture_read_unlock
,
671 .started
= rcu_batches_started_sched
,
672 .completed
= rcu_batches_completed_sched
,
673 .deferred_free
= rcu_sched_torture_deferred_free
,
674 .sync
= synchronize_sched
,
675 .exp_sync
= synchronize_sched_expedited
,
676 .get_state
= get_state_synchronize_sched
,
677 .cond_sync
= cond_synchronize_sched
,
678 .call
= call_rcu_sched
,
679 .cb_barrier
= rcu_barrier_sched
,
680 .fqs
= rcu_sched_force_quiescent_state
,
686 #ifdef CONFIG_TASKS_RCU
689 * Definitions for RCU-tasks torture testing.
692 static int tasks_torture_read_lock(void)
697 static void tasks_torture_read_unlock(int idx
)
701 static void rcu_tasks_torture_deferred_free(struct rcu_torture
*p
)
703 call_rcu_tasks(&p
->rtort_rcu
, rcu_torture_cb
);
706 static struct rcu_torture_ops tasks_ops
= {
707 .ttype
= RCU_TASKS_FLAVOR
,
708 .init
= rcu_sync_torture_init
,
709 .readlock
= tasks_torture_read_lock
,
710 .read_delay
= rcu_read_delay
, /* just reuse rcu's version. */
711 .readunlock
= tasks_torture_read_unlock
,
712 .started
= rcu_no_completed
,
713 .completed
= rcu_no_completed
,
714 .deferred_free
= rcu_tasks_torture_deferred_free
,
715 .sync
= synchronize_rcu_tasks
,
716 .exp_sync
= synchronize_rcu_tasks
,
717 .call
= call_rcu_tasks
,
718 .cb_barrier
= rcu_barrier_tasks
,
725 #define RCUTORTURE_TASKS_OPS &tasks_ops,
727 static bool __maybe_unused
torturing_tasks(void)
729 return cur_ops
== &tasks_ops
;
732 #else /* #ifdef CONFIG_TASKS_RCU */
734 #define RCUTORTURE_TASKS_OPS
736 static bool __maybe_unused
torturing_tasks(void)
741 #endif /* #else #ifdef CONFIG_TASKS_RCU */
744 * RCU torture priority-boost testing. Runs one real-time thread per
745 * CPU for moderate bursts, repeatedly registering RCU callbacks and
746 * spinning waiting for them to be invoked. If a given callback takes
747 * too long to be invoked, we assume that priority inversion has occurred.
750 struct rcu_boost_inflight
{
755 static void rcu_torture_boost_cb(struct rcu_head
*head
)
757 struct rcu_boost_inflight
*rbip
=
758 container_of(head
, struct rcu_boost_inflight
, rcu
);
760 /* Ensure RCU-core accesses precede clearing ->inflight */
761 smp_store_release(&rbip
->inflight
, 0);
764 static int rcu_torture_boost(void *arg
)
766 unsigned long call_rcu_time
;
767 unsigned long endtime
;
768 unsigned long oldstarttime
;
769 struct rcu_boost_inflight rbi
= { .inflight
= 0 };
770 struct sched_param sp
;
772 VERBOSE_TOROUT_STRING("rcu_torture_boost started");
774 /* Set real-time priority. */
775 sp
.sched_priority
= 1;
776 if (sched_setscheduler(current
, SCHED_FIFO
, &sp
) < 0) {
777 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!");
778 n_rcu_torture_boost_rterror
++;
781 init_rcu_head_on_stack(&rbi
.rcu
);
782 /* Each pass through the following loop does one boost-test cycle. */
784 /* Wait for the next test interval. */
785 oldstarttime
= boost_starttime
;
786 while (ULONG_CMP_LT(jiffies
, oldstarttime
)) {
787 schedule_timeout_interruptible(oldstarttime
- jiffies
);
788 stutter_wait("rcu_torture_boost");
789 if (torture_must_stop())
793 /* Do one boost-test interval. */
794 endtime
= oldstarttime
+ test_boost_duration
* HZ
;
795 call_rcu_time
= jiffies
;
796 while (ULONG_CMP_LT(jiffies
, endtime
)) {
797 /* If we don't have a callback in flight, post one. */
798 if (!smp_load_acquire(&rbi
.inflight
)) {
799 /* RCU core before ->inflight = 1. */
800 smp_store_release(&rbi
.inflight
, 1);
801 call_rcu(&rbi
.rcu
, rcu_torture_boost_cb
);
802 if (jiffies
- call_rcu_time
>
803 test_boost_duration
* HZ
- HZ
/ 2) {
804 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed");
805 n_rcu_torture_boost_failure
++;
807 call_rcu_time
= jiffies
;
809 stutter_wait("rcu_torture_boost");
810 if (torture_must_stop())
815 * Set the start time of the next test interval.
816 * Yes, this is vulnerable to long delays, but such
817 * delays simply cause a false negative for the next
818 * interval. Besides, we are running at RT priority,
819 * so delays should be relatively rare.
821 while (oldstarttime
== boost_starttime
&&
822 !kthread_should_stop()) {
823 if (mutex_trylock(&boost_mutex
)) {
824 boost_starttime
= jiffies
+
825 test_boost_interval
* HZ
;
826 n_rcu_torture_boosts
++;
827 mutex_unlock(&boost_mutex
);
830 schedule_timeout_uninterruptible(1);
833 /* Go do the stutter. */
834 checkwait
: stutter_wait("rcu_torture_boost");
835 } while (!torture_must_stop());
837 /* Clean up and exit. */
838 while (!kthread_should_stop() || smp_load_acquire(&rbi
.inflight
)) {
839 torture_shutdown_absorb("rcu_torture_boost");
840 schedule_timeout_uninterruptible(1);
842 destroy_rcu_head_on_stack(&rbi
.rcu
);
843 torture_kthread_stopping("rcu_torture_boost");
847 static void rcu_torture_cbflood_cb(struct rcu_head
*rhp
)
852 * RCU torture callback-flood kthread. Repeatedly induces bursts of calls
853 * to call_rcu() or analogous, increasing the probability of occurrence
854 * of callback-overflow corner cases.
857 rcu_torture_cbflood(void *arg
)
862 struct rcu_head
*rhp
;
864 if (cbflood_n_per_burst
> 0 &&
865 cbflood_inter_holdoff
> 0 &&
866 cbflood_intra_holdoff
> 0 &&
868 cur_ops
->cb_barrier
) {
869 rhp
= vmalloc(sizeof(*rhp
) *
870 cbflood_n_burst
* cbflood_n_per_burst
);
874 VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM");
877 VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started");
879 schedule_timeout_interruptible(cbflood_inter_holdoff
);
880 atomic_long_inc(&n_cbfloods
);
881 WARN_ON(signal_pending(current
));
882 for (i
= 0; i
< cbflood_n_burst
; i
++) {
883 for (j
= 0; j
< cbflood_n_per_burst
; j
++) {
884 cur_ops
->call(&rhp
[i
* cbflood_n_per_burst
+ j
],
885 rcu_torture_cbflood_cb
);
887 schedule_timeout_interruptible(cbflood_intra_holdoff
);
888 WARN_ON(signal_pending(current
));
890 cur_ops
->cb_barrier();
891 stutter_wait("rcu_torture_cbflood");
892 } while (!torture_must_stop());
895 torture_kthread_stopping("rcu_torture_cbflood");
900 * RCU torture force-quiescent-state kthread. Repeatedly induces
901 * bursts of calls to force_quiescent_state(), increasing the probability
902 * of occurrence of some important types of race conditions.
905 rcu_torture_fqs(void *arg
)
907 unsigned long fqs_resume_time
;
908 int fqs_burst_remaining
;
910 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started");
912 fqs_resume_time
= jiffies
+ fqs_stutter
* HZ
;
913 while (ULONG_CMP_LT(jiffies
, fqs_resume_time
) &&
914 !kthread_should_stop()) {
915 schedule_timeout_interruptible(1);
917 fqs_burst_remaining
= fqs_duration
;
918 while (fqs_burst_remaining
> 0 &&
919 !kthread_should_stop()) {
922 fqs_burst_remaining
-= fqs_holdoff
;
924 stutter_wait("rcu_torture_fqs");
925 } while (!torture_must_stop());
926 torture_kthread_stopping("rcu_torture_fqs");
931 * RCU torture writer kthread. Repeatedly substitutes a new structure
932 * for that pointed to by rcu_torture_current, freeing the old structure
933 * after a series of grace periods (the "pipeline").
936 rcu_torture_writer(void *arg
)
938 bool can_expedite
= !rcu_gp_is_expedited() && !rcu_gp_is_normal();
940 unsigned long gp_snap
;
941 bool gp_cond1
= gp_cond
, gp_exp1
= gp_exp
, gp_normal1
= gp_normal
;
942 bool gp_sync1
= gp_sync
;
944 struct rcu_torture
*rp
;
945 struct rcu_torture
*old_rp
;
946 static DEFINE_TORTURE_RANDOM(rand
);
947 int synctype
[] = { RTWS_DEF_FREE
, RTWS_EXP_SYNC
,
948 RTWS_COND_GET
, RTWS_SYNC
};
951 VERBOSE_TOROUT_STRING("rcu_torture_writer task started");
953 pr_alert("%s" TORTURE_FLAG
954 " GP expediting controlled from boot/sysfs for %s,\n",
955 torture_type
, cur_ops
->name
);
956 pr_alert("%s" TORTURE_FLAG
957 " Disabled dynamic grace-period expediting.\n",
961 /* Initialize synctype[] array. If none set, take default. */
962 if (!gp_cond1
&& !gp_exp1
&& !gp_normal1
&& !gp_sync1
)
963 gp_cond1
= gp_exp1
= gp_normal1
= gp_sync1
= true;
964 if (gp_cond1
&& cur_ops
->get_state
&& cur_ops
->cond_sync
)
965 synctype
[nsynctypes
++] = RTWS_COND_GET
;
966 else if (gp_cond
&& (!cur_ops
->get_state
|| !cur_ops
->cond_sync
))
967 pr_alert("rcu_torture_writer: gp_cond without primitives.\n");
968 if (gp_exp1
&& cur_ops
->exp_sync
)
969 synctype
[nsynctypes
++] = RTWS_EXP_SYNC
;
970 else if (gp_exp
&& !cur_ops
->exp_sync
)
971 pr_alert("rcu_torture_writer: gp_exp without primitives.\n");
972 if (gp_normal1
&& cur_ops
->deferred_free
)
973 synctype
[nsynctypes
++] = RTWS_DEF_FREE
;
974 else if (gp_normal
&& !cur_ops
->deferred_free
)
975 pr_alert("rcu_torture_writer: gp_normal without primitives.\n");
976 if (gp_sync1
&& cur_ops
->sync
)
977 synctype
[nsynctypes
++] = RTWS_SYNC
;
978 else if (gp_sync
&& !cur_ops
->sync
)
979 pr_alert("rcu_torture_writer: gp_sync without primitives.\n");
980 if (WARN_ONCE(nsynctypes
== 0,
981 "rcu_torture_writer: No update-side primitives.\n")) {
983 * No updates primitives, so don't try updating.
984 * The resulting test won't be testing much, hence the
987 rcu_torture_writer_state
= RTWS_STOPPING
;
988 torture_kthread_stopping("rcu_torture_writer");
992 rcu_torture_writer_state
= RTWS_FIXED_DELAY
;
993 schedule_timeout_uninterruptible(1);
994 rp
= rcu_torture_alloc();
997 rp
->rtort_pipe_count
= 0;
998 rcu_torture_writer_state
= RTWS_DELAY
;
999 udelay(torture_random(&rand
) & 0x3ff);
1000 rcu_torture_writer_state
= RTWS_REPLACE
;
1001 old_rp
= rcu_dereference_check(rcu_torture_current
,
1002 current
== writer_task
);
1003 rp
->rtort_mbtest
= 1;
1004 rcu_assign_pointer(rcu_torture_current
, rp
);
1005 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
1007 i
= old_rp
->rtort_pipe_count
;
1008 if (i
> RCU_TORTURE_PIPE_LEN
)
1009 i
= RCU_TORTURE_PIPE_LEN
;
1010 atomic_inc(&rcu_torture_wcount
[i
]);
1011 old_rp
->rtort_pipe_count
++;
1012 switch (synctype
[torture_random(&rand
) % nsynctypes
]) {
1014 rcu_torture_writer_state
= RTWS_DEF_FREE
;
1015 cur_ops
->deferred_free(old_rp
);
1018 rcu_torture_writer_state
= RTWS_EXP_SYNC
;
1019 cur_ops
->exp_sync();
1020 rcu_torture_pipe_update(old_rp
);
1023 rcu_torture_writer_state
= RTWS_COND_GET
;
1024 gp_snap
= cur_ops
->get_state();
1025 i
= torture_random(&rand
) % 16;
1027 schedule_timeout_interruptible(i
);
1028 udelay(torture_random(&rand
) % 1000);
1029 rcu_torture_writer_state
= RTWS_COND_SYNC
;
1030 cur_ops
->cond_sync(gp_snap
);
1031 rcu_torture_pipe_update(old_rp
);
1034 rcu_torture_writer_state
= RTWS_SYNC
;
1036 rcu_torture_pipe_update(old_rp
);
1043 rcutorture_record_progress(++rcu_torture_current_version
);
1044 /* Cycle through nesting levels of rcu_expedite_gp() calls. */
1046 !(torture_random(&rand
) & 0xff & (!!expediting
- 1))) {
1047 WARN_ON_ONCE(expediting
== 0 && rcu_gp_is_expedited());
1048 if (expediting
>= 0)
1051 rcu_unexpedite_gp();
1052 if (++expediting
> 3)
1053 expediting
= -expediting
;
1055 rcu_torture_writer_state
= RTWS_STUTTER
;
1056 stutter_wait("rcu_torture_writer");
1057 } while (!torture_must_stop());
1058 /* Reset expediting back to unexpedited. */
1060 expediting
= -expediting
;
1061 while (can_expedite
&& expediting
++ < 0)
1062 rcu_unexpedite_gp();
1063 WARN_ON_ONCE(can_expedite
&& rcu_gp_is_expedited());
1064 rcu_torture_writer_state
= RTWS_STOPPING
;
1065 torture_kthread_stopping("rcu_torture_writer");
1070 * RCU torture fake writer kthread. Repeatedly calls sync, with a random
1071 * delay between calls.
1074 rcu_torture_fakewriter(void *arg
)
1076 DEFINE_TORTURE_RANDOM(rand
);
1078 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started");
1079 set_user_nice(current
, MAX_NICE
);
1082 schedule_timeout_uninterruptible(1 + torture_random(&rand
)%10);
1083 udelay(torture_random(&rand
) & 0x3ff);
1084 if (cur_ops
->cb_barrier
!= NULL
&&
1085 torture_random(&rand
) % (nfakewriters
* 8) == 0) {
1086 cur_ops
->cb_barrier();
1087 } else if (gp_normal
== gp_exp
) {
1088 if (torture_random(&rand
) & 0x80)
1091 cur_ops
->exp_sync();
1092 } else if (gp_normal
) {
1095 cur_ops
->exp_sync();
1097 stutter_wait("rcu_torture_fakewriter");
1098 } while (!torture_must_stop());
1100 torture_kthread_stopping("rcu_torture_fakewriter");
1105 * RCU torture reader from timer handler. Dereferences rcu_torture_current,
1106 * incrementing the corresponding element of the pipeline array. The
1107 * counter in the element should never be greater than 1, otherwise, the
1108 * RCU implementation is broken.
1110 static void rcu_torture_timer(unsigned long unused
)
1113 unsigned long started
;
1114 unsigned long completed
;
1115 static DEFINE_TORTURE_RANDOM(rand
);
1116 static DEFINE_SPINLOCK(rand_lock
);
1117 struct rcu_torture
*p
;
1119 unsigned long long ts
;
1121 idx
= cur_ops
->readlock();
1122 if (cur_ops
->started
)
1123 started
= cur_ops
->started();
1125 started
= cur_ops
->completed();
1126 ts
= rcu_trace_clock_local();
1127 p
= rcu_dereference_check(rcu_torture_current
,
1128 rcu_read_lock_bh_held() ||
1129 rcu_read_lock_sched_held() ||
1130 srcu_read_lock_held(srcu_ctlp
) ||
1133 /* Leave because rcu_torture_writer is not yet underway */
1134 cur_ops
->readunlock(idx
);
1137 if (p
->rtort_mbtest
== 0)
1138 atomic_inc(&n_rcu_torture_mberror
);
1139 spin_lock(&rand_lock
);
1140 cur_ops
->read_delay(&rand
);
1141 n_rcu_torture_timers
++;
1142 spin_unlock(&rand_lock
);
1144 pipe_count
= p
->rtort_pipe_count
;
1145 if (pipe_count
> RCU_TORTURE_PIPE_LEN
) {
1146 /* Should not happen, but... */
1147 pipe_count
= RCU_TORTURE_PIPE_LEN
;
1149 completed
= cur_ops
->completed();
1150 if (pipe_count
> 1) {
1151 do_trace_rcu_torture_read(cur_ops
->name
, &p
->rtort_rcu
, ts
,
1152 started
, completed
);
1153 rcu_ftrace_dump(DUMP_ALL
);
1155 __this_cpu_inc(rcu_torture_count
[pipe_count
]);
1156 completed
= completed
- started
;
1157 if (cur_ops
->started
)
1159 if (completed
> RCU_TORTURE_PIPE_LEN
) {
1160 /* Should not happen, but... */
1161 completed
= RCU_TORTURE_PIPE_LEN
;
1163 __this_cpu_inc(rcu_torture_batch
[completed
]);
1165 cur_ops
->readunlock(idx
);
1169 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current,
1170 * incrementing the corresponding element of the pipeline array. The
1171 * counter in the element should never be greater than 1, otherwise, the
1172 * RCU implementation is broken.
1175 rcu_torture_reader(void *arg
)
1177 unsigned long started
;
1178 unsigned long completed
;
1180 DEFINE_TORTURE_RANDOM(rand
);
1181 struct rcu_torture
*p
;
1183 struct timer_list t
;
1184 unsigned long long ts
;
1186 VERBOSE_TOROUT_STRING("rcu_torture_reader task started");
1187 set_user_nice(current
, MAX_NICE
);
1188 if (irqreader
&& cur_ops
->irq_capable
)
1189 setup_timer_on_stack(&t
, rcu_torture_timer
, 0);
1192 if (irqreader
&& cur_ops
->irq_capable
) {
1193 if (!timer_pending(&t
))
1194 mod_timer(&t
, jiffies
+ 1);
1196 idx
= cur_ops
->readlock();
1197 if (cur_ops
->started
)
1198 started
= cur_ops
->started();
1200 started
= cur_ops
->completed();
1201 ts
= rcu_trace_clock_local();
1202 p
= rcu_dereference_check(rcu_torture_current
,
1203 rcu_read_lock_bh_held() ||
1204 rcu_read_lock_sched_held() ||
1205 srcu_read_lock_held(srcu_ctlp
) ||
1208 /* Wait for rcu_torture_writer to get underway */
1209 cur_ops
->readunlock(idx
);
1210 schedule_timeout_interruptible(HZ
);
1213 if (p
->rtort_mbtest
== 0)
1214 atomic_inc(&n_rcu_torture_mberror
);
1215 cur_ops
->read_delay(&rand
);
1217 pipe_count
= p
->rtort_pipe_count
;
1218 if (pipe_count
> RCU_TORTURE_PIPE_LEN
) {
1219 /* Should not happen, but... */
1220 pipe_count
= RCU_TORTURE_PIPE_LEN
;
1222 completed
= cur_ops
->completed();
1223 if (pipe_count
> 1) {
1224 do_trace_rcu_torture_read(cur_ops
->name
, &p
->rtort_rcu
,
1225 ts
, started
, completed
);
1226 rcu_ftrace_dump(DUMP_ALL
);
1228 __this_cpu_inc(rcu_torture_count
[pipe_count
]);
1229 completed
= completed
- started
;
1230 if (cur_ops
->started
)
1232 if (completed
> RCU_TORTURE_PIPE_LEN
) {
1233 /* Should not happen, but... */
1234 completed
= RCU_TORTURE_PIPE_LEN
;
1236 __this_cpu_inc(rcu_torture_batch
[completed
]);
1238 cur_ops
->readunlock(idx
);
1239 stutter_wait("rcu_torture_reader");
1240 } while (!torture_must_stop());
1241 if (irqreader
&& cur_ops
->irq_capable
) {
1243 destroy_timer_on_stack(&t
);
1245 torture_kthread_stopping("rcu_torture_reader");
1250 * Print torture statistics. Caller must ensure that there is only
1251 * one call to this function at a given time!!! This is normally
1252 * accomplished by relying on the module system to only have one copy
1253 * of the module loaded, and then by giving the rcu_torture_stats
1254 * kthread full control (or the init/cleanup functions when rcu_torture_stats
1255 * thread is not running).
1258 rcu_torture_stats_print(void)
1262 long pipesummary
[RCU_TORTURE_PIPE_LEN
+ 1] = { 0 };
1263 long batchsummary
[RCU_TORTURE_PIPE_LEN
+ 1] = { 0 };
1264 static unsigned long rtcv_snap
= ULONG_MAX
;
1265 struct task_struct
*wtp
;
1267 for_each_possible_cpu(cpu
) {
1268 for (i
= 0; i
< RCU_TORTURE_PIPE_LEN
+ 1; i
++) {
1269 pipesummary
[i
] += per_cpu(rcu_torture_count
, cpu
)[i
];
1270 batchsummary
[i
] += per_cpu(rcu_torture_batch
, cpu
)[i
];
1273 for (i
= RCU_TORTURE_PIPE_LEN
- 1; i
>= 0; i
--) {
1274 if (pipesummary
[i
] != 0)
1278 pr_alert("%s%s ", torture_type
, TORTURE_FLAG
);
1279 pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",
1280 rcu_torture_current
,
1281 rcu_torture_current_version
,
1282 list_empty(&rcu_torture_freelist
),
1283 atomic_read(&n_rcu_torture_alloc
),
1284 atomic_read(&n_rcu_torture_alloc_fail
),
1285 atomic_read(&n_rcu_torture_free
));
1286 pr_cont("rtmbe: %d rtbe: %ld rtbke: %ld rtbre: %ld ",
1287 atomic_read(&n_rcu_torture_mberror
),
1288 n_rcu_torture_barrier_error
,
1289 n_rcu_torture_boost_ktrerror
,
1290 n_rcu_torture_boost_rterror
);
1291 pr_cont("rtbf: %ld rtb: %ld nt: %ld ",
1292 n_rcu_torture_boost_failure
,
1293 n_rcu_torture_boosts
,
1294 n_rcu_torture_timers
);
1295 torture_onoff_stats();
1296 pr_cont("barrier: %ld/%ld:%ld ",
1297 n_barrier_successes
,
1299 n_rcu_torture_barrier_error
);
1300 pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods
));
1302 pr_alert("%s%s ", torture_type
, TORTURE_FLAG
);
1303 if (atomic_read(&n_rcu_torture_mberror
) != 0 ||
1304 n_rcu_torture_barrier_error
!= 0 ||
1305 n_rcu_torture_boost_ktrerror
!= 0 ||
1306 n_rcu_torture_boost_rterror
!= 0 ||
1307 n_rcu_torture_boost_failure
!= 0 ||
1309 pr_cont("%s", "!!! ");
1310 atomic_inc(&n_rcu_torture_error
);
1313 pr_cont("Reader Pipe: ");
1314 for (i
= 0; i
< RCU_TORTURE_PIPE_LEN
+ 1; i
++)
1315 pr_cont(" %ld", pipesummary
[i
]);
1318 pr_alert("%s%s ", torture_type
, TORTURE_FLAG
);
1319 pr_cont("Reader Batch: ");
1320 for (i
= 0; i
< RCU_TORTURE_PIPE_LEN
+ 1; i
++)
1321 pr_cont(" %ld", batchsummary
[i
]);
1324 pr_alert("%s%s ", torture_type
, TORTURE_FLAG
);
1325 pr_cont("Free-Block Circulation: ");
1326 for (i
= 0; i
< RCU_TORTURE_PIPE_LEN
+ 1; i
++) {
1327 pr_cont(" %d", atomic_read(&rcu_torture_wcount
[i
]));
1333 if (rtcv_snap
== rcu_torture_current_version
&&
1334 rcu_torture_current
!= NULL
) {
1335 int __maybe_unused flags
;
1336 unsigned long __maybe_unused gpnum
;
1337 unsigned long __maybe_unused completed
;
1339 rcutorture_get_gp_data(cur_ops
->ttype
,
1340 &flags
, &gpnum
, &completed
);
1341 wtp
= READ_ONCE(writer_task
);
1342 pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x ->state %#lx\n",
1343 rcu_torture_writer_state_getname(),
1344 rcu_torture_writer_state
,
1345 gpnum
, completed
, flags
,
1346 wtp
== NULL
? ~0UL : wtp
->state
);
1347 show_rcu_gp_kthreads();
1348 rcu_ftrace_dump(DUMP_ALL
);
1350 rtcv_snap
= rcu_torture_current_version
;
1354 * Periodically prints torture statistics, if periodic statistics printing
1355 * was specified via the stat_interval module parameter.
1358 rcu_torture_stats(void *arg
)
1360 VERBOSE_TOROUT_STRING("rcu_torture_stats task started");
1362 schedule_timeout_interruptible(stat_interval
* HZ
);
1363 rcu_torture_stats_print();
1364 torture_shutdown_absorb("rcu_torture_stats");
1365 } while (!torture_must_stop());
1366 torture_kthread_stopping("rcu_torture_stats");
1371 rcu_torture_print_module_parms(struct rcu_torture_ops
*cur_ops
, const char *tag
)
1373 pr_alert("%s" TORTURE_FLAG
1374 "--- %s: nreaders=%d nfakewriters=%d "
1375 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
1376 "shuffle_interval=%d stutter=%d irqreader=%d "
1377 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
1378 "test_boost=%d/%d test_boost_interval=%d "
1379 "test_boost_duration=%d shutdown_secs=%d "
1380 "stall_cpu=%d stall_cpu_holdoff=%d "
1382 "onoff_interval=%d onoff_holdoff=%d\n",
1383 torture_type
, tag
, nrealreaders
, nfakewriters
,
1384 stat_interval
, verbose
, test_no_idle_hz
, shuffle_interval
,
1385 stutter
, irqreader
, fqs_duration
, fqs_holdoff
, fqs_stutter
,
1386 test_boost
, cur_ops
->can_boost
,
1387 test_boost_interval
, test_boost_duration
, shutdown_secs
,
1388 stall_cpu
, stall_cpu_holdoff
,
1390 onoff_interval
, onoff_holdoff
);
1393 static int rcutorture_booster_cleanup(unsigned int cpu
)
1395 struct task_struct
*t
;
1397 if (boost_tasks
[cpu
] == NULL
)
1399 mutex_lock(&boost_mutex
);
1400 t
= boost_tasks
[cpu
];
1401 boost_tasks
[cpu
] = NULL
;
1402 mutex_unlock(&boost_mutex
);
1404 /* This must be outside of the mutex, otherwise deadlock! */
1405 torture_stop_kthread(rcu_torture_boost
, t
);
1409 static int rcutorture_booster_init(unsigned int cpu
)
1413 if (boost_tasks
[cpu
] != NULL
)
1414 return 0; /* Already created, nothing more to do. */
1416 /* Don't allow time recalculation while creating a new task. */
1417 mutex_lock(&boost_mutex
);
1418 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task");
1419 boost_tasks
[cpu
] = kthread_create_on_node(rcu_torture_boost
, NULL
,
1421 "rcu_torture_boost");
1422 if (IS_ERR(boost_tasks
[cpu
])) {
1423 retval
= PTR_ERR(boost_tasks
[cpu
]);
1424 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed");
1425 n_rcu_torture_boost_ktrerror
++;
1426 boost_tasks
[cpu
] = NULL
;
1427 mutex_unlock(&boost_mutex
);
1430 kthread_bind(boost_tasks
[cpu
], cpu
);
1431 wake_up_process(boost_tasks
[cpu
]);
1432 mutex_unlock(&boost_mutex
);
1437 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then
1438 * induces a CPU stall for the time specified by stall_cpu.
1440 static int rcu_torture_stall(void *args
)
1442 unsigned long stop_at
;
1444 VERBOSE_TOROUT_STRING("rcu_torture_stall task started");
1445 if (stall_cpu_holdoff
> 0) {
1446 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff");
1447 schedule_timeout_interruptible(stall_cpu_holdoff
* HZ
);
1448 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff");
1450 if (!kthread_should_stop()) {
1451 stop_at
= get_seconds() + stall_cpu
;
1452 /* RCU CPU stall is expected behavior in following code. */
1453 pr_alert("rcu_torture_stall start.\n");
1456 while (ULONG_CMP_LT(get_seconds(), stop_at
))
1457 continue; /* Induce RCU CPU stall warning. */
1460 pr_alert("rcu_torture_stall end.\n");
1462 torture_shutdown_absorb("rcu_torture_stall");
1463 while (!kthread_should_stop())
1464 schedule_timeout_interruptible(10 * HZ
);
1468 /* Spawn CPU-stall kthread, if stall_cpu specified. */
1469 static int __init
rcu_torture_stall_init(void)
1473 return torture_create_kthread(rcu_torture_stall
, NULL
, stall_task
);
1476 /* Callback function for RCU barrier testing. */
1477 static void rcu_torture_barrier_cbf(struct rcu_head
*rcu
)
1479 atomic_inc(&barrier_cbs_invoked
);
1482 /* kthread function to register callbacks used to test RCU barriers. */
1483 static int rcu_torture_barrier_cbs(void *arg
)
1485 long myid
= (long)arg
;
1488 struct rcu_head rcu
;
1490 init_rcu_head_on_stack(&rcu
);
1491 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started");
1492 set_user_nice(current
, MAX_NICE
);
1494 wait_event(barrier_cbs_wq
[myid
],
1496 smp_load_acquire(&barrier_phase
)) != lastphase
||
1497 torture_must_stop());
1498 lastphase
= newphase
;
1499 if (torture_must_stop())
1502 * The above smp_load_acquire() ensures barrier_phase load
1503 * is ordered before the following ->call().
1505 local_irq_disable(); /* Just to test no-irq call_rcu(). */
1506 cur_ops
->call(&rcu
, rcu_torture_barrier_cbf
);
1508 if (atomic_dec_and_test(&barrier_cbs_count
))
1509 wake_up(&barrier_wq
);
1510 } while (!torture_must_stop());
1511 if (cur_ops
->cb_barrier
!= NULL
)
1512 cur_ops
->cb_barrier();
1513 destroy_rcu_head_on_stack(&rcu
);
1514 torture_kthread_stopping("rcu_torture_barrier_cbs");
1518 /* kthread function to drive and coordinate RCU barrier testing. */
1519 static int rcu_torture_barrier(void *arg
)
1523 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting");
1525 atomic_set(&barrier_cbs_invoked
, 0);
1526 atomic_set(&barrier_cbs_count
, n_barrier_cbs
);
1527 /* Ensure barrier_phase ordered after prior assignments. */
1528 smp_store_release(&barrier_phase
, !barrier_phase
);
1529 for (i
= 0; i
< n_barrier_cbs
; i
++)
1530 wake_up(&barrier_cbs_wq
[i
]);
1531 wait_event(barrier_wq
,
1532 atomic_read(&barrier_cbs_count
) == 0 ||
1533 torture_must_stop());
1534 if (torture_must_stop())
1536 n_barrier_attempts
++;
1537 cur_ops
->cb_barrier(); /* Implies smp_mb() for wait_event(). */
1538 if (atomic_read(&barrier_cbs_invoked
) != n_barrier_cbs
) {
1539 n_rcu_torture_barrier_error
++;
1540 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n",
1541 atomic_read(&barrier_cbs_invoked
),
1545 n_barrier_successes
++;
1546 schedule_timeout_interruptible(HZ
/ 10);
1547 } while (!torture_must_stop());
1548 torture_kthread_stopping("rcu_torture_barrier");
1552 /* Initialize RCU barrier testing. */
1553 static int rcu_torture_barrier_init(void)
1558 if (n_barrier_cbs
<= 0)
1560 if (cur_ops
->call
== NULL
|| cur_ops
->cb_barrier
== NULL
) {
1561 pr_alert("%s" TORTURE_FLAG
1562 " Call or barrier ops missing for %s,\n",
1563 torture_type
, cur_ops
->name
);
1564 pr_alert("%s" TORTURE_FLAG
1565 " RCU barrier testing omitted from run.\n",
1569 atomic_set(&barrier_cbs_count
, 0);
1570 atomic_set(&barrier_cbs_invoked
, 0);
1572 kzalloc(n_barrier_cbs
* sizeof(barrier_cbs_tasks
[0]),
1575 kzalloc(n_barrier_cbs
* sizeof(barrier_cbs_wq
[0]),
1577 if (barrier_cbs_tasks
== NULL
|| !barrier_cbs_wq
)
1579 for (i
= 0; i
< n_barrier_cbs
; i
++) {
1580 init_waitqueue_head(&barrier_cbs_wq
[i
]);
1581 ret
= torture_create_kthread(rcu_torture_barrier_cbs
,
1583 barrier_cbs_tasks
[i
]);
1587 return torture_create_kthread(rcu_torture_barrier
, NULL
, barrier_task
);
1590 /* Clean up after RCU barrier testing. */
1591 static void rcu_torture_barrier_cleanup(void)
1595 torture_stop_kthread(rcu_torture_barrier
, barrier_task
);
1596 if (barrier_cbs_tasks
!= NULL
) {
1597 for (i
= 0; i
< n_barrier_cbs
; i
++)
1598 torture_stop_kthread(rcu_torture_barrier_cbs
,
1599 barrier_cbs_tasks
[i
]);
1600 kfree(barrier_cbs_tasks
);
1601 barrier_cbs_tasks
= NULL
;
1603 if (barrier_cbs_wq
!= NULL
) {
1604 kfree(barrier_cbs_wq
);
1605 barrier_cbs_wq
= NULL
;
1609 static enum cpuhp_state rcutor_hp
;
1612 rcu_torture_cleanup(void)
1616 rcutorture_record_test_transition();
1617 if (torture_cleanup_begin()) {
1618 if (cur_ops
->cb_barrier
!= NULL
)
1619 cur_ops
->cb_barrier();
1623 rcu_torture_barrier_cleanup();
1624 torture_stop_kthread(rcu_torture_stall
, stall_task
);
1625 torture_stop_kthread(rcu_torture_writer
, writer_task
);
1628 for (i
= 0; i
< nrealreaders
; i
++)
1629 torture_stop_kthread(rcu_torture_reader
,
1631 kfree(reader_tasks
);
1633 rcu_torture_current
= NULL
;
1635 if (fakewriter_tasks
) {
1636 for (i
= 0; i
< nfakewriters
; i
++) {
1637 torture_stop_kthread(rcu_torture_fakewriter
,
1638 fakewriter_tasks
[i
]);
1640 kfree(fakewriter_tasks
);
1641 fakewriter_tasks
= NULL
;
1644 torture_stop_kthread(rcu_torture_stats
, stats_task
);
1645 torture_stop_kthread(rcu_torture_fqs
, fqs_task
);
1646 for (i
= 0; i
< ncbflooders
; i
++)
1647 torture_stop_kthread(rcu_torture_cbflood
, cbflood_task
[i
]);
1648 if ((test_boost
== 1 && cur_ops
->can_boost
) ||
1650 cpuhp_remove_state(rcutor_hp
);
1653 * Wait for all RCU callbacks to fire, then do flavor-specific
1654 * cleanup operations.
1656 if (cur_ops
->cb_barrier
!= NULL
)
1657 cur_ops
->cb_barrier();
1658 if (cur_ops
->cleanup
!= NULL
)
1661 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */
1663 if (atomic_read(&n_rcu_torture_error
) || n_rcu_torture_barrier_error
)
1664 rcu_torture_print_module_parms(cur_ops
, "End of test: FAILURE");
1665 else if (torture_onoff_failures())
1666 rcu_torture_print_module_parms(cur_ops
,
1667 "End of test: RCU_HOTPLUG");
1669 rcu_torture_print_module_parms(cur_ops
, "End of test: SUCCESS");
1670 torture_cleanup_end();
1673 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
1674 static void rcu_torture_leak_cb(struct rcu_head
*rhp
)
1678 static void rcu_torture_err_cb(struct rcu_head
*rhp
)
1681 * This -might- happen due to race conditions, but is unlikely.
1682 * The scenario that leads to this happening is that the
1683 * first of the pair of duplicate callbacks is queued,
1684 * someone else starts a grace period that includes that
1685 * callback, then the second of the pair must wait for the
1686 * next grace period. Unlikely, but can happen. If it
1687 * does happen, the debug-objects subsystem won't have splatted.
1689 pr_alert("rcutorture: duplicated callback was invoked.\n");
1691 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
1694 * Verify that double-free causes debug-objects to complain, but only
1695 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test
1696 * cannot be carried out.
1698 static void rcu_test_debug_objects(void)
1700 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
1701 struct rcu_head rh1
;
1702 struct rcu_head rh2
;
1704 init_rcu_head_on_stack(&rh1
);
1705 init_rcu_head_on_stack(&rh2
);
1706 pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n");
1708 /* Try to queue the rh2 pair of callbacks for the same grace period. */
1709 preempt_disable(); /* Prevent preemption from interrupting test. */
1710 rcu_read_lock(); /* Make it impossible to finish a grace period. */
1711 call_rcu(&rh1
, rcu_torture_leak_cb
); /* Start grace period. */
1712 local_irq_disable(); /* Make it harder to start a new grace period. */
1713 call_rcu(&rh2
, rcu_torture_leak_cb
);
1714 call_rcu(&rh2
, rcu_torture_err_cb
); /* Duplicate callback. */
1719 /* Wait for them all to get done so we can safely return. */
1721 pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n");
1722 destroy_rcu_head_on_stack(&rh1
);
1723 destroy_rcu_head_on_stack(&rh2
);
1724 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
1725 pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n");
1726 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
1730 rcu_torture_init(void)
1735 static struct rcu_torture_ops
*torture_ops
[] = {
1736 &rcu_ops
, &rcu_bh_ops
, &rcu_busted_ops
, &srcu_ops
, &srcud_ops
,
1737 &sched_ops
, RCUTORTURE_TASKS_OPS
1740 if (!torture_init_begin(torture_type
, verbose
, &torture_runnable
))
1743 /* Process args and tell the world that the torturer is on the job. */
1744 for (i
= 0; i
< ARRAY_SIZE(torture_ops
); i
++) {
1745 cur_ops
= torture_ops
[i
];
1746 if (strcmp(torture_type
, cur_ops
->name
) == 0)
1749 if (i
== ARRAY_SIZE(torture_ops
)) {
1750 pr_alert("rcu-torture: invalid torture type: \"%s\"\n",
1752 pr_alert("rcu-torture types:");
1753 for (i
= 0; i
< ARRAY_SIZE(torture_ops
); i
++)
1754 pr_alert(" %s", torture_ops
[i
]->name
);
1759 if (cur_ops
->fqs
== NULL
&& fqs_duration
!= 0) {
1760 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n");
1766 if (nreaders
>= 0) {
1767 nrealreaders
= nreaders
;
1769 nrealreaders
= num_online_cpus() - 2 - nreaders
;
1770 if (nrealreaders
<= 0)
1773 rcu_torture_print_module_parms(cur_ops
, "Start of test");
1775 /* Set up the freelist. */
1777 INIT_LIST_HEAD(&rcu_torture_freelist
);
1778 for (i
= 0; i
< ARRAY_SIZE(rcu_tortures
); i
++) {
1779 rcu_tortures
[i
].rtort_mbtest
= 0;
1780 list_add_tail(&rcu_tortures
[i
].rtort_free
,
1781 &rcu_torture_freelist
);
1784 /* Initialize the statistics so that each run gets its own numbers. */
1786 rcu_torture_current
= NULL
;
1787 rcu_torture_current_version
= 0;
1788 atomic_set(&n_rcu_torture_alloc
, 0);
1789 atomic_set(&n_rcu_torture_alloc_fail
, 0);
1790 atomic_set(&n_rcu_torture_free
, 0);
1791 atomic_set(&n_rcu_torture_mberror
, 0);
1792 atomic_set(&n_rcu_torture_error
, 0);
1793 n_rcu_torture_barrier_error
= 0;
1794 n_rcu_torture_boost_ktrerror
= 0;
1795 n_rcu_torture_boost_rterror
= 0;
1796 n_rcu_torture_boost_failure
= 0;
1797 n_rcu_torture_boosts
= 0;
1798 for (i
= 0; i
< RCU_TORTURE_PIPE_LEN
+ 1; i
++)
1799 atomic_set(&rcu_torture_wcount
[i
], 0);
1800 for_each_possible_cpu(cpu
) {
1801 for (i
= 0; i
< RCU_TORTURE_PIPE_LEN
+ 1; i
++) {
1802 per_cpu(rcu_torture_count
, cpu
)[i
] = 0;
1803 per_cpu(rcu_torture_batch
, cpu
)[i
] = 0;
1807 /* Start up the kthreads. */
1809 firsterr
= torture_create_kthread(rcu_torture_writer
, NULL
,
1813 if (nfakewriters
> 0) {
1814 fakewriter_tasks
= kzalloc(nfakewriters
*
1815 sizeof(fakewriter_tasks
[0]),
1817 if (fakewriter_tasks
== NULL
) {
1818 VERBOSE_TOROUT_ERRSTRING("out of memory");
1823 for (i
= 0; i
< nfakewriters
; i
++) {
1824 firsterr
= torture_create_kthread(rcu_torture_fakewriter
,
1825 NULL
, fakewriter_tasks
[i
]);
1829 reader_tasks
= kzalloc(nrealreaders
* sizeof(reader_tasks
[0]),
1831 if (reader_tasks
== NULL
) {
1832 VERBOSE_TOROUT_ERRSTRING("out of memory");
1836 for (i
= 0; i
< nrealreaders
; i
++) {
1837 firsterr
= torture_create_kthread(rcu_torture_reader
, NULL
,
1842 if (stat_interval
> 0) {
1843 firsterr
= torture_create_kthread(rcu_torture_stats
, NULL
,
1848 if (test_no_idle_hz
&& shuffle_interval
> 0) {
1849 firsterr
= torture_shuffle_init(shuffle_interval
* HZ
);
1856 firsterr
= torture_stutter_init(stutter
* HZ
);
1860 if (fqs_duration
< 0)
1863 /* Create the fqs thread */
1864 firsterr
= torture_create_kthread(rcu_torture_fqs
, NULL
,
1869 if (test_boost_interval
< 1)
1870 test_boost_interval
= 1;
1871 if (test_boost_duration
< 2)
1872 test_boost_duration
= 2;
1873 if ((test_boost
== 1 && cur_ops
->can_boost
) ||
1876 boost_starttime
= jiffies
+ test_boost_interval
* HZ
;
1878 firsterr
= cpuhp_setup_state(CPUHP_AP_ONLINE_DYN
, "RCU_TORTURE",
1879 rcutorture_booster_init
,
1880 rcutorture_booster_cleanup
);
1883 rcutor_hp
= firsterr
;
1885 firsterr
= torture_shutdown_init(shutdown_secs
, rcu_torture_cleanup
);
1888 firsterr
= torture_onoff_init(onoff_holdoff
* HZ
, onoff_interval
* HZ
);
1891 firsterr
= rcu_torture_stall_init();
1894 firsterr
= rcu_torture_barrier_init();
1898 rcu_test_debug_objects();
1899 if (cbflood_n_burst
> 0) {
1900 /* Create the cbflood threads */
1901 ncbflooders
= (num_online_cpus() + 3) / 4;
1902 cbflood_task
= kcalloc(ncbflooders
, sizeof(*cbflood_task
),
1904 if (!cbflood_task
) {
1905 VERBOSE_TOROUT_ERRSTRING("out of memory");
1909 for (i
= 0; i
< ncbflooders
; i
++) {
1910 firsterr
= torture_create_kthread(rcu_torture_cbflood
,
1917 rcutorture_record_test_transition();
1923 rcu_torture_cleanup();
1927 module_init(rcu_torture_init
);
1928 module_exit(rcu_torture_cleanup
);