2 * Read-Copy Update module-based torture test facility
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, you can access it online at
16 * http://www.gnu.org/licenses/gpl-2.0.html.
18 * Copyright (C) IBM Corporation, 2005, 2006
20 * Authors: Paul E. McKenney <paulmck@us.ibm.com>
21 * Josh Triplett <josh@joshtriplett.org>
23 * See also: Documentation/RCU/torture.txt
25 #include <linux/types.h>
26 #include <linux/kernel.h>
27 #include <linux/init.h>
28 #include <linux/module.h>
29 #include <linux/kthread.h>
30 #include <linux/err.h>
31 #include <linux/spinlock.h>
32 #include <linux/smp.h>
33 #include <linux/rcupdate.h>
34 #include <linux/interrupt.h>
35 #include <linux/sched/signal.h>
36 #include <uapi/linux/sched/types.h>
37 #include <linux/atomic.h>
38 #include <linux/bitops.h>
39 #include <linux/completion.h>
40 #include <linux/moduleparam.h>
41 #include <linux/percpu.h>
42 #include <linux/notifier.h>
43 #include <linux/reboot.h>
44 #include <linux/freezer.h>
45 #include <linux/cpu.h>
46 #include <linux/delay.h>
47 #include <linux/stat.h>
48 #include <linux/srcu.h>
49 #include <linux/slab.h>
50 #include <linux/trace_clock.h>
51 #include <asm/byteorder.h>
52 #include <linux/torture.h>
53 #include <linux/vmalloc.h>
55 MODULE_LICENSE("GPL");
56 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and Josh Triplett <josh@joshtriplett.org>");
59 torture_param(int, cbflood_inter_holdoff
, HZ
,
60 "Holdoff between floods (jiffies)");
61 torture_param(int, cbflood_intra_holdoff
, 1,
62 "Holdoff between bursts (jiffies)");
63 torture_param(int, cbflood_n_burst
, 3, "# bursts in flood, zero to disable");
64 torture_param(int, cbflood_n_per_burst
, 20000,
65 "# callbacks per burst in flood");
66 torture_param(int, fqs_duration
, 0,
67 "Duration of fqs bursts (us), 0 to disable");
68 torture_param(int, fqs_holdoff
, 0, "Holdoff time within fqs bursts (us)");
69 torture_param(int, fqs_stutter
, 3, "Wait time between fqs bursts (s)");
70 torture_param(bool, gp_cond
, false, "Use conditional/async GP wait primitives");
71 torture_param(bool, gp_exp
, false, "Use expedited GP wait primitives");
72 torture_param(bool, gp_normal
, false,
73 "Use normal (non-expedited) GP wait primitives");
74 torture_param(bool, gp_sync
, false, "Use synchronous GP wait primitives");
75 torture_param(int, irqreader
, 1, "Allow RCU readers from irq handlers");
76 torture_param(int, n_barrier_cbs
, 0,
77 "# of callbacks/kthreads for barrier testing");
78 torture_param(int, nfakewriters
, 4, "Number of RCU fake writer threads");
79 torture_param(int, nreaders
, -1, "Number of RCU reader threads");
80 torture_param(int, object_debug
, 0,
81 "Enable debug-object double call_rcu() testing");
82 torture_param(int, onoff_holdoff
, 0, "Time after boot before CPU hotplugs (s)");
83 torture_param(int, onoff_interval
, 0,
84 "Time between CPU hotplugs (s), 0=disable");
85 torture_param(int, shuffle_interval
, 3, "Number of seconds between shuffles");
86 torture_param(int, shutdown_secs
, 0, "Shutdown time (s), <= zero to disable.");
87 torture_param(int, stall_cpu
, 0, "Stall duration (s), zero to disable.");
88 torture_param(int, stall_cpu_holdoff
, 10,
89 "Time to wait before starting stall (s).");
90 torture_param(int, stat_interval
, 60,
91 "Number of seconds between stats printk()s");
92 torture_param(int, stutter
, 5, "Number of seconds to run/halt test");
93 torture_param(int, test_boost
, 1, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
94 torture_param(int, test_boost_duration
, 4,
95 "Duration of each boost test, seconds.");
96 torture_param(int, test_boost_interval
, 7,
97 "Interval between boost tests, seconds.");
98 torture_param(bool, test_no_idle_hz
, true,
99 "Test support for tickless idle CPUs");
100 torture_param(bool, verbose
, true,
101 "Enable verbose debugging printk()s");
103 static char *torture_type
= "rcu";
104 module_param(torture_type
, charp
, 0444);
105 MODULE_PARM_DESC(torture_type
, "Type of RCU to torture (rcu, rcu_bh, ...)");
107 static int nrealreaders
;
108 static int ncbflooders
;
109 static struct task_struct
*writer_task
;
110 static struct task_struct
**fakewriter_tasks
;
111 static struct task_struct
**reader_tasks
;
112 static struct task_struct
*stats_task
;
113 static struct task_struct
**cbflood_task
;
114 static struct task_struct
*fqs_task
;
115 static struct task_struct
*boost_tasks
[NR_CPUS
];
116 static struct task_struct
*stall_task
;
117 static struct task_struct
**barrier_cbs_tasks
;
118 static struct task_struct
*barrier_task
;
120 #define RCU_TORTURE_PIPE_LEN 10
123 struct rcu_head rtort_rcu
;
124 int rtort_pipe_count
;
125 struct list_head rtort_free
;
129 static LIST_HEAD(rcu_torture_freelist
);
130 static struct rcu_torture __rcu
*rcu_torture_current
;
131 static unsigned long rcu_torture_current_version
;
132 static struct rcu_torture rcu_tortures
[10 * RCU_TORTURE_PIPE_LEN
];
133 static DEFINE_SPINLOCK(rcu_torture_lock
);
134 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN
+ 1], rcu_torture_count
);
135 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN
+ 1], rcu_torture_batch
);
136 static atomic_t rcu_torture_wcount
[RCU_TORTURE_PIPE_LEN
+ 1];
137 static atomic_t n_rcu_torture_alloc
;
138 static atomic_t n_rcu_torture_alloc_fail
;
139 static atomic_t n_rcu_torture_free
;
140 static atomic_t n_rcu_torture_mberror
;
141 static atomic_t n_rcu_torture_error
;
142 static long n_rcu_torture_barrier_error
;
143 static long n_rcu_torture_boost_ktrerror
;
144 static long n_rcu_torture_boost_rterror
;
145 static long n_rcu_torture_boost_failure
;
146 static long n_rcu_torture_boosts
;
147 static long n_rcu_torture_timers
;
148 static long n_barrier_attempts
;
149 static long n_barrier_successes
;
150 static atomic_long_t n_cbfloods
;
151 static struct list_head rcu_torture_removed
;
153 static int rcu_torture_writer_state
;
154 #define RTWS_FIXED_DELAY 0
156 #define RTWS_REPLACE 2
157 #define RTWS_DEF_FREE 3
158 #define RTWS_EXP_SYNC 4
159 #define RTWS_COND_GET 5
160 #define RTWS_COND_SYNC 6
162 #define RTWS_STUTTER 8
163 #define RTWS_STOPPING 9
164 static const char * const rcu_torture_writer_state_names
[] = {
177 static const char *rcu_torture_writer_state_getname(void)
179 unsigned int i
= READ_ONCE(rcu_torture_writer_state
);
181 if (i
>= ARRAY_SIZE(rcu_torture_writer_state_names
))
183 return rcu_torture_writer_state_names
[i
];
186 static int torture_runnable
= IS_ENABLED(MODULE
);
187 module_param(torture_runnable
, int, 0444);
188 MODULE_PARM_DESC(torture_runnable
, "Start rcutorture at boot");
190 #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU)
191 #define rcu_can_boost() 1
192 #else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
193 #define rcu_can_boost() 0
194 #endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */
196 #ifdef CONFIG_RCU_TRACE
197 static u64 notrace
rcu_trace_clock_local(void)
199 u64 ts
= trace_clock_local();
200 unsigned long __maybe_unused ts_rem
= do_div(ts
, NSEC_PER_USEC
);
203 #else /* #ifdef CONFIG_RCU_TRACE */
204 static u64 notrace
rcu_trace_clock_local(void)
208 #endif /* #else #ifdef CONFIG_RCU_TRACE */
210 static unsigned long boost_starttime
; /* jiffies of next boost test start. */
211 static DEFINE_MUTEX(boost_mutex
); /* protect setting boost_starttime */
212 /* and boost task create/destroy. */
213 static atomic_t barrier_cbs_count
; /* Barrier callbacks registered. */
214 static bool barrier_phase
; /* Test phase. */
215 static atomic_t barrier_cbs_invoked
; /* Barrier callbacks invoked. */
216 static wait_queue_head_t
*barrier_cbs_wq
; /* Coordinate barrier testing. */
217 static DECLARE_WAIT_QUEUE_HEAD(barrier_wq
);
220 * Allocate an element from the rcu_tortures pool.
222 static struct rcu_torture
*
223 rcu_torture_alloc(void)
227 spin_lock_bh(&rcu_torture_lock
);
228 if (list_empty(&rcu_torture_freelist
)) {
229 atomic_inc(&n_rcu_torture_alloc_fail
);
230 spin_unlock_bh(&rcu_torture_lock
);
233 atomic_inc(&n_rcu_torture_alloc
);
234 p
= rcu_torture_freelist
.next
;
236 spin_unlock_bh(&rcu_torture_lock
);
237 return container_of(p
, struct rcu_torture
, rtort_free
);
241 * Free an element to the rcu_tortures pool.
244 rcu_torture_free(struct rcu_torture
*p
)
246 atomic_inc(&n_rcu_torture_free
);
247 spin_lock_bh(&rcu_torture_lock
);
248 list_add_tail(&p
->rtort_free
, &rcu_torture_freelist
);
249 spin_unlock_bh(&rcu_torture_lock
);
253 * Operations vector for selecting different types of tests.
256 struct rcu_torture_ops
{
259 void (*cleanup
)(void);
260 int (*readlock
)(void);
261 void (*read_delay
)(struct torture_random_state
*rrsp
);
262 void (*readunlock
)(int idx
);
263 unsigned long (*started
)(void);
264 unsigned long (*completed
)(void);
265 void (*deferred_free
)(struct rcu_torture
*p
);
267 void (*exp_sync
)(void);
268 unsigned long (*get_state
)(void);
269 void (*cond_sync
)(unsigned long oldstate
);
270 call_rcu_func_t call
;
271 void (*cb_barrier
)(void);
279 static struct rcu_torture_ops
*cur_ops
;
282 * Definitions for rcu torture testing.
285 static int rcu_torture_read_lock(void) __acquires(RCU
)
291 static void rcu_read_delay(struct torture_random_state
*rrsp
)
293 unsigned long started
;
294 unsigned long completed
;
295 const unsigned long shortdelay_us
= 200;
296 const unsigned long longdelay_ms
= 50;
297 unsigned long long ts
;
299 /* We want a short delay sometimes to make a reader delay the grace
300 * period, and we want a long delay occasionally to trigger
301 * force_quiescent_state. */
303 if (!(torture_random(rrsp
) % (nrealreaders
* 2000 * longdelay_ms
))) {
304 started
= cur_ops
->completed();
305 ts
= rcu_trace_clock_local();
306 mdelay(longdelay_ms
);
307 completed
= cur_ops
->completed();
308 do_trace_rcu_torture_read(cur_ops
->name
, NULL
, ts
,
311 if (!(torture_random(rrsp
) % (nrealreaders
* 2 * shortdelay_us
)))
312 udelay(shortdelay_us
);
313 #ifdef CONFIG_PREEMPT
314 if (!preempt_count() &&
315 !(torture_random(rrsp
) % (nrealreaders
* 20000)))
316 preempt_schedule(); /* No QS if preempt_disable() in effect */
320 static void rcu_torture_read_unlock(int idx
) __releases(RCU
)
326 * Update callback in the pipe. This should be invoked after a grace period.
329 rcu_torture_pipe_update_one(struct rcu_torture
*rp
)
333 i
= rp
->rtort_pipe_count
;
334 if (i
> RCU_TORTURE_PIPE_LEN
)
335 i
= RCU_TORTURE_PIPE_LEN
;
336 atomic_inc(&rcu_torture_wcount
[i
]);
337 if (++rp
->rtort_pipe_count
>= RCU_TORTURE_PIPE_LEN
) {
338 rp
->rtort_mbtest
= 0;
345 * Update all callbacks in the pipe. Suitable for synchronous grace-period
349 rcu_torture_pipe_update(struct rcu_torture
*old_rp
)
351 struct rcu_torture
*rp
;
352 struct rcu_torture
*rp1
;
355 list_add(&old_rp
->rtort_free
, &rcu_torture_removed
);
356 list_for_each_entry_safe(rp
, rp1
, &rcu_torture_removed
, rtort_free
) {
357 if (rcu_torture_pipe_update_one(rp
)) {
358 list_del(&rp
->rtort_free
);
359 rcu_torture_free(rp
);
365 rcu_torture_cb(struct rcu_head
*p
)
367 struct rcu_torture
*rp
= container_of(p
, struct rcu_torture
, rtort_rcu
);
369 if (torture_must_stop_irq()) {
370 /* Test is ending, just drop callbacks on the floor. */
371 /* The next initialization will pick up the pieces. */
374 if (rcu_torture_pipe_update_one(rp
))
375 rcu_torture_free(rp
);
377 cur_ops
->deferred_free(rp
);
380 static unsigned long rcu_no_completed(void)
385 static void rcu_torture_deferred_free(struct rcu_torture
*p
)
387 call_rcu(&p
->rtort_rcu
, rcu_torture_cb
);
390 static void rcu_sync_torture_init(void)
392 INIT_LIST_HEAD(&rcu_torture_removed
);
395 static struct rcu_torture_ops rcu_ops
= {
397 .init
= rcu_sync_torture_init
,
398 .readlock
= rcu_torture_read_lock
,
399 .read_delay
= rcu_read_delay
,
400 .readunlock
= rcu_torture_read_unlock
,
401 .started
= rcu_batches_started
,
402 .completed
= rcu_batches_completed
,
403 .deferred_free
= rcu_torture_deferred_free
,
404 .sync
= synchronize_rcu
,
405 .exp_sync
= synchronize_rcu_expedited
,
406 .get_state
= get_state_synchronize_rcu
,
407 .cond_sync
= cond_synchronize_rcu
,
409 .cb_barrier
= rcu_barrier
,
410 .fqs
= rcu_force_quiescent_state
,
413 .can_boost
= rcu_can_boost(),
418 * Definitions for rcu_bh torture testing.
421 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH
)
427 static void rcu_bh_torture_read_unlock(int idx
) __releases(RCU_BH
)
429 rcu_read_unlock_bh();
432 static void rcu_bh_torture_deferred_free(struct rcu_torture
*p
)
434 call_rcu_bh(&p
->rtort_rcu
, rcu_torture_cb
);
437 static struct rcu_torture_ops rcu_bh_ops
= {
438 .ttype
= RCU_BH_FLAVOR
,
439 .init
= rcu_sync_torture_init
,
440 .readlock
= rcu_bh_torture_read_lock
,
441 .read_delay
= rcu_read_delay
, /* just reuse rcu's version. */
442 .readunlock
= rcu_bh_torture_read_unlock
,
443 .started
= rcu_batches_started_bh
,
444 .completed
= rcu_batches_completed_bh
,
445 .deferred_free
= rcu_bh_torture_deferred_free
,
446 .sync
= synchronize_rcu_bh
,
447 .exp_sync
= synchronize_rcu_bh_expedited
,
449 .cb_barrier
= rcu_barrier_bh
,
450 .fqs
= rcu_bh_force_quiescent_state
,
457 * Don't even think about trying any of these in real life!!!
458 * The names includes "busted", and they really means it!
459 * The only purpose of these functions is to provide a buggy RCU
460 * implementation to make sure that rcutorture correctly emits
461 * buggy-RCU error messages.
463 static void rcu_busted_torture_deferred_free(struct rcu_torture
*p
)
465 /* This is a deliberate bug for testing purposes only! */
466 rcu_torture_cb(&p
->rtort_rcu
);
469 static void synchronize_rcu_busted(void)
471 /* This is a deliberate bug for testing purposes only! */
475 call_rcu_busted(struct rcu_head
*head
, rcu_callback_t func
)
477 /* This is a deliberate bug for testing purposes only! */
481 static struct rcu_torture_ops rcu_busted_ops
= {
482 .ttype
= INVALID_RCU_FLAVOR
,
483 .init
= rcu_sync_torture_init
,
484 .readlock
= rcu_torture_read_lock
,
485 .read_delay
= rcu_read_delay
, /* just reuse rcu's version. */
486 .readunlock
= rcu_torture_read_unlock
,
487 .started
= rcu_no_completed
,
488 .completed
= rcu_no_completed
,
489 .deferred_free
= rcu_busted_torture_deferred_free
,
490 .sync
= synchronize_rcu_busted
,
491 .exp_sync
= synchronize_rcu_busted
,
492 .call
= call_rcu_busted
,
501 * Definitions for srcu torture testing.
504 DEFINE_STATIC_SRCU(srcu_ctl
);
505 static struct srcu_struct srcu_ctld
;
506 static struct srcu_struct
*srcu_ctlp
= &srcu_ctl
;
508 static int srcu_torture_read_lock(void) __acquires(srcu_ctlp
)
510 return srcu_read_lock(srcu_ctlp
);
513 static void srcu_read_delay(struct torture_random_state
*rrsp
)
516 const long uspertick
= 1000000 / HZ
;
517 const long longdelay
= 10;
519 /* We want there to be long-running readers, but not all the time. */
521 delay
= torture_random(rrsp
) %
522 (nrealreaders
* 2 * longdelay
* uspertick
);
524 schedule_timeout_interruptible(longdelay
);
526 rcu_read_delay(rrsp
);
529 static void srcu_torture_read_unlock(int idx
) __releases(srcu_ctlp
)
531 srcu_read_unlock(srcu_ctlp
, idx
);
534 static unsigned long srcu_torture_completed(void)
536 return srcu_batches_completed(srcu_ctlp
);
539 static void srcu_torture_deferred_free(struct rcu_torture
*rp
)
541 call_srcu(srcu_ctlp
, &rp
->rtort_rcu
, rcu_torture_cb
);
544 static void srcu_torture_synchronize(void)
546 synchronize_srcu(srcu_ctlp
);
549 static void srcu_torture_call(struct rcu_head
*head
,
552 call_srcu(srcu_ctlp
, head
, func
);
555 static void srcu_torture_barrier(void)
557 srcu_barrier(srcu_ctlp
);
560 static void srcu_torture_stats(void)
562 int __maybe_unused cpu
;
565 #if defined(CONFIG_TREE_SRCU) || defined(CONFIG_CLASSIC_SRCU)
566 #ifdef CONFIG_TREE_SRCU
567 idx
= srcu_ctlp
->srcu_idx
& 0x1;
568 #else /* #ifdef CONFIG_TREE_SRCU */
569 idx
= srcu_ctlp
->completed
& 0x1;
570 #endif /* #else #ifdef CONFIG_TREE_SRCU */
571 pr_alert("%s%s Tree SRCU per-CPU(idx=%d):",
572 torture_type
, TORTURE_FLAG
, idx
);
573 for_each_possible_cpu(cpu
) {
574 unsigned long l0
, l1
;
575 unsigned long u0
, u1
;
577 #ifdef CONFIG_TREE_SRCU
578 struct srcu_data
*counts
;
580 counts
= per_cpu_ptr(srcu_ctlp
->sda
, cpu
);
581 u0
= counts
->srcu_unlock_count
[!idx
];
582 u1
= counts
->srcu_unlock_count
[idx
];
583 #else /* #ifdef CONFIG_TREE_SRCU */
584 struct srcu_array
*counts
;
586 counts
= per_cpu_ptr(srcu_ctlp
->per_cpu_ref
, cpu
);
587 u0
= counts
->unlock_count
[!idx
];
588 u1
= counts
->unlock_count
[idx
];
589 #endif /* #else #ifdef CONFIG_TREE_SRCU */
592 * Make sure that a lock is always counted if the corresponding
597 #ifdef CONFIG_TREE_SRCU
598 l0
= counts
->srcu_lock_count
[!idx
];
599 l1
= counts
->srcu_lock_count
[idx
];
600 #else /* #ifdef CONFIG_TREE_SRCU */
601 l0
= counts
->lock_count
[!idx
];
602 l1
= counts
->lock_count
[idx
];
603 #endif /* #else #ifdef CONFIG_TREE_SRCU */
607 pr_cont(" %d(%ld,%ld)", cpu
, c0
, c1
);
610 #elif defined(CONFIG_TINY_SRCU)
611 idx
= READ_ONCE(srcu_ctlp
->srcu_idx
) & 0x1;
612 pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%d,%d)\n",
613 torture_type
, TORTURE_FLAG
, idx
,
614 READ_ONCE(srcu_ctlp
->srcu_lock_nesting
[!idx
]),
615 READ_ONCE(srcu_ctlp
->srcu_lock_nesting
[idx
]));
619 static void srcu_torture_synchronize_expedited(void)
621 synchronize_srcu_expedited(srcu_ctlp
);
624 static struct rcu_torture_ops srcu_ops
= {
625 .ttype
= SRCU_FLAVOR
,
626 .init
= rcu_sync_torture_init
,
627 .readlock
= srcu_torture_read_lock
,
628 .read_delay
= srcu_read_delay
,
629 .readunlock
= srcu_torture_read_unlock
,
631 .completed
= srcu_torture_completed
,
632 .deferred_free
= srcu_torture_deferred_free
,
633 .sync
= srcu_torture_synchronize
,
634 .exp_sync
= srcu_torture_synchronize_expedited
,
635 .call
= srcu_torture_call
,
636 .cb_barrier
= srcu_torture_barrier
,
637 .stats
= srcu_torture_stats
,
641 static void srcu_torture_init(void)
643 rcu_sync_torture_init();
644 WARN_ON(init_srcu_struct(&srcu_ctld
));
645 srcu_ctlp
= &srcu_ctld
;
648 static void srcu_torture_cleanup(void)
650 cleanup_srcu_struct(&srcu_ctld
);
651 srcu_ctlp
= &srcu_ctl
; /* In case of a later rcutorture run. */
654 /* As above, but dynamically allocated. */
655 static struct rcu_torture_ops srcud_ops
= {
656 .ttype
= SRCU_FLAVOR
,
657 .init
= srcu_torture_init
,
658 .cleanup
= srcu_torture_cleanup
,
659 .readlock
= srcu_torture_read_lock
,
660 .read_delay
= srcu_read_delay
,
661 .readunlock
= srcu_torture_read_unlock
,
663 .completed
= srcu_torture_completed
,
664 .deferred_free
= srcu_torture_deferred_free
,
665 .sync
= srcu_torture_synchronize
,
666 .exp_sync
= srcu_torture_synchronize_expedited
,
667 .call
= srcu_torture_call
,
668 .cb_barrier
= srcu_torture_barrier
,
669 .stats
= srcu_torture_stats
,
674 * Definitions for sched torture testing.
677 static int sched_torture_read_lock(void)
683 static void sched_torture_read_unlock(int idx
)
688 static void rcu_sched_torture_deferred_free(struct rcu_torture
*p
)
690 call_rcu_sched(&p
->rtort_rcu
, rcu_torture_cb
);
693 static struct rcu_torture_ops sched_ops
= {
694 .ttype
= RCU_SCHED_FLAVOR
,
695 .init
= rcu_sync_torture_init
,
696 .readlock
= sched_torture_read_lock
,
697 .read_delay
= rcu_read_delay
, /* just reuse rcu's version. */
698 .readunlock
= sched_torture_read_unlock
,
699 .started
= rcu_batches_started_sched
,
700 .completed
= rcu_batches_completed_sched
,
701 .deferred_free
= rcu_sched_torture_deferred_free
,
702 .sync
= synchronize_sched
,
703 .exp_sync
= synchronize_sched_expedited
,
704 .get_state
= get_state_synchronize_sched
,
705 .cond_sync
= cond_synchronize_sched
,
706 .call
= call_rcu_sched
,
707 .cb_barrier
= rcu_barrier_sched
,
708 .fqs
= rcu_sched_force_quiescent_state
,
714 #ifdef CONFIG_TASKS_RCU
717 * Definitions for RCU-tasks torture testing.
720 static int tasks_torture_read_lock(void)
725 static void tasks_torture_read_unlock(int idx
)
729 static void rcu_tasks_torture_deferred_free(struct rcu_torture
*p
)
731 call_rcu_tasks(&p
->rtort_rcu
, rcu_torture_cb
);
734 static struct rcu_torture_ops tasks_ops
= {
735 .ttype
= RCU_TASKS_FLAVOR
,
736 .init
= rcu_sync_torture_init
,
737 .readlock
= tasks_torture_read_lock
,
738 .read_delay
= rcu_read_delay
, /* just reuse rcu's version. */
739 .readunlock
= tasks_torture_read_unlock
,
740 .started
= rcu_no_completed
,
741 .completed
= rcu_no_completed
,
742 .deferred_free
= rcu_tasks_torture_deferred_free
,
743 .sync
= synchronize_rcu_tasks
,
744 .exp_sync
= synchronize_rcu_tasks
,
745 .call
= call_rcu_tasks
,
746 .cb_barrier
= rcu_barrier_tasks
,
753 #define RCUTORTURE_TASKS_OPS &tasks_ops,
755 static bool __maybe_unused
torturing_tasks(void)
757 return cur_ops
== &tasks_ops
;
760 #else /* #ifdef CONFIG_TASKS_RCU */
762 #define RCUTORTURE_TASKS_OPS
764 static bool __maybe_unused
torturing_tasks(void)
769 #endif /* #else #ifdef CONFIG_TASKS_RCU */
772 * RCU torture priority-boost testing. Runs one real-time thread per
773 * CPU for moderate bursts, repeatedly registering RCU callbacks and
774 * spinning waiting for them to be invoked. If a given callback takes
775 * too long to be invoked, we assume that priority inversion has occurred.
778 struct rcu_boost_inflight
{
783 static void rcu_torture_boost_cb(struct rcu_head
*head
)
785 struct rcu_boost_inflight
*rbip
=
786 container_of(head
, struct rcu_boost_inflight
, rcu
);
788 /* Ensure RCU-core accesses precede clearing ->inflight */
789 smp_store_release(&rbip
->inflight
, 0);
792 static int rcu_torture_boost(void *arg
)
794 unsigned long call_rcu_time
;
795 unsigned long endtime
;
796 unsigned long oldstarttime
;
797 struct rcu_boost_inflight rbi
= { .inflight
= 0 };
798 struct sched_param sp
;
800 VERBOSE_TOROUT_STRING("rcu_torture_boost started");
802 /* Set real-time priority. */
803 sp
.sched_priority
= 1;
804 if (sched_setscheduler(current
, SCHED_FIFO
, &sp
) < 0) {
805 VERBOSE_TOROUT_STRING("rcu_torture_boost RT prio failed!");
806 n_rcu_torture_boost_rterror
++;
809 init_rcu_head_on_stack(&rbi
.rcu
);
810 /* Each pass through the following loop does one boost-test cycle. */
812 /* Wait for the next test interval. */
813 oldstarttime
= boost_starttime
;
814 while (ULONG_CMP_LT(jiffies
, oldstarttime
)) {
815 schedule_timeout_interruptible(oldstarttime
- jiffies
);
816 stutter_wait("rcu_torture_boost");
817 if (torture_must_stop())
821 /* Do one boost-test interval. */
822 endtime
= oldstarttime
+ test_boost_duration
* HZ
;
823 call_rcu_time
= jiffies
;
824 while (ULONG_CMP_LT(jiffies
, endtime
)) {
825 /* If we don't have a callback in flight, post one. */
826 if (!smp_load_acquire(&rbi
.inflight
)) {
827 /* RCU core before ->inflight = 1. */
828 smp_store_release(&rbi
.inflight
, 1);
829 call_rcu(&rbi
.rcu
, rcu_torture_boost_cb
);
830 if (jiffies
- call_rcu_time
>
831 test_boost_duration
* HZ
- HZ
/ 2) {
832 VERBOSE_TOROUT_STRING("rcu_torture_boost boosting failed");
833 n_rcu_torture_boost_failure
++;
835 call_rcu_time
= jiffies
;
837 stutter_wait("rcu_torture_boost");
838 if (torture_must_stop())
843 * Set the start time of the next test interval.
844 * Yes, this is vulnerable to long delays, but such
845 * delays simply cause a false negative for the next
846 * interval. Besides, we are running at RT priority,
847 * so delays should be relatively rare.
849 while (oldstarttime
== boost_starttime
&&
850 !kthread_should_stop()) {
851 if (mutex_trylock(&boost_mutex
)) {
852 boost_starttime
= jiffies
+
853 test_boost_interval
* HZ
;
854 n_rcu_torture_boosts
++;
855 mutex_unlock(&boost_mutex
);
858 schedule_timeout_uninterruptible(1);
861 /* Go do the stutter. */
862 checkwait
: stutter_wait("rcu_torture_boost");
863 } while (!torture_must_stop());
865 /* Clean up and exit. */
866 while (!kthread_should_stop() || smp_load_acquire(&rbi
.inflight
)) {
867 torture_shutdown_absorb("rcu_torture_boost");
868 schedule_timeout_uninterruptible(1);
870 destroy_rcu_head_on_stack(&rbi
.rcu
);
871 torture_kthread_stopping("rcu_torture_boost");
875 static void rcu_torture_cbflood_cb(struct rcu_head
*rhp
)
880 * RCU torture callback-flood kthread. Repeatedly induces bursts of calls
881 * to call_rcu() or analogous, increasing the probability of occurrence
882 * of callback-overflow corner cases.
885 rcu_torture_cbflood(void *arg
)
890 struct rcu_head
*rhp
;
892 if (cbflood_n_per_burst
> 0 &&
893 cbflood_inter_holdoff
> 0 &&
894 cbflood_intra_holdoff
> 0 &&
896 cur_ops
->cb_barrier
) {
897 rhp
= vmalloc(sizeof(*rhp
) *
898 cbflood_n_burst
* cbflood_n_per_burst
);
902 VERBOSE_TOROUT_STRING("rcu_torture_cbflood disabled: Bad args or OOM");
905 VERBOSE_TOROUT_STRING("rcu_torture_cbflood task started");
907 schedule_timeout_interruptible(cbflood_inter_holdoff
);
908 atomic_long_inc(&n_cbfloods
);
909 WARN_ON(signal_pending(current
));
910 for (i
= 0; i
< cbflood_n_burst
; i
++) {
911 for (j
= 0; j
< cbflood_n_per_burst
; j
++) {
912 cur_ops
->call(&rhp
[i
* cbflood_n_per_burst
+ j
],
913 rcu_torture_cbflood_cb
);
915 schedule_timeout_interruptible(cbflood_intra_holdoff
);
916 WARN_ON(signal_pending(current
));
918 cur_ops
->cb_barrier();
919 stutter_wait("rcu_torture_cbflood");
920 } while (!torture_must_stop());
923 torture_kthread_stopping("rcu_torture_cbflood");
928 * RCU torture force-quiescent-state kthread. Repeatedly induces
929 * bursts of calls to force_quiescent_state(), increasing the probability
930 * of occurrence of some important types of race conditions.
933 rcu_torture_fqs(void *arg
)
935 unsigned long fqs_resume_time
;
936 int fqs_burst_remaining
;
938 VERBOSE_TOROUT_STRING("rcu_torture_fqs task started");
940 fqs_resume_time
= jiffies
+ fqs_stutter
* HZ
;
941 while (ULONG_CMP_LT(jiffies
, fqs_resume_time
) &&
942 !kthread_should_stop()) {
943 schedule_timeout_interruptible(1);
945 fqs_burst_remaining
= fqs_duration
;
946 while (fqs_burst_remaining
> 0 &&
947 !kthread_should_stop()) {
950 fqs_burst_remaining
-= fqs_holdoff
;
952 stutter_wait("rcu_torture_fqs");
953 } while (!torture_must_stop());
954 torture_kthread_stopping("rcu_torture_fqs");
959 * RCU torture writer kthread. Repeatedly substitutes a new structure
960 * for that pointed to by rcu_torture_current, freeing the old structure
961 * after a series of grace periods (the "pipeline").
964 rcu_torture_writer(void *arg
)
966 bool can_expedite
= !rcu_gp_is_expedited() && !rcu_gp_is_normal();
968 unsigned long gp_snap
;
969 bool gp_cond1
= gp_cond
, gp_exp1
= gp_exp
, gp_normal1
= gp_normal
;
970 bool gp_sync1
= gp_sync
;
972 struct rcu_torture
*rp
;
973 struct rcu_torture
*old_rp
;
974 static DEFINE_TORTURE_RANDOM(rand
);
975 int synctype
[] = { RTWS_DEF_FREE
, RTWS_EXP_SYNC
,
976 RTWS_COND_GET
, RTWS_SYNC
};
979 VERBOSE_TOROUT_STRING("rcu_torture_writer task started");
981 pr_alert("%s" TORTURE_FLAG
982 " GP expediting controlled from boot/sysfs for %s,\n",
983 torture_type
, cur_ops
->name
);
984 pr_alert("%s" TORTURE_FLAG
985 " Disabled dynamic grace-period expediting.\n",
989 /* Initialize synctype[] array. If none set, take default. */
990 if (!gp_cond1
&& !gp_exp1
&& !gp_normal1
&& !gp_sync1
)
991 gp_cond1
= gp_exp1
= gp_normal1
= gp_sync1
= true;
992 if (gp_cond1
&& cur_ops
->get_state
&& cur_ops
->cond_sync
)
993 synctype
[nsynctypes
++] = RTWS_COND_GET
;
994 else if (gp_cond
&& (!cur_ops
->get_state
|| !cur_ops
->cond_sync
))
995 pr_alert("rcu_torture_writer: gp_cond without primitives.\n");
996 if (gp_exp1
&& cur_ops
->exp_sync
)
997 synctype
[nsynctypes
++] = RTWS_EXP_SYNC
;
998 else if (gp_exp
&& !cur_ops
->exp_sync
)
999 pr_alert("rcu_torture_writer: gp_exp without primitives.\n");
1000 if (gp_normal1
&& cur_ops
->deferred_free
)
1001 synctype
[nsynctypes
++] = RTWS_DEF_FREE
;
1002 else if (gp_normal
&& !cur_ops
->deferred_free
)
1003 pr_alert("rcu_torture_writer: gp_normal without primitives.\n");
1004 if (gp_sync1
&& cur_ops
->sync
)
1005 synctype
[nsynctypes
++] = RTWS_SYNC
;
1006 else if (gp_sync
&& !cur_ops
->sync
)
1007 pr_alert("rcu_torture_writer: gp_sync without primitives.\n");
1008 if (WARN_ONCE(nsynctypes
== 0,
1009 "rcu_torture_writer: No update-side primitives.\n")) {
1011 * No updates primitives, so don't try updating.
1012 * The resulting test won't be testing much, hence the
1013 * above WARN_ONCE().
1015 rcu_torture_writer_state
= RTWS_STOPPING
;
1016 torture_kthread_stopping("rcu_torture_writer");
1020 rcu_torture_writer_state
= RTWS_FIXED_DELAY
;
1021 schedule_timeout_uninterruptible(1);
1022 rp
= rcu_torture_alloc();
1025 rp
->rtort_pipe_count
= 0;
1026 rcu_torture_writer_state
= RTWS_DELAY
;
1027 udelay(torture_random(&rand
) & 0x3ff);
1028 rcu_torture_writer_state
= RTWS_REPLACE
;
1029 old_rp
= rcu_dereference_check(rcu_torture_current
,
1030 current
== writer_task
);
1031 rp
->rtort_mbtest
= 1;
1032 rcu_assign_pointer(rcu_torture_current
, rp
);
1033 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
1035 i
= old_rp
->rtort_pipe_count
;
1036 if (i
> RCU_TORTURE_PIPE_LEN
)
1037 i
= RCU_TORTURE_PIPE_LEN
;
1038 atomic_inc(&rcu_torture_wcount
[i
]);
1039 old_rp
->rtort_pipe_count
++;
1040 switch (synctype
[torture_random(&rand
) % nsynctypes
]) {
1042 rcu_torture_writer_state
= RTWS_DEF_FREE
;
1043 cur_ops
->deferred_free(old_rp
);
1046 rcu_torture_writer_state
= RTWS_EXP_SYNC
;
1047 cur_ops
->exp_sync();
1048 rcu_torture_pipe_update(old_rp
);
1051 rcu_torture_writer_state
= RTWS_COND_GET
;
1052 gp_snap
= cur_ops
->get_state();
1053 i
= torture_random(&rand
) % 16;
1055 schedule_timeout_interruptible(i
);
1056 udelay(torture_random(&rand
) % 1000);
1057 rcu_torture_writer_state
= RTWS_COND_SYNC
;
1058 cur_ops
->cond_sync(gp_snap
);
1059 rcu_torture_pipe_update(old_rp
);
1062 rcu_torture_writer_state
= RTWS_SYNC
;
1064 rcu_torture_pipe_update(old_rp
);
1071 rcutorture_record_progress(++rcu_torture_current_version
);
1072 /* Cycle through nesting levels of rcu_expedite_gp() calls. */
1074 !(torture_random(&rand
) & 0xff & (!!expediting
- 1))) {
1075 WARN_ON_ONCE(expediting
== 0 && rcu_gp_is_expedited());
1076 if (expediting
>= 0)
1079 rcu_unexpedite_gp();
1080 if (++expediting
> 3)
1081 expediting
= -expediting
;
1083 rcu_torture_writer_state
= RTWS_STUTTER
;
1084 stutter_wait("rcu_torture_writer");
1085 } while (!torture_must_stop());
1086 /* Reset expediting back to unexpedited. */
1088 expediting
= -expediting
;
1089 while (can_expedite
&& expediting
++ < 0)
1090 rcu_unexpedite_gp();
1091 WARN_ON_ONCE(can_expedite
&& rcu_gp_is_expedited());
1092 rcu_torture_writer_state
= RTWS_STOPPING
;
1093 torture_kthread_stopping("rcu_torture_writer");
1098 * RCU torture fake writer kthread. Repeatedly calls sync, with a random
1099 * delay between calls.
1102 rcu_torture_fakewriter(void *arg
)
1104 DEFINE_TORTURE_RANDOM(rand
);
1106 VERBOSE_TOROUT_STRING("rcu_torture_fakewriter task started");
1107 set_user_nice(current
, MAX_NICE
);
1110 schedule_timeout_uninterruptible(1 + torture_random(&rand
)%10);
1111 udelay(torture_random(&rand
) & 0x3ff);
1112 if (cur_ops
->cb_barrier
!= NULL
&&
1113 torture_random(&rand
) % (nfakewriters
* 8) == 0) {
1114 cur_ops
->cb_barrier();
1115 } else if (gp_normal
== gp_exp
) {
1116 if (torture_random(&rand
) & 0x80)
1119 cur_ops
->exp_sync();
1120 } else if (gp_normal
) {
1123 cur_ops
->exp_sync();
1125 stutter_wait("rcu_torture_fakewriter");
1126 } while (!torture_must_stop());
1128 torture_kthread_stopping("rcu_torture_fakewriter");
1133 * RCU torture reader from timer handler. Dereferences rcu_torture_current,
1134 * incrementing the corresponding element of the pipeline array. The
1135 * counter in the element should never be greater than 1, otherwise, the
1136 * RCU implementation is broken.
1138 static void rcu_torture_timer(unsigned long unused
)
1141 unsigned long started
;
1142 unsigned long completed
;
1143 static DEFINE_TORTURE_RANDOM(rand
);
1144 static DEFINE_SPINLOCK(rand_lock
);
1145 struct rcu_torture
*p
;
1147 unsigned long long ts
;
1149 idx
= cur_ops
->readlock();
1150 if (cur_ops
->started
)
1151 started
= cur_ops
->started();
1153 started
= cur_ops
->completed();
1154 ts
= rcu_trace_clock_local();
1155 p
= rcu_dereference_check(rcu_torture_current
,
1156 rcu_read_lock_bh_held() ||
1157 rcu_read_lock_sched_held() ||
1158 srcu_read_lock_held(srcu_ctlp
) ||
1161 /* Leave because rcu_torture_writer is not yet underway */
1162 cur_ops
->readunlock(idx
);
1165 if (p
->rtort_mbtest
== 0)
1166 atomic_inc(&n_rcu_torture_mberror
);
1167 spin_lock(&rand_lock
);
1168 cur_ops
->read_delay(&rand
);
1169 n_rcu_torture_timers
++;
1170 spin_unlock(&rand_lock
);
1172 pipe_count
= p
->rtort_pipe_count
;
1173 if (pipe_count
> RCU_TORTURE_PIPE_LEN
) {
1174 /* Should not happen, but... */
1175 pipe_count
= RCU_TORTURE_PIPE_LEN
;
1177 completed
= cur_ops
->completed();
1178 if (pipe_count
> 1) {
1179 do_trace_rcu_torture_read(cur_ops
->name
, &p
->rtort_rcu
, ts
,
1180 started
, completed
);
1181 rcu_ftrace_dump(DUMP_ALL
);
1183 __this_cpu_inc(rcu_torture_count
[pipe_count
]);
1184 completed
= completed
- started
;
1185 if (cur_ops
->started
)
1187 if (completed
> RCU_TORTURE_PIPE_LEN
) {
1188 /* Should not happen, but... */
1189 completed
= RCU_TORTURE_PIPE_LEN
;
1191 __this_cpu_inc(rcu_torture_batch
[completed
]);
1193 cur_ops
->readunlock(idx
);
1197 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current,
1198 * incrementing the corresponding element of the pipeline array. The
1199 * counter in the element should never be greater than 1, otherwise, the
1200 * RCU implementation is broken.
1203 rcu_torture_reader(void *arg
)
1205 unsigned long started
;
1206 unsigned long completed
;
1208 DEFINE_TORTURE_RANDOM(rand
);
1209 struct rcu_torture
*p
;
1211 struct timer_list t
;
1212 unsigned long long ts
;
1214 VERBOSE_TOROUT_STRING("rcu_torture_reader task started");
1215 set_user_nice(current
, MAX_NICE
);
1216 if (irqreader
&& cur_ops
->irq_capable
)
1217 setup_timer_on_stack(&t
, rcu_torture_timer
, 0);
1220 if (irqreader
&& cur_ops
->irq_capable
) {
1221 if (!timer_pending(&t
))
1222 mod_timer(&t
, jiffies
+ 1);
1224 idx
= cur_ops
->readlock();
1225 if (cur_ops
->started
)
1226 started
= cur_ops
->started();
1228 started
= cur_ops
->completed();
1229 ts
= rcu_trace_clock_local();
1230 p
= rcu_dereference_check(rcu_torture_current
,
1231 rcu_read_lock_bh_held() ||
1232 rcu_read_lock_sched_held() ||
1233 srcu_read_lock_held(srcu_ctlp
) ||
1236 /* Wait for rcu_torture_writer to get underway */
1237 cur_ops
->readunlock(idx
);
1238 schedule_timeout_interruptible(HZ
);
1241 if (p
->rtort_mbtest
== 0)
1242 atomic_inc(&n_rcu_torture_mberror
);
1243 cur_ops
->read_delay(&rand
);
1245 pipe_count
= p
->rtort_pipe_count
;
1246 if (pipe_count
> RCU_TORTURE_PIPE_LEN
) {
1247 /* Should not happen, but... */
1248 pipe_count
= RCU_TORTURE_PIPE_LEN
;
1250 completed
= cur_ops
->completed();
1251 if (pipe_count
> 1) {
1252 do_trace_rcu_torture_read(cur_ops
->name
, &p
->rtort_rcu
,
1253 ts
, started
, completed
);
1254 rcu_ftrace_dump(DUMP_ALL
);
1256 __this_cpu_inc(rcu_torture_count
[pipe_count
]);
1257 completed
= completed
- started
;
1258 if (cur_ops
->started
)
1260 if (completed
> RCU_TORTURE_PIPE_LEN
) {
1261 /* Should not happen, but... */
1262 completed
= RCU_TORTURE_PIPE_LEN
;
1264 __this_cpu_inc(rcu_torture_batch
[completed
]);
1266 cur_ops
->readunlock(idx
);
1267 stutter_wait("rcu_torture_reader");
1268 } while (!torture_must_stop());
1269 if (irqreader
&& cur_ops
->irq_capable
) {
1271 destroy_timer_on_stack(&t
);
1273 torture_kthread_stopping("rcu_torture_reader");
1278 * Print torture statistics. Caller must ensure that there is only
1279 * one call to this function at a given time!!! This is normally
1280 * accomplished by relying on the module system to only have one copy
1281 * of the module loaded, and then by giving the rcu_torture_stats
1282 * kthread full control (or the init/cleanup functions when rcu_torture_stats
1283 * thread is not running).
1286 rcu_torture_stats_print(void)
1290 long pipesummary
[RCU_TORTURE_PIPE_LEN
+ 1] = { 0 };
1291 long batchsummary
[RCU_TORTURE_PIPE_LEN
+ 1] = { 0 };
1292 static unsigned long rtcv_snap
= ULONG_MAX
;
1293 struct task_struct
*wtp
;
1295 for_each_possible_cpu(cpu
) {
1296 for (i
= 0; i
< RCU_TORTURE_PIPE_LEN
+ 1; i
++) {
1297 pipesummary
[i
] += per_cpu(rcu_torture_count
, cpu
)[i
];
1298 batchsummary
[i
] += per_cpu(rcu_torture_batch
, cpu
)[i
];
1301 for (i
= RCU_TORTURE_PIPE_LEN
- 1; i
>= 0; i
--) {
1302 if (pipesummary
[i
] != 0)
1306 pr_alert("%s%s ", torture_type
, TORTURE_FLAG
);
1307 pr_cont("rtc: %p ver: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",
1308 rcu_torture_current
,
1309 rcu_torture_current_version
,
1310 list_empty(&rcu_torture_freelist
),
1311 atomic_read(&n_rcu_torture_alloc
),
1312 atomic_read(&n_rcu_torture_alloc_fail
),
1313 atomic_read(&n_rcu_torture_free
));
1314 pr_cont("rtmbe: %d rtbe: %ld rtbke: %ld rtbre: %ld ",
1315 atomic_read(&n_rcu_torture_mberror
),
1316 n_rcu_torture_barrier_error
,
1317 n_rcu_torture_boost_ktrerror
,
1318 n_rcu_torture_boost_rterror
);
1319 pr_cont("rtbf: %ld rtb: %ld nt: %ld ",
1320 n_rcu_torture_boost_failure
,
1321 n_rcu_torture_boosts
,
1322 n_rcu_torture_timers
);
1323 torture_onoff_stats();
1324 pr_cont("barrier: %ld/%ld:%ld ",
1325 n_barrier_successes
,
1327 n_rcu_torture_barrier_error
);
1328 pr_cont("cbflood: %ld\n", atomic_long_read(&n_cbfloods
));
1330 pr_alert("%s%s ", torture_type
, TORTURE_FLAG
);
1331 if (atomic_read(&n_rcu_torture_mberror
) != 0 ||
1332 n_rcu_torture_barrier_error
!= 0 ||
1333 n_rcu_torture_boost_ktrerror
!= 0 ||
1334 n_rcu_torture_boost_rterror
!= 0 ||
1335 n_rcu_torture_boost_failure
!= 0 ||
1337 pr_cont("%s", "!!! ");
1338 atomic_inc(&n_rcu_torture_error
);
1341 pr_cont("Reader Pipe: ");
1342 for (i
= 0; i
< RCU_TORTURE_PIPE_LEN
+ 1; i
++)
1343 pr_cont(" %ld", pipesummary
[i
]);
1346 pr_alert("%s%s ", torture_type
, TORTURE_FLAG
);
1347 pr_cont("Reader Batch: ");
1348 for (i
= 0; i
< RCU_TORTURE_PIPE_LEN
+ 1; i
++)
1349 pr_cont(" %ld", batchsummary
[i
]);
1352 pr_alert("%s%s ", torture_type
, TORTURE_FLAG
);
1353 pr_cont("Free-Block Circulation: ");
1354 for (i
= 0; i
< RCU_TORTURE_PIPE_LEN
+ 1; i
++) {
1355 pr_cont(" %d", atomic_read(&rcu_torture_wcount
[i
]));
1361 if (rtcv_snap
== rcu_torture_current_version
&&
1362 rcu_torture_current
!= NULL
) {
1363 int __maybe_unused flags
= 0;
1364 unsigned long __maybe_unused gpnum
= 0;
1365 unsigned long __maybe_unused completed
= 0;
1367 rcutorture_get_gp_data(cur_ops
->ttype
,
1368 &flags
, &gpnum
, &completed
);
1369 srcutorture_get_gp_data(cur_ops
->ttype
, srcu_ctlp
,
1370 &flags
, &gpnum
, &completed
);
1371 wtp
= READ_ONCE(writer_task
);
1372 pr_alert("??? Writer stall state %s(%d) g%lu c%lu f%#x ->state %#lx\n",
1373 rcu_torture_writer_state_getname(),
1374 rcu_torture_writer_state
,
1375 gpnum
, completed
, flags
,
1376 wtp
== NULL
? ~0UL : wtp
->state
);
1377 show_rcu_gp_kthreads();
1378 rcu_ftrace_dump(DUMP_ALL
);
1380 rtcv_snap
= rcu_torture_current_version
;
1384 * Periodically prints torture statistics, if periodic statistics printing
1385 * was specified via the stat_interval module parameter.
1388 rcu_torture_stats(void *arg
)
1390 VERBOSE_TOROUT_STRING("rcu_torture_stats task started");
1392 schedule_timeout_interruptible(stat_interval
* HZ
);
1393 rcu_torture_stats_print();
1394 torture_shutdown_absorb("rcu_torture_stats");
1395 } while (!torture_must_stop());
1396 torture_kthread_stopping("rcu_torture_stats");
1401 rcu_torture_print_module_parms(struct rcu_torture_ops
*cur_ops
, const char *tag
)
1403 pr_alert("%s" TORTURE_FLAG
1404 "--- %s: nreaders=%d nfakewriters=%d "
1405 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
1406 "shuffle_interval=%d stutter=%d irqreader=%d "
1407 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
1408 "test_boost=%d/%d test_boost_interval=%d "
1409 "test_boost_duration=%d shutdown_secs=%d "
1410 "stall_cpu=%d stall_cpu_holdoff=%d "
1412 "onoff_interval=%d onoff_holdoff=%d\n",
1413 torture_type
, tag
, nrealreaders
, nfakewriters
,
1414 stat_interval
, verbose
, test_no_idle_hz
, shuffle_interval
,
1415 stutter
, irqreader
, fqs_duration
, fqs_holdoff
, fqs_stutter
,
1416 test_boost
, cur_ops
->can_boost
,
1417 test_boost_interval
, test_boost_duration
, shutdown_secs
,
1418 stall_cpu
, stall_cpu_holdoff
,
1420 onoff_interval
, onoff_holdoff
);
1423 static int rcutorture_booster_cleanup(unsigned int cpu
)
1425 struct task_struct
*t
;
1427 if (boost_tasks
[cpu
] == NULL
)
1429 mutex_lock(&boost_mutex
);
1430 t
= boost_tasks
[cpu
];
1431 boost_tasks
[cpu
] = NULL
;
1432 mutex_unlock(&boost_mutex
);
1434 /* This must be outside of the mutex, otherwise deadlock! */
1435 torture_stop_kthread(rcu_torture_boost
, t
);
1439 static int rcutorture_booster_init(unsigned int cpu
)
1443 if (boost_tasks
[cpu
] != NULL
)
1444 return 0; /* Already created, nothing more to do. */
1446 /* Don't allow time recalculation while creating a new task. */
1447 mutex_lock(&boost_mutex
);
1448 VERBOSE_TOROUT_STRING("Creating rcu_torture_boost task");
1449 boost_tasks
[cpu
] = kthread_create_on_node(rcu_torture_boost
, NULL
,
1451 "rcu_torture_boost");
1452 if (IS_ERR(boost_tasks
[cpu
])) {
1453 retval
= PTR_ERR(boost_tasks
[cpu
]);
1454 VERBOSE_TOROUT_STRING("rcu_torture_boost task create failed");
1455 n_rcu_torture_boost_ktrerror
++;
1456 boost_tasks
[cpu
] = NULL
;
1457 mutex_unlock(&boost_mutex
);
1460 kthread_bind(boost_tasks
[cpu
], cpu
);
1461 wake_up_process(boost_tasks
[cpu
]);
1462 mutex_unlock(&boost_mutex
);
1467 * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then
1468 * induces a CPU stall for the time specified by stall_cpu.
1470 static int rcu_torture_stall(void *args
)
1472 unsigned long stop_at
;
1474 VERBOSE_TOROUT_STRING("rcu_torture_stall task started");
1475 if (stall_cpu_holdoff
> 0) {
1476 VERBOSE_TOROUT_STRING("rcu_torture_stall begin holdoff");
1477 schedule_timeout_interruptible(stall_cpu_holdoff
* HZ
);
1478 VERBOSE_TOROUT_STRING("rcu_torture_stall end holdoff");
1480 if (!kthread_should_stop()) {
1481 stop_at
= get_seconds() + stall_cpu
;
1482 /* RCU CPU stall is expected behavior in following code. */
1483 pr_alert("rcu_torture_stall start.\n");
1486 while (ULONG_CMP_LT(get_seconds(), stop_at
))
1487 continue; /* Induce RCU CPU stall warning. */
1490 pr_alert("rcu_torture_stall end.\n");
1492 torture_shutdown_absorb("rcu_torture_stall");
1493 while (!kthread_should_stop())
1494 schedule_timeout_interruptible(10 * HZ
);
1498 /* Spawn CPU-stall kthread, if stall_cpu specified. */
1499 static int __init
rcu_torture_stall_init(void)
1503 return torture_create_kthread(rcu_torture_stall
, NULL
, stall_task
);
1506 /* Callback function for RCU barrier testing. */
1507 static void rcu_torture_barrier_cbf(struct rcu_head
*rcu
)
1509 atomic_inc(&barrier_cbs_invoked
);
1512 /* kthread function to register callbacks used to test RCU barriers. */
1513 static int rcu_torture_barrier_cbs(void *arg
)
1515 long myid
= (long)arg
;
1518 struct rcu_head rcu
;
1520 init_rcu_head_on_stack(&rcu
);
1521 VERBOSE_TOROUT_STRING("rcu_torture_barrier_cbs task started");
1522 set_user_nice(current
, MAX_NICE
);
1524 wait_event(barrier_cbs_wq
[myid
],
1526 smp_load_acquire(&barrier_phase
)) != lastphase
||
1527 torture_must_stop());
1528 lastphase
= newphase
;
1529 if (torture_must_stop())
1532 * The above smp_load_acquire() ensures barrier_phase load
1533 * is ordered before the following ->call().
1535 local_irq_disable(); /* Just to test no-irq call_rcu(). */
1536 cur_ops
->call(&rcu
, rcu_torture_barrier_cbf
);
1538 if (atomic_dec_and_test(&barrier_cbs_count
))
1539 wake_up(&barrier_wq
);
1540 } while (!torture_must_stop());
1541 if (cur_ops
->cb_barrier
!= NULL
)
1542 cur_ops
->cb_barrier();
1543 destroy_rcu_head_on_stack(&rcu
);
1544 torture_kthread_stopping("rcu_torture_barrier_cbs");
1548 /* kthread function to drive and coordinate RCU barrier testing. */
1549 static int rcu_torture_barrier(void *arg
)
1553 VERBOSE_TOROUT_STRING("rcu_torture_barrier task starting");
1555 atomic_set(&barrier_cbs_invoked
, 0);
1556 atomic_set(&barrier_cbs_count
, n_barrier_cbs
);
1557 /* Ensure barrier_phase ordered after prior assignments. */
1558 smp_store_release(&barrier_phase
, !barrier_phase
);
1559 for (i
= 0; i
< n_barrier_cbs
; i
++)
1560 wake_up(&barrier_cbs_wq
[i
]);
1561 wait_event(barrier_wq
,
1562 atomic_read(&barrier_cbs_count
) == 0 ||
1563 torture_must_stop());
1564 if (torture_must_stop())
1566 n_barrier_attempts
++;
1567 cur_ops
->cb_barrier(); /* Implies smp_mb() for wait_event(). */
1568 if (atomic_read(&barrier_cbs_invoked
) != n_barrier_cbs
) {
1569 n_rcu_torture_barrier_error
++;
1570 pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n",
1571 atomic_read(&barrier_cbs_invoked
),
1575 n_barrier_successes
++;
1576 schedule_timeout_interruptible(HZ
/ 10);
1577 } while (!torture_must_stop());
1578 torture_kthread_stopping("rcu_torture_barrier");
1582 /* Initialize RCU barrier testing. */
1583 static int rcu_torture_barrier_init(void)
1588 if (n_barrier_cbs
<= 0)
1590 if (cur_ops
->call
== NULL
|| cur_ops
->cb_barrier
== NULL
) {
1591 pr_alert("%s" TORTURE_FLAG
1592 " Call or barrier ops missing for %s,\n",
1593 torture_type
, cur_ops
->name
);
1594 pr_alert("%s" TORTURE_FLAG
1595 " RCU barrier testing omitted from run.\n",
1599 atomic_set(&barrier_cbs_count
, 0);
1600 atomic_set(&barrier_cbs_invoked
, 0);
1602 kzalloc(n_barrier_cbs
* sizeof(barrier_cbs_tasks
[0]),
1605 kzalloc(n_barrier_cbs
* sizeof(barrier_cbs_wq
[0]),
1607 if (barrier_cbs_tasks
== NULL
|| !barrier_cbs_wq
)
1609 for (i
= 0; i
< n_barrier_cbs
; i
++) {
1610 init_waitqueue_head(&barrier_cbs_wq
[i
]);
1611 ret
= torture_create_kthread(rcu_torture_barrier_cbs
,
1613 barrier_cbs_tasks
[i
]);
1617 return torture_create_kthread(rcu_torture_barrier
, NULL
, barrier_task
);
1620 /* Clean up after RCU barrier testing. */
1621 static void rcu_torture_barrier_cleanup(void)
1625 torture_stop_kthread(rcu_torture_barrier
, barrier_task
);
1626 if (barrier_cbs_tasks
!= NULL
) {
1627 for (i
= 0; i
< n_barrier_cbs
; i
++)
1628 torture_stop_kthread(rcu_torture_barrier_cbs
,
1629 barrier_cbs_tasks
[i
]);
1630 kfree(barrier_cbs_tasks
);
1631 barrier_cbs_tasks
= NULL
;
1633 if (barrier_cbs_wq
!= NULL
) {
1634 kfree(barrier_cbs_wq
);
1635 barrier_cbs_wq
= NULL
;
1639 static enum cpuhp_state rcutor_hp
;
1642 rcu_torture_cleanup(void)
1646 rcutorture_record_test_transition();
1647 if (torture_cleanup_begin()) {
1648 if (cur_ops
->cb_barrier
!= NULL
)
1649 cur_ops
->cb_barrier();
1653 rcu_torture_barrier_cleanup();
1654 torture_stop_kthread(rcu_torture_stall
, stall_task
);
1655 torture_stop_kthread(rcu_torture_writer
, writer_task
);
1658 for (i
= 0; i
< nrealreaders
; i
++)
1659 torture_stop_kthread(rcu_torture_reader
,
1661 kfree(reader_tasks
);
1663 rcu_torture_current
= NULL
;
1665 if (fakewriter_tasks
) {
1666 for (i
= 0; i
< nfakewriters
; i
++) {
1667 torture_stop_kthread(rcu_torture_fakewriter
,
1668 fakewriter_tasks
[i
]);
1670 kfree(fakewriter_tasks
);
1671 fakewriter_tasks
= NULL
;
1674 torture_stop_kthread(rcu_torture_stats
, stats_task
);
1675 torture_stop_kthread(rcu_torture_fqs
, fqs_task
);
1676 for (i
= 0; i
< ncbflooders
; i
++)
1677 torture_stop_kthread(rcu_torture_cbflood
, cbflood_task
[i
]);
1678 if ((test_boost
== 1 && cur_ops
->can_boost
) ||
1680 cpuhp_remove_state(rcutor_hp
);
1683 * Wait for all RCU callbacks to fire, then do flavor-specific
1684 * cleanup operations.
1686 if (cur_ops
->cb_barrier
!= NULL
)
1687 cur_ops
->cb_barrier();
1688 if (cur_ops
->cleanup
!= NULL
)
1691 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */
1693 if (atomic_read(&n_rcu_torture_error
) || n_rcu_torture_barrier_error
)
1694 rcu_torture_print_module_parms(cur_ops
, "End of test: FAILURE");
1695 else if (torture_onoff_failures())
1696 rcu_torture_print_module_parms(cur_ops
,
1697 "End of test: RCU_HOTPLUG");
1699 rcu_torture_print_module_parms(cur_ops
, "End of test: SUCCESS");
1700 torture_cleanup_end();
1703 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
1704 static void rcu_torture_leak_cb(struct rcu_head
*rhp
)
1708 static void rcu_torture_err_cb(struct rcu_head
*rhp
)
1711 * This -might- happen due to race conditions, but is unlikely.
1712 * The scenario that leads to this happening is that the
1713 * first of the pair of duplicate callbacks is queued,
1714 * someone else starts a grace period that includes that
1715 * callback, then the second of the pair must wait for the
1716 * next grace period. Unlikely, but can happen. If it
1717 * does happen, the debug-objects subsystem won't have splatted.
1719 pr_alert("rcutorture: duplicated callback was invoked.\n");
1721 #endif /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
1724 * Verify that double-free causes debug-objects to complain, but only
1725 * if CONFIG_DEBUG_OBJECTS_RCU_HEAD=y. Otherwise, say that the test
1726 * cannot be carried out.
1728 static void rcu_test_debug_objects(void)
1730 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
1731 struct rcu_head rh1
;
1732 struct rcu_head rh2
;
1734 init_rcu_head_on_stack(&rh1
);
1735 init_rcu_head_on_stack(&rh2
);
1736 pr_alert("rcutorture: WARN: Duplicate call_rcu() test starting.\n");
1738 /* Try to queue the rh2 pair of callbacks for the same grace period. */
1739 preempt_disable(); /* Prevent preemption from interrupting test. */
1740 rcu_read_lock(); /* Make it impossible to finish a grace period. */
1741 call_rcu(&rh1
, rcu_torture_leak_cb
); /* Start grace period. */
1742 local_irq_disable(); /* Make it harder to start a new grace period. */
1743 call_rcu(&rh2
, rcu_torture_leak_cb
);
1744 call_rcu(&rh2
, rcu_torture_err_cb
); /* Duplicate callback. */
1749 /* Wait for them all to get done so we can safely return. */
1751 pr_alert("rcutorture: WARN: Duplicate call_rcu() test complete.\n");
1752 destroy_rcu_head_on_stack(&rh1
);
1753 destroy_rcu_head_on_stack(&rh2
);
1754 #else /* #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
1755 pr_alert("rcutorture: !CONFIG_DEBUG_OBJECTS_RCU_HEAD, not testing duplicate call_rcu()\n");
1756 #endif /* #else #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD */
1760 rcu_torture_init(void)
1765 static struct rcu_torture_ops
*torture_ops
[] = {
1766 &rcu_ops
, &rcu_bh_ops
, &rcu_busted_ops
, &srcu_ops
, &srcud_ops
,
1767 &sched_ops
, RCUTORTURE_TASKS_OPS
1770 if (!torture_init_begin(torture_type
, verbose
, &torture_runnable
))
1773 /* Process args and tell the world that the torturer is on the job. */
1774 for (i
= 0; i
< ARRAY_SIZE(torture_ops
); i
++) {
1775 cur_ops
= torture_ops
[i
];
1776 if (strcmp(torture_type
, cur_ops
->name
) == 0)
1779 if (i
== ARRAY_SIZE(torture_ops
)) {
1780 pr_alert("rcu-torture: invalid torture type: \"%s\"\n",
1782 pr_alert("rcu-torture types:");
1783 for (i
= 0; i
< ARRAY_SIZE(torture_ops
); i
++)
1784 pr_alert(" %s", torture_ops
[i
]->name
);
1789 if (cur_ops
->fqs
== NULL
&& fqs_duration
!= 0) {
1790 pr_alert("rcu-torture: ->fqs NULL and non-zero fqs_duration, fqs disabled.\n");
1796 if (nreaders
>= 0) {
1797 nrealreaders
= nreaders
;
1799 nrealreaders
= num_online_cpus() - 2 - nreaders
;
1800 if (nrealreaders
<= 0)
1803 rcu_torture_print_module_parms(cur_ops
, "Start of test");
1805 /* Set up the freelist. */
1807 INIT_LIST_HEAD(&rcu_torture_freelist
);
1808 for (i
= 0; i
< ARRAY_SIZE(rcu_tortures
); i
++) {
1809 rcu_tortures
[i
].rtort_mbtest
= 0;
1810 list_add_tail(&rcu_tortures
[i
].rtort_free
,
1811 &rcu_torture_freelist
);
1814 /* Initialize the statistics so that each run gets its own numbers. */
1816 rcu_torture_current
= NULL
;
1817 rcu_torture_current_version
= 0;
1818 atomic_set(&n_rcu_torture_alloc
, 0);
1819 atomic_set(&n_rcu_torture_alloc_fail
, 0);
1820 atomic_set(&n_rcu_torture_free
, 0);
1821 atomic_set(&n_rcu_torture_mberror
, 0);
1822 atomic_set(&n_rcu_torture_error
, 0);
1823 n_rcu_torture_barrier_error
= 0;
1824 n_rcu_torture_boost_ktrerror
= 0;
1825 n_rcu_torture_boost_rterror
= 0;
1826 n_rcu_torture_boost_failure
= 0;
1827 n_rcu_torture_boosts
= 0;
1828 for (i
= 0; i
< RCU_TORTURE_PIPE_LEN
+ 1; i
++)
1829 atomic_set(&rcu_torture_wcount
[i
], 0);
1830 for_each_possible_cpu(cpu
) {
1831 for (i
= 0; i
< RCU_TORTURE_PIPE_LEN
+ 1; i
++) {
1832 per_cpu(rcu_torture_count
, cpu
)[i
] = 0;
1833 per_cpu(rcu_torture_batch
, cpu
)[i
] = 0;
1837 /* Start up the kthreads. */
1839 firsterr
= torture_create_kthread(rcu_torture_writer
, NULL
,
1843 if (nfakewriters
> 0) {
1844 fakewriter_tasks
= kzalloc(nfakewriters
*
1845 sizeof(fakewriter_tasks
[0]),
1847 if (fakewriter_tasks
== NULL
) {
1848 VERBOSE_TOROUT_ERRSTRING("out of memory");
1853 for (i
= 0; i
< nfakewriters
; i
++) {
1854 firsterr
= torture_create_kthread(rcu_torture_fakewriter
,
1855 NULL
, fakewriter_tasks
[i
]);
1859 reader_tasks
= kzalloc(nrealreaders
* sizeof(reader_tasks
[0]),
1861 if (reader_tasks
== NULL
) {
1862 VERBOSE_TOROUT_ERRSTRING("out of memory");
1866 for (i
= 0; i
< nrealreaders
; i
++) {
1867 firsterr
= torture_create_kthread(rcu_torture_reader
, NULL
,
1872 if (stat_interval
> 0) {
1873 firsterr
= torture_create_kthread(rcu_torture_stats
, NULL
,
1878 if (test_no_idle_hz
&& shuffle_interval
> 0) {
1879 firsterr
= torture_shuffle_init(shuffle_interval
* HZ
);
1886 firsterr
= torture_stutter_init(stutter
* HZ
);
1890 if (fqs_duration
< 0)
1893 /* Create the fqs thread */
1894 firsterr
= torture_create_kthread(rcu_torture_fqs
, NULL
,
1899 if (test_boost_interval
< 1)
1900 test_boost_interval
= 1;
1901 if (test_boost_duration
< 2)
1902 test_boost_duration
= 2;
1903 if ((test_boost
== 1 && cur_ops
->can_boost
) ||
1906 boost_starttime
= jiffies
+ test_boost_interval
* HZ
;
1908 firsterr
= cpuhp_setup_state(CPUHP_AP_ONLINE_DYN
, "RCU_TORTURE",
1909 rcutorture_booster_init
,
1910 rcutorture_booster_cleanup
);
1913 rcutor_hp
= firsterr
;
1915 firsterr
= torture_shutdown_init(shutdown_secs
, rcu_torture_cleanup
);
1918 firsterr
= torture_onoff_init(onoff_holdoff
* HZ
, onoff_interval
* HZ
);
1921 firsterr
= rcu_torture_stall_init();
1924 firsterr
= rcu_torture_barrier_init();
1928 rcu_test_debug_objects();
1929 if (cbflood_n_burst
> 0) {
1930 /* Create the cbflood threads */
1931 ncbflooders
= (num_online_cpus() + 3) / 4;
1932 cbflood_task
= kcalloc(ncbflooders
, sizeof(*cbflood_task
),
1934 if (!cbflood_task
) {
1935 VERBOSE_TOROUT_ERRSTRING("out of memory");
1939 for (i
= 0; i
< ncbflooders
; i
++) {
1940 firsterr
= torture_create_kthread(rcu_torture_cbflood
,
1947 rcutorture_record_test_transition();
1953 rcu_torture_cleanup();
1957 module_init(rcu_torture_init
);
1958 module_exit(rcu_torture_cleanup
);