]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - kernel/rcutorture.c
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs-2.6
[mirror_ubuntu-artful-kernel.git] / kernel / rcutorture.c
CommitLineData
a241ec65 1/*
29766f1e 2 * Read-Copy Update module-based torture test facility
a241ec65
PM
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
b772e1dd 18 * Copyright (C) IBM Corporation, 2005, 2006
a241ec65
PM
19 *
20 * Authors: Paul E. McKenney <paulmck@us.ibm.com>
a71fca58 21 * Josh Triplett <josh@freedesktop.org>
a241ec65
PM
22 *
23 * See also: Documentation/RCU/torture.txt
24 */
25#include <linux/types.h>
26#include <linux/kernel.h>
27#include <linux/init.h>
28#include <linux/module.h>
29#include <linux/kthread.h>
30#include <linux/err.h>
31#include <linux/spinlock.h>
32#include <linux/smp.h>
33#include <linux/rcupdate.h>
34#include <linux/interrupt.h>
35#include <linux/sched.h>
36#include <asm/atomic.h>
37#include <linux/bitops.h>
a241ec65
PM
38#include <linux/completion.h>
39#include <linux/moduleparam.h>
40#include <linux/percpu.h>
41#include <linux/notifier.h>
343e9099 42#include <linux/reboot.h>
83144186 43#include <linux/freezer.h>
a241ec65 44#include <linux/cpu.h>
a241ec65 45#include <linux/delay.h>
a241ec65 46#include <linux/stat.h>
b2896d2e 47#include <linux/srcu.h>
1aeb272c 48#include <linux/slab.h>
f07767fd 49#include <asm/byteorder.h>
a241ec65
PM
50
51MODULE_LICENSE("GPL");
b772e1dd 52MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and "
a71fca58 53 "Josh Triplett <josh@freedesktop.org>");
a241ec65 54
4802211c 55static int nreaders = -1; /* # reader threads, defaults to 2*ncpus */
b772e1dd 56static int nfakewriters = 4; /* # fake writer threads */
d84f5203 57static int stat_interval; /* Interval between stats, in seconds. */
a241ec65 58 /* Defaults to "only at end of test". */
d84f5203
SV
59static int verbose; /* Print more debug info. */
60static int test_no_idle_hz; /* Test RCU's support for tickless idle CPUs. */
d120f65f
PM
61static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/
62static int stutter = 5; /* Start/stop testing interval (in sec) */
0729fbf3 63static int irqreader = 1; /* RCU readers from irq (timers). */
bf66f18e
PM
64static int fqs_duration = 0; /* Duration of bursts (us), 0 to disable. */
65static int fqs_holdoff = 0; /* Hold time within burst (us). */
66static int fqs_stutter = 3; /* Wait time between bursts (s). */
20d2e428 67static char *torture_type = "rcu"; /* What RCU implementation to torture. */
a241ec65 68
d6ad6711 69module_param(nreaders, int, 0444);
a241ec65 70MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
d6ad6711 71module_param(nfakewriters, int, 0444);
b772e1dd 72MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");
d6ad6711 73module_param(stat_interval, int, 0444);
a241ec65 74MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
d6ad6711 75module_param(verbose, bool, 0444);
a241ec65 76MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
d6ad6711 77module_param(test_no_idle_hz, bool, 0444);
d84f5203 78MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
d6ad6711 79module_param(shuffle_interval, int, 0444);
d84f5203 80MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
d120f65f
PM
81module_param(stutter, int, 0444);
82MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");
0729fbf3
PM
83module_param(irqreader, int, 0444);
84MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");
bf66f18e
PM
85module_param(fqs_duration, int, 0444);
86MODULE_PARM_DESC(fqs_duration, "Duration of fqs bursts (us)");
87module_param(fqs_holdoff, int, 0444);
88MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)");
89module_param(fqs_stutter, int, 0444);
90MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");
d6ad6711 91module_param(torture_type, charp, 0444);
b2896d2e 92MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
72e9bb54
PM
93
94#define TORTURE_FLAG "-torture:"
a241ec65 95#define PRINTK_STRING(s) \
72e9bb54 96 do { printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
a241ec65 97#define VERBOSE_PRINTK_STRING(s) \
72e9bb54 98 do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
a241ec65 99#define VERBOSE_PRINTK_ERRSTRING(s) \
72e9bb54 100 do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0)
a241ec65
PM
101
102static char printk_buf[4096];
103
104static int nrealreaders;
105static struct task_struct *writer_task;
b772e1dd 106static struct task_struct **fakewriter_tasks;
a241ec65
PM
107static struct task_struct **reader_tasks;
108static struct task_struct *stats_task;
d84f5203 109static struct task_struct *shuffler_task;
d120f65f 110static struct task_struct *stutter_task;
bf66f18e 111static struct task_struct *fqs_task;
a241ec65
PM
112
113#define RCU_TORTURE_PIPE_LEN 10
114
115struct rcu_torture {
116 struct rcu_head rtort_rcu;
117 int rtort_pipe_count;
118 struct list_head rtort_free;
996417d2 119 int rtort_mbtest;
a241ec65
PM
120};
121
a241ec65 122static LIST_HEAD(rcu_torture_freelist);
0ddea0ea 123static struct rcu_torture __rcu *rcu_torture_current;
a71fca58 124static long rcu_torture_current_version;
a241ec65
PM
125static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
126static DEFINE_SPINLOCK(rcu_torture_lock);
127static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =
128 { 0 };
129static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =
130 { 0 };
131static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
b2896d2e
PM
132static atomic_t n_rcu_torture_alloc;
133static atomic_t n_rcu_torture_alloc_fail;
134static atomic_t n_rcu_torture_free;
135static atomic_t n_rcu_torture_mberror;
136static atomic_t n_rcu_torture_error;
a71fca58 137static long n_rcu_torture_timers;
e3033736 138static struct list_head rcu_torture_removed;
73d0a4b1 139static cpumask_var_t shuffle_tmp_mask;
a241ec65 140
a71fca58 141static int stutter_pause_test;
d120f65f 142
31a72bce
PM
143#if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)
144#define RCUTORTURE_RUNNABLE_INIT 1
145#else
146#define RCUTORTURE_RUNNABLE_INIT 0
147#endif
148int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
149
c9d557c1
PM
150/* Mediate rmmod and system shutdown. Concurrent rmmod & shutdown illegal! */
151
152#define FULLSTOP_DONTSTOP 0 /* Normal operation. */
153#define FULLSTOP_SHUTDOWN 1 /* System shutdown with rcutorture running. */
154#define FULLSTOP_RMMOD 2 /* Normal rmmod of rcutorture. */
155static int fullstop = FULLSTOP_RMMOD;
0ddea0ea
PM
156/*
157 * Protect fullstop transitions and spawning of kthreads.
158 */
159static DEFINE_MUTEX(fullstop_mutex);
343e9099
PM
160
161/*
c9d557c1 162 * Detect and respond to a system shutdown.
343e9099
PM
163 */
164static int
165rcutorture_shutdown_notify(struct notifier_block *unused1,
166 unsigned long unused2, void *unused3)
167{
c59ab97e 168 mutex_lock(&fullstop_mutex);
c9d557c1 169 if (fullstop == FULLSTOP_DONTSTOP)
c59ab97e 170 fullstop = FULLSTOP_SHUTDOWN;
c9d557c1
PM
171 else
172 printk(KERN_WARNING /* but going down anyway, so... */
173 "Concurrent 'rmmod rcutorture' and shutdown illegal!\n");
c59ab97e 174 mutex_unlock(&fullstop_mutex);
343e9099
PM
175 return NOTIFY_DONE;
176}
177
c9d557c1
PM
178/*
179 * Absorb kthreads into a kernel function that won't return, so that
180 * they won't ever access module text or data again.
181 */
182static void rcutorture_shutdown_absorb(char *title)
183{
184 if (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {
185 printk(KERN_NOTICE
186 "rcutorture thread %s parking due to system shutdown\n",
187 title);
188 schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);
189 }
190}
191
a241ec65
PM
192/*
193 * Allocate an element from the rcu_tortures pool.
194 */
97a41e26 195static struct rcu_torture *
a241ec65
PM
196rcu_torture_alloc(void)
197{
198 struct list_head *p;
199
adac1665 200 spin_lock_bh(&rcu_torture_lock);
a241ec65
PM
201 if (list_empty(&rcu_torture_freelist)) {
202 atomic_inc(&n_rcu_torture_alloc_fail);
adac1665 203 spin_unlock_bh(&rcu_torture_lock);
a241ec65
PM
204 return NULL;
205 }
206 atomic_inc(&n_rcu_torture_alloc);
207 p = rcu_torture_freelist.next;
208 list_del_init(p);
adac1665 209 spin_unlock_bh(&rcu_torture_lock);
a241ec65
PM
210 return container_of(p, struct rcu_torture, rtort_free);
211}
212
213/*
214 * Free an element to the rcu_tortures pool.
215 */
216static void
217rcu_torture_free(struct rcu_torture *p)
218{
219 atomic_inc(&n_rcu_torture_free);
adac1665 220 spin_lock_bh(&rcu_torture_lock);
a241ec65 221 list_add_tail(&p->rtort_free, &rcu_torture_freelist);
adac1665 222 spin_unlock_bh(&rcu_torture_lock);
a241ec65
PM
223}
224
a241ec65
PM
225struct rcu_random_state {
226 unsigned long rrs_state;
75cfef32 227 long rrs_count;
a241ec65
PM
228};
229
230#define RCU_RANDOM_MULT 39916801 /* prime */
231#define RCU_RANDOM_ADD 479001701 /* prime */
232#define RCU_RANDOM_REFRESH 10000
233
234#define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 }
235
236/*
237 * Crude but fast random-number generator. Uses a linear congruential
c17ac855 238 * generator, with occasional help from cpu_clock().
a241ec65 239 */
75cfef32 240static unsigned long
a241ec65
PM
241rcu_random(struct rcu_random_state *rrsp)
242{
a241ec65 243 if (--rrsp->rrs_count < 0) {
c676329a 244 rrsp->rrs_state += (unsigned long)local_clock();
a241ec65
PM
245 rrsp->rrs_count = RCU_RANDOM_REFRESH;
246 }
247 rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;
248 return swahw32(rrsp->rrs_state);
249}
250
d120f65f 251static void
c9d557c1 252rcu_stutter_wait(char *title)
d120f65f 253{
c9d557c1 254 while (stutter_pause_test || !rcutorture_runnable) {
e3d7be27
PM
255 if (rcutorture_runnable)
256 schedule_timeout_interruptible(1);
257 else
3ccf79f4 258 schedule_timeout_interruptible(round_jiffies_relative(HZ));
c9d557c1 259 rcutorture_shutdown_absorb(title);
343e9099 260 }
d120f65f
PM
261}
262
72e9bb54
PM
263/*
264 * Operations vector for selecting different types of tests.
265 */
266
267struct rcu_torture_ops {
268 void (*init)(void);
269 void (*cleanup)(void);
270 int (*readlock)(void);
0acc512c 271 void (*read_delay)(struct rcu_random_state *rrsp);
72e9bb54
PM
272 void (*readunlock)(int idx);
273 int (*completed)(void);
0acc512c 274 void (*deferred_free)(struct rcu_torture *p);
b772e1dd 275 void (*sync)(void);
2326974d 276 void (*cb_barrier)(void);
bf66f18e 277 void (*fqs)(void);
72e9bb54 278 int (*stats)(char *page);
0acc512c 279 int irq_capable;
72e9bb54
PM
280 char *name;
281};
a71fca58
PM
282
283static struct rcu_torture_ops *cur_ops;
72e9bb54
PM
284
285/*
286 * Definitions for rcu torture testing.
287 */
288
a49a4af7 289static int rcu_torture_read_lock(void) __acquires(RCU)
72e9bb54
PM
290{
291 rcu_read_lock();
292 return 0;
293}
294
b2896d2e
PM
295static void rcu_read_delay(struct rcu_random_state *rrsp)
296{
b8d57a76
JT
297 const unsigned long shortdelay_us = 200;
298 const unsigned long longdelay_ms = 50;
b2896d2e 299
b8d57a76
JT
300 /* We want a short delay sometimes to make a reader delay the grace
301 * period, and we want a long delay occasionally to trigger
302 * force_quiescent_state. */
b2896d2e 303
b8d57a76
JT
304 if (!(rcu_random(rrsp) % (nrealreaders * 2000 * longdelay_ms)))
305 mdelay(longdelay_ms);
306 if (!(rcu_random(rrsp) % (nrealreaders * 2 * shortdelay_us)))
307 udelay(shortdelay_us);
e546f485
LJ
308#ifdef CONFIG_PREEMPT
309 if (!preempt_count() && !(rcu_random(rrsp) % (nrealreaders * 20000)))
310 preempt_schedule(); /* No QS if preempt_disable() in effect */
311#endif
b2896d2e
PM
312}
313
a49a4af7 314static void rcu_torture_read_unlock(int idx) __releases(RCU)
72e9bb54
PM
315{
316 rcu_read_unlock();
317}
318
319static int rcu_torture_completed(void)
320{
321 return rcu_batches_completed();
322}
323
324static void
325rcu_torture_cb(struct rcu_head *p)
326{
327 int i;
328 struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
329
c9d557c1 330 if (fullstop != FULLSTOP_DONTSTOP) {
72e9bb54
PM
331 /* Test is ending, just drop callbacks on the floor. */
332 /* The next initialization will pick up the pieces. */
333 return;
334 }
335 i = rp->rtort_pipe_count;
336 if (i > RCU_TORTURE_PIPE_LEN)
337 i = RCU_TORTURE_PIPE_LEN;
338 atomic_inc(&rcu_torture_wcount[i]);
339 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
340 rp->rtort_mbtest = 0;
341 rcu_torture_free(rp);
342 } else
0acc512c 343 cur_ops->deferred_free(rp);
72e9bb54
PM
344}
345
d9a3da06
PM
346static int rcu_no_completed(void)
347{
348 return 0;
349}
350
72e9bb54
PM
351static void rcu_torture_deferred_free(struct rcu_torture *p)
352{
353 call_rcu(&p->rtort_rcu, rcu_torture_cb);
354}
355
356static struct rcu_torture_ops rcu_ops = {
0acc512c
PM
357 .init = NULL,
358 .cleanup = NULL,
359 .readlock = rcu_torture_read_lock,
360 .read_delay = rcu_read_delay,
361 .readunlock = rcu_torture_read_unlock,
362 .completed = rcu_torture_completed,
363 .deferred_free = rcu_torture_deferred_free,
364 .sync = synchronize_rcu,
365 .cb_barrier = rcu_barrier,
bf66f18e 366 .fqs = rcu_force_quiescent_state,
0acc512c 367 .stats = NULL,
a71fca58
PM
368 .irq_capable = 1,
369 .name = "rcu"
72e9bb54
PM
370};
371
e3033736
JT
372static void rcu_sync_torture_deferred_free(struct rcu_torture *p)
373{
374 int i;
375 struct rcu_torture *rp;
376 struct rcu_torture *rp1;
377
378 cur_ops->sync();
379 list_add(&p->rtort_free, &rcu_torture_removed);
380 list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {
381 i = rp->rtort_pipe_count;
382 if (i > RCU_TORTURE_PIPE_LEN)
383 i = RCU_TORTURE_PIPE_LEN;
384 atomic_inc(&rcu_torture_wcount[i]);
385 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
386 rp->rtort_mbtest = 0;
387 list_del(&rp->rtort_free);
388 rcu_torture_free(rp);
389 }
390 }
391}
392
393static void rcu_sync_torture_init(void)
394{
395 INIT_LIST_HEAD(&rcu_torture_removed);
396}
397
20d2e428 398static struct rcu_torture_ops rcu_sync_ops = {
0acc512c
PM
399 .init = rcu_sync_torture_init,
400 .cleanup = NULL,
401 .readlock = rcu_torture_read_lock,
402 .read_delay = rcu_read_delay,
403 .readunlock = rcu_torture_read_unlock,
404 .completed = rcu_torture_completed,
405 .deferred_free = rcu_sync_torture_deferred_free,
406 .sync = synchronize_rcu,
407 .cb_barrier = NULL,
bf66f18e 408 .fqs = rcu_force_quiescent_state,
0acc512c
PM
409 .stats = NULL,
410 .irq_capable = 1,
411 .name = "rcu_sync"
20d2e428
JT
412};
413
d9a3da06
PM
414static struct rcu_torture_ops rcu_expedited_ops = {
415 .init = rcu_sync_torture_init,
416 .cleanup = NULL,
417 .readlock = rcu_torture_read_lock,
418 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
419 .readunlock = rcu_torture_read_unlock,
420 .completed = rcu_no_completed,
421 .deferred_free = rcu_sync_torture_deferred_free,
422 .sync = synchronize_rcu_expedited,
423 .cb_barrier = NULL,
bf66f18e 424 .fqs = rcu_force_quiescent_state,
d9a3da06
PM
425 .stats = NULL,
426 .irq_capable = 1,
427 .name = "rcu_expedited"
428};
429
c32e0660
PM
430/*
431 * Definitions for rcu_bh torture testing.
432 */
433
a49a4af7 434static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
c32e0660
PM
435{
436 rcu_read_lock_bh();
437 return 0;
438}
439
a49a4af7 440static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
c32e0660
PM
441{
442 rcu_read_unlock_bh();
443}
444
445static int rcu_bh_torture_completed(void)
446{
447 return rcu_batches_completed_bh();
448}
449
450static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
451{
452 call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
453}
454
b772e1dd
JT
455struct rcu_bh_torture_synchronize {
456 struct rcu_head head;
457 struct completion completion;
458};
459
460static void rcu_bh_torture_wakeme_after_cb(struct rcu_head *head)
461{
462 struct rcu_bh_torture_synchronize *rcu;
463
464 rcu = container_of(head, struct rcu_bh_torture_synchronize, head);
465 complete(&rcu->completion);
466}
467
468static void rcu_bh_torture_synchronize(void)
469{
470 struct rcu_bh_torture_synchronize rcu;
471
72d5a9f7 472 init_rcu_head_on_stack(&rcu.head);
b772e1dd
JT
473 init_completion(&rcu.completion);
474 call_rcu_bh(&rcu.head, rcu_bh_torture_wakeme_after_cb);
475 wait_for_completion(&rcu.completion);
72d5a9f7 476 destroy_rcu_head_on_stack(&rcu.head);
b772e1dd
JT
477}
478
c32e0660 479static struct rcu_torture_ops rcu_bh_ops = {
0acc512c
PM
480 .init = NULL,
481 .cleanup = NULL,
482 .readlock = rcu_bh_torture_read_lock,
483 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
484 .readunlock = rcu_bh_torture_read_unlock,
485 .completed = rcu_bh_torture_completed,
486 .deferred_free = rcu_bh_torture_deferred_free,
487 .sync = rcu_bh_torture_synchronize,
488 .cb_barrier = rcu_barrier_bh,
bf66f18e 489 .fqs = rcu_bh_force_quiescent_state,
0acc512c
PM
490 .stats = NULL,
491 .irq_capable = 1,
492 .name = "rcu_bh"
c32e0660
PM
493};
494
11a14701 495static struct rcu_torture_ops rcu_bh_sync_ops = {
0acc512c
PM
496 .init = rcu_sync_torture_init,
497 .cleanup = NULL,
498 .readlock = rcu_bh_torture_read_lock,
499 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
500 .readunlock = rcu_bh_torture_read_unlock,
501 .completed = rcu_bh_torture_completed,
502 .deferred_free = rcu_sync_torture_deferred_free,
503 .sync = rcu_bh_torture_synchronize,
504 .cb_barrier = NULL,
bf66f18e 505 .fqs = rcu_bh_force_quiescent_state,
0acc512c
PM
506 .stats = NULL,
507 .irq_capable = 1,
508 .name = "rcu_bh_sync"
11a14701
JT
509};
510
b2896d2e
PM
511/*
512 * Definitions for srcu torture testing.
513 */
514
515static struct srcu_struct srcu_ctl;
b2896d2e
PM
516
517static void srcu_torture_init(void)
518{
519 init_srcu_struct(&srcu_ctl);
e3033736 520 rcu_sync_torture_init();
b2896d2e
PM
521}
522
523static void srcu_torture_cleanup(void)
524{
525 synchronize_srcu(&srcu_ctl);
526 cleanup_srcu_struct(&srcu_ctl);
527}
528
012d3ca8 529static int srcu_torture_read_lock(void) __acquires(&srcu_ctl)
b2896d2e
PM
530{
531 return srcu_read_lock(&srcu_ctl);
532}
533
534static void srcu_read_delay(struct rcu_random_state *rrsp)
535{
536 long delay;
537 const long uspertick = 1000000 / HZ;
538 const long longdelay = 10;
539
540 /* We want there to be long-running readers, but not all the time. */
541
542 delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick);
543 if (!delay)
544 schedule_timeout_interruptible(longdelay);
e546f485
LJ
545 else
546 rcu_read_delay(rrsp);
b2896d2e
PM
547}
548
012d3ca8 549static void srcu_torture_read_unlock(int idx) __releases(&srcu_ctl)
b2896d2e
PM
550{
551 srcu_read_unlock(&srcu_ctl, idx);
552}
553
554static int srcu_torture_completed(void)
555{
556 return srcu_batches_completed(&srcu_ctl);
557}
558
b772e1dd
JT
559static void srcu_torture_synchronize(void)
560{
561 synchronize_srcu(&srcu_ctl);
562}
563
b2896d2e
PM
564static int srcu_torture_stats(char *page)
565{
566 int cnt = 0;
567 int cpu;
568 int idx = srcu_ctl.completed & 0x1;
569
570 cnt += sprintf(&page[cnt], "%s%s per-CPU(idx=%d):",
571 torture_type, TORTURE_FLAG, idx);
572 for_each_possible_cpu(cpu) {
573 cnt += sprintf(&page[cnt], " %d(%d,%d)", cpu,
574 per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx],
575 per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]);
576 }
577 cnt += sprintf(&page[cnt], "\n");
578 return cnt;
579}
580
581static struct rcu_torture_ops srcu_ops = {
0acc512c
PM
582 .init = srcu_torture_init,
583 .cleanup = srcu_torture_cleanup,
584 .readlock = srcu_torture_read_lock,
585 .read_delay = srcu_read_delay,
586 .readunlock = srcu_torture_read_unlock,
587 .completed = srcu_torture_completed,
588 .deferred_free = rcu_sync_torture_deferred_free,
589 .sync = srcu_torture_synchronize,
590 .cb_barrier = NULL,
591 .stats = srcu_torture_stats,
592 .name = "srcu"
b2896d2e
PM
593};
594
804bb837
PM
595static void srcu_torture_synchronize_expedited(void)
596{
597 synchronize_srcu_expedited(&srcu_ctl);
598}
599
600static struct rcu_torture_ops srcu_expedited_ops = {
601 .init = srcu_torture_init,
602 .cleanup = srcu_torture_cleanup,
603 .readlock = srcu_torture_read_lock,
604 .read_delay = srcu_read_delay,
605 .readunlock = srcu_torture_read_unlock,
606 .completed = srcu_torture_completed,
607 .deferred_free = rcu_sync_torture_deferred_free,
608 .sync = srcu_torture_synchronize_expedited,
609 .cb_barrier = NULL,
610 .stats = srcu_torture_stats,
611 .name = "srcu_expedited"
612};
613
4b6c2cca
JT
614/*
615 * Definitions for sched torture testing.
616 */
617
618static int sched_torture_read_lock(void)
619{
620 preempt_disable();
621 return 0;
622}
623
624static void sched_torture_read_unlock(int idx)
625{
626 preempt_enable();
627}
628
2326974d
PM
629static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
630{
631 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
632}
633
4b6c2cca
JT
634static void sched_torture_synchronize(void)
635{
636 synchronize_sched();
637}
638
639static struct rcu_torture_ops sched_ops = {
0acc512c
PM
640 .init = rcu_sync_torture_init,
641 .cleanup = NULL,
642 .readlock = sched_torture_read_lock,
643 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
644 .readunlock = sched_torture_read_unlock,
d9a3da06 645 .completed = rcu_no_completed,
0acc512c
PM
646 .deferred_free = rcu_sched_torture_deferred_free,
647 .sync = sched_torture_synchronize,
648 .cb_barrier = rcu_barrier_sched,
bf66f18e 649 .fqs = rcu_sched_force_quiescent_state,
0acc512c
PM
650 .stats = NULL,
651 .irq_capable = 1,
652 .name = "sched"
4b6c2cca
JT
653};
654
804bb837 655static struct rcu_torture_ops sched_sync_ops = {
0acc512c
PM
656 .init = rcu_sync_torture_init,
657 .cleanup = NULL,
658 .readlock = sched_torture_read_lock,
659 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
660 .readunlock = sched_torture_read_unlock,
d9a3da06 661 .completed = rcu_no_completed,
0acc512c
PM
662 .deferred_free = rcu_sync_torture_deferred_free,
663 .sync = sched_torture_synchronize,
664 .cb_barrier = NULL,
bf66f18e 665 .fqs = rcu_sched_force_quiescent_state,
0acc512c
PM
666 .stats = NULL,
667 .name = "sched_sync"
668};
669
0acc512c
PM
670static struct rcu_torture_ops sched_expedited_ops = {
671 .init = rcu_sync_torture_init,
672 .cleanup = NULL,
673 .readlock = sched_torture_read_lock,
674 .read_delay = rcu_read_delay, /* just reuse rcu's version. */
675 .readunlock = sched_torture_read_unlock,
d9a3da06 676 .completed = rcu_no_completed,
0acc512c
PM
677 .deferred_free = rcu_sync_torture_deferred_free,
678 .sync = synchronize_sched_expedited,
679 .cb_barrier = NULL,
bf66f18e 680 .fqs = rcu_sched_force_quiescent_state,
969c7921 681 .stats = NULL,
0acc512c
PM
682 .irq_capable = 1,
683 .name = "sched_expedited"
2326974d
PM
684};
685
bf66f18e
PM
686/*
687 * RCU torture force-quiescent-state kthread. Repeatedly induces
688 * bursts of calls to force_quiescent_state(), increasing the probability
689 * of occurrence of some important types of race conditions.
690 */
691static int
692rcu_torture_fqs(void *arg)
693{
694 unsigned long fqs_resume_time;
695 int fqs_burst_remaining;
696
697 VERBOSE_PRINTK_STRING("rcu_torture_fqs task started");
698 do {
699 fqs_resume_time = jiffies + fqs_stutter * HZ;
700 while (jiffies - fqs_resume_time > LONG_MAX) {
701 schedule_timeout_interruptible(1);
702 }
703 fqs_burst_remaining = fqs_duration;
704 while (fqs_burst_remaining > 0) {
705 cur_ops->fqs();
706 udelay(fqs_holdoff);
707 fqs_burst_remaining -= fqs_holdoff;
708 }
709 rcu_stutter_wait("rcu_torture_fqs");
710 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
711 VERBOSE_PRINTK_STRING("rcu_torture_fqs task stopping");
712 rcutorture_shutdown_absorb("rcu_torture_fqs");
713 while (!kthread_should_stop())
714 schedule_timeout_uninterruptible(1);
715 return 0;
716}
717
a241ec65
PM
718/*
719 * RCU torture writer kthread. Repeatedly substitutes a new structure
720 * for that pointed to by rcu_torture_current, freeing the old structure
721 * after a series of grace periods (the "pipeline").
722 */
723static int
724rcu_torture_writer(void *arg)
725{
726 int i;
727 long oldbatch = rcu_batches_completed();
728 struct rcu_torture *rp;
729 struct rcu_torture *old_rp;
730 static DEFINE_RCU_RANDOM(rand);
731
732 VERBOSE_PRINTK_STRING("rcu_torture_writer task started");
dbdf65b1
IM
733 set_user_nice(current, 19);
734
a241ec65
PM
735 do {
736 schedule_timeout_uninterruptible(1);
a71fca58
PM
737 rp = rcu_torture_alloc();
738 if (rp == NULL)
a241ec65
PM
739 continue;
740 rp->rtort_pipe_count = 0;
741 udelay(rcu_random(&rand) & 0x3ff);
0ddea0ea
PM
742 old_rp = rcu_dereference_check(rcu_torture_current,
743 current == writer_task);
996417d2 744 rp->rtort_mbtest = 1;
a241ec65 745 rcu_assign_pointer(rcu_torture_current, rp);
9b2619af 746 smp_wmb(); /* Mods to old_rp must follow rcu_assign_pointer() */
c8e5b163 747 if (old_rp) {
a241ec65
PM
748 i = old_rp->rtort_pipe_count;
749 if (i > RCU_TORTURE_PIPE_LEN)
750 i = RCU_TORTURE_PIPE_LEN;
751 atomic_inc(&rcu_torture_wcount[i]);
752 old_rp->rtort_pipe_count++;
0acc512c 753 cur_ops->deferred_free(old_rp);
a241ec65
PM
754 }
755 rcu_torture_current_version++;
72e9bb54 756 oldbatch = cur_ops->completed();
c9d557c1
PM
757 rcu_stutter_wait("rcu_torture_writer");
758 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
a241ec65 759 VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
c9d557c1
PM
760 rcutorture_shutdown_absorb("rcu_torture_writer");
761 while (!kthread_should_stop())
a241ec65
PM
762 schedule_timeout_uninterruptible(1);
763 return 0;
764}
765
b772e1dd
JT
766/*
767 * RCU torture fake writer kthread. Repeatedly calls sync, with a random
768 * delay between calls.
769 */
770static int
771rcu_torture_fakewriter(void *arg)
772{
773 DEFINE_RCU_RANDOM(rand);
774
775 VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task started");
776 set_user_nice(current, 19);
777
778 do {
779 schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10);
780 udelay(rcu_random(&rand) & 0x3ff);
781 cur_ops->sync();
c9d557c1
PM
782 rcu_stutter_wait("rcu_torture_fakewriter");
783 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
b772e1dd
JT
784
785 VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping");
c9d557c1
PM
786 rcutorture_shutdown_absorb("rcu_torture_fakewriter");
787 while (!kthread_should_stop())
b772e1dd
JT
788 schedule_timeout_uninterruptible(1);
789 return 0;
790}
791
0729fbf3
PM
792/*
793 * RCU torture reader from timer handler. Dereferences rcu_torture_current,
794 * incrementing the corresponding element of the pipeline array. The
795 * counter in the element should never be greater than 1, otherwise, the
796 * RCU implementation is broken.
797 */
798static void rcu_torture_timer(unsigned long unused)
799{
800 int idx;
801 int completed;
802 static DEFINE_RCU_RANDOM(rand);
803 static DEFINE_SPINLOCK(rand_lock);
804 struct rcu_torture *p;
805 int pipe_count;
806
807 idx = cur_ops->readlock();
808 completed = cur_ops->completed();
632ee200
PM
809 p = rcu_dereference_check(rcu_torture_current,
810 rcu_read_lock_held() ||
811 rcu_read_lock_bh_held() ||
812 rcu_read_lock_sched_held() ||
813 srcu_read_lock_held(&srcu_ctl));
0729fbf3
PM
814 if (p == NULL) {
815 /* Leave because rcu_torture_writer is not yet underway */
816 cur_ops->readunlock(idx);
817 return;
818 }
819 if (p->rtort_mbtest == 0)
820 atomic_inc(&n_rcu_torture_mberror);
821 spin_lock(&rand_lock);
0acc512c 822 cur_ops->read_delay(&rand);
0729fbf3
PM
823 n_rcu_torture_timers++;
824 spin_unlock(&rand_lock);
825 preempt_disable();
826 pipe_count = p->rtort_pipe_count;
827 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
828 /* Should not happen, but... */
829 pipe_count = RCU_TORTURE_PIPE_LEN;
830 }
dd17c8f7 831 __this_cpu_inc(rcu_torture_count[pipe_count]);
0729fbf3
PM
832 completed = cur_ops->completed() - completed;
833 if (completed > RCU_TORTURE_PIPE_LEN) {
834 /* Should not happen, but... */
835 completed = RCU_TORTURE_PIPE_LEN;
836 }
dd17c8f7 837 __this_cpu_inc(rcu_torture_batch[completed]);
0729fbf3
PM
838 preempt_enable();
839 cur_ops->readunlock(idx);
840}
841
a241ec65
PM
842/*
843 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current,
844 * incrementing the corresponding element of the pipeline array. The
845 * counter in the element should never be greater than 1, otherwise, the
846 * RCU implementation is broken.
847 */
848static int
849rcu_torture_reader(void *arg)
850{
851 int completed;
72e9bb54 852 int idx;
a241ec65
PM
853 DEFINE_RCU_RANDOM(rand);
854 struct rcu_torture *p;
855 int pipe_count;
0729fbf3 856 struct timer_list t;
a241ec65
PM
857
858 VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
dbdf65b1 859 set_user_nice(current, 19);
0acc512c 860 if (irqreader && cur_ops->irq_capable)
0729fbf3 861 setup_timer_on_stack(&t, rcu_torture_timer, 0);
dbdf65b1 862
a241ec65 863 do {
0acc512c 864 if (irqreader && cur_ops->irq_capable) {
0729fbf3 865 if (!timer_pending(&t))
6155fec9 866 mod_timer(&t, jiffies + 1);
0729fbf3 867 }
72e9bb54
PM
868 idx = cur_ops->readlock();
869 completed = cur_ops->completed();
632ee200
PM
870 p = rcu_dereference_check(rcu_torture_current,
871 rcu_read_lock_held() ||
872 rcu_read_lock_bh_held() ||
873 rcu_read_lock_sched_held() ||
874 srcu_read_lock_held(&srcu_ctl));
a241ec65
PM
875 if (p == NULL) {
876 /* Wait for rcu_torture_writer to get underway */
72e9bb54 877 cur_ops->readunlock(idx);
a241ec65
PM
878 schedule_timeout_interruptible(HZ);
879 continue;
880 }
996417d2
PM
881 if (p->rtort_mbtest == 0)
882 atomic_inc(&n_rcu_torture_mberror);
0acc512c 883 cur_ops->read_delay(&rand);
a241ec65
PM
884 preempt_disable();
885 pipe_count = p->rtort_pipe_count;
886 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
887 /* Should not happen, but... */
888 pipe_count = RCU_TORTURE_PIPE_LEN;
889 }
dd17c8f7 890 __this_cpu_inc(rcu_torture_count[pipe_count]);
72e9bb54 891 completed = cur_ops->completed() - completed;
a241ec65
PM
892 if (completed > RCU_TORTURE_PIPE_LEN) {
893 /* Should not happen, but... */
894 completed = RCU_TORTURE_PIPE_LEN;
895 }
dd17c8f7 896 __this_cpu_inc(rcu_torture_batch[completed]);
a241ec65 897 preempt_enable();
72e9bb54 898 cur_ops->readunlock(idx);
a241ec65 899 schedule();
c9d557c1
PM
900 rcu_stutter_wait("rcu_torture_reader");
901 } while (!kthread_should_stop() && fullstop == FULLSTOP_DONTSTOP);
a241ec65 902 VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
c9d557c1 903 rcutorture_shutdown_absorb("rcu_torture_reader");
0acc512c 904 if (irqreader && cur_ops->irq_capable)
0729fbf3 905 del_timer_sync(&t);
c9d557c1 906 while (!kthread_should_stop())
a241ec65
PM
907 schedule_timeout_uninterruptible(1);
908 return 0;
909}
910
911/*
912 * Create an RCU-torture statistics message in the specified buffer.
913 */
914static int
915rcu_torture_printk(char *page)
916{
917 int cnt = 0;
918 int cpu;
919 int i;
920 long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
921 long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
922
0a945022 923 for_each_possible_cpu(cpu) {
a241ec65
PM
924 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
925 pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
926 batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
927 }
928 }
929 for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
930 if (pipesummary[i] != 0)
931 break;
932 }
72e9bb54 933 cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG);
a241ec65 934 cnt += sprintf(&page[cnt],
996417d2 935 "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d "
0729fbf3 936 "rtmbe: %d nt: %ld",
a241ec65
PM
937 rcu_torture_current,
938 rcu_torture_current_version,
939 list_empty(&rcu_torture_freelist),
940 atomic_read(&n_rcu_torture_alloc),
941 atomic_read(&n_rcu_torture_alloc_fail),
996417d2 942 atomic_read(&n_rcu_torture_free),
0729fbf3
PM
943 atomic_read(&n_rcu_torture_mberror),
944 n_rcu_torture_timers);
996417d2
PM
945 if (atomic_read(&n_rcu_torture_mberror) != 0)
946 cnt += sprintf(&page[cnt], " !!!");
72e9bb54 947 cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
996417d2 948 if (i > 1) {
a241ec65 949 cnt += sprintf(&page[cnt], "!!! ");
996417d2 950 atomic_inc(&n_rcu_torture_error);
5af970a4 951 WARN_ON_ONCE(1);
996417d2 952 }
a241ec65
PM
953 cnt += sprintf(&page[cnt], "Reader Pipe: ");
954 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
955 cnt += sprintf(&page[cnt], " %ld", pipesummary[i]);
72e9bb54 956 cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
a241ec65 957 cnt += sprintf(&page[cnt], "Reader Batch: ");
72e9bb54 958 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
a241ec65 959 cnt += sprintf(&page[cnt], " %ld", batchsummary[i]);
72e9bb54 960 cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
a241ec65
PM
961 cnt += sprintf(&page[cnt], "Free-Block Circulation: ");
962 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
963 cnt += sprintf(&page[cnt], " %d",
964 atomic_read(&rcu_torture_wcount[i]));
965 }
966 cnt += sprintf(&page[cnt], "\n");
c8e5b163 967 if (cur_ops->stats)
72e9bb54 968 cnt += cur_ops->stats(&page[cnt]);
a241ec65
PM
969 return cnt;
970}
971
972/*
973 * Print torture statistics. Caller must ensure that there is only
974 * one call to this function at a given time!!! This is normally
975 * accomplished by relying on the module system to only have one copy
976 * of the module loaded, and then by giving the rcu_torture_stats
977 * kthread full control (or the init/cleanup functions when rcu_torture_stats
978 * thread is not running).
979 */
980static void
981rcu_torture_stats_print(void)
982{
983 int cnt;
984
985 cnt = rcu_torture_printk(printk_buf);
986 printk(KERN_ALERT "%s", printk_buf);
987}
988
989/*
990 * Periodically prints torture statistics, if periodic statistics printing
991 * was specified via the stat_interval module parameter.
992 *
993 * No need to worry about fullstop here, since this one doesn't reference
994 * volatile state or register callbacks.
995 */
996static int
997rcu_torture_stats(void *arg)
998{
999 VERBOSE_PRINTK_STRING("rcu_torture_stats task started");
1000 do {
1001 schedule_timeout_interruptible(stat_interval * HZ);
1002 rcu_torture_stats_print();
c9d557c1
PM
1003 rcutorture_shutdown_absorb("rcu_torture_stats");
1004 } while (!kthread_should_stop());
a241ec65
PM
1005 VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
1006 return 0;
1007}
1008
d84f5203
SV
1009static int rcu_idle_cpu; /* Force all torture tasks off this CPU */
1010
1011/* Shuffle tasks such that we allow @rcu_idle_cpu to become idle. A special case
1012 * is when @rcu_idle_cpu = -1, when we allow the tasks to run on all CPUs.
1013 */
b2896d2e 1014static void rcu_torture_shuffle_tasks(void)
d84f5203 1015{
d84f5203
SV
1016 int i;
1017
73d0a4b1 1018 cpumask_setall(shuffle_tmp_mask);
86ef5c9a 1019 get_online_cpus();
d84f5203
SV
1020
1021 /* No point in shuffling if there is only one online CPU (ex: UP) */
c9d557c1
PM
1022 if (num_online_cpus() == 1) {
1023 put_online_cpus();
1024 return;
1025 }
d84f5203
SV
1026
1027 if (rcu_idle_cpu != -1)
73d0a4b1 1028 cpumask_clear_cpu(rcu_idle_cpu, shuffle_tmp_mask);
d84f5203 1029
73d0a4b1 1030 set_cpus_allowed_ptr(current, shuffle_tmp_mask);
d84f5203 1031
c8e5b163 1032 if (reader_tasks) {
d84f5203
SV
1033 for (i = 0; i < nrealreaders; i++)
1034 if (reader_tasks[i])
f70316da 1035 set_cpus_allowed_ptr(reader_tasks[i],
73d0a4b1 1036 shuffle_tmp_mask);
d84f5203
SV
1037 }
1038
c8e5b163 1039 if (fakewriter_tasks) {
b772e1dd
JT
1040 for (i = 0; i < nfakewriters; i++)
1041 if (fakewriter_tasks[i])
f70316da 1042 set_cpus_allowed_ptr(fakewriter_tasks[i],
73d0a4b1 1043 shuffle_tmp_mask);
b772e1dd
JT
1044 }
1045
d84f5203 1046 if (writer_task)
73d0a4b1 1047 set_cpus_allowed_ptr(writer_task, shuffle_tmp_mask);
d84f5203
SV
1048
1049 if (stats_task)
73d0a4b1 1050 set_cpus_allowed_ptr(stats_task, shuffle_tmp_mask);
d84f5203
SV
1051
1052 if (rcu_idle_cpu == -1)
1053 rcu_idle_cpu = num_online_cpus() - 1;
1054 else
1055 rcu_idle_cpu--;
1056
86ef5c9a 1057 put_online_cpus();
d84f5203
SV
1058}
1059
1060/* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
1061 * system to become idle at a time and cut off its timer ticks. This is meant
1062 * to test the support for such tickless idle CPU in RCU.
1063 */
1064static int
1065rcu_torture_shuffle(void *arg)
1066{
1067 VERBOSE_PRINTK_STRING("rcu_torture_shuffle task started");
1068 do {
1069 schedule_timeout_interruptible(shuffle_interval * HZ);
1070 rcu_torture_shuffle_tasks();
c9d557c1
PM
1071 rcutorture_shutdown_absorb("rcu_torture_shuffle");
1072 } while (!kthread_should_stop());
d84f5203
SV
1073 VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping");
1074 return 0;
1075}
1076
d120f65f
PM
1077/* Cause the rcutorture test to "stutter", starting and stopping all
1078 * threads periodically.
1079 */
1080static int
1081rcu_torture_stutter(void *arg)
1082{
1083 VERBOSE_PRINTK_STRING("rcu_torture_stutter task started");
1084 do {
1085 schedule_timeout_interruptible(stutter * HZ);
1086 stutter_pause_test = 1;
c9d557c1 1087 if (!kthread_should_stop())
d120f65f
PM
1088 schedule_timeout_interruptible(stutter * HZ);
1089 stutter_pause_test = 0;
c9d557c1
PM
1090 rcutorture_shutdown_absorb("rcu_torture_stutter");
1091 } while (!kthread_should_stop());
d120f65f
PM
1092 VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping");
1093 return 0;
1094}
1095
95c38322
PM
1096static inline void
1097rcu_torture_print_module_parms(char *tag)
1098{
b772e1dd
JT
1099 printk(KERN_ALERT "%s" TORTURE_FLAG
1100 "--- %s: nreaders=%d nfakewriters=%d "
95c38322 1101 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
bf66f18e
PM
1102 "shuffle_interval=%d stutter=%d irqreader=%d "
1103 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d\n",
b772e1dd 1104 torture_type, tag, nrealreaders, nfakewriters,
d120f65f 1105 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
bf66f18e 1106 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter);
95c38322
PM
1107}
1108
343e9099
PM
1109static struct notifier_block rcutorture_nb = {
1110 .notifier_call = rcutorture_shutdown_notify,
1111};
1112
a241ec65
PM
1113static void
1114rcu_torture_cleanup(void)
1115{
1116 int i;
1117
343e9099 1118 mutex_lock(&fullstop_mutex);
c9d557c1
PM
1119 if (fullstop == FULLSTOP_SHUTDOWN) {
1120 printk(KERN_WARNING /* but going down anyway, so... */
1121 "Concurrent 'rmmod rcutorture' and shutdown illegal!\n");
343e9099 1122 mutex_unlock(&fullstop_mutex);
c9d557c1 1123 schedule_timeout_uninterruptible(10);
343e9099
PM
1124 if (cur_ops->cb_barrier != NULL)
1125 cur_ops->cb_barrier();
1126 return;
1127 }
c9d557c1 1128 fullstop = FULLSTOP_RMMOD;
343e9099
PM
1129 mutex_unlock(&fullstop_mutex);
1130 unregister_reboot_notifier(&rcutorture_nb);
d120f65f
PM
1131 if (stutter_task) {
1132 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
1133 kthread_stop(stutter_task);
1134 }
1135 stutter_task = NULL;
c8e5b163 1136 if (shuffler_task) {
d84f5203
SV
1137 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task");
1138 kthread_stop(shuffler_task);
73d0a4b1 1139 free_cpumask_var(shuffle_tmp_mask);
d84f5203
SV
1140 }
1141 shuffler_task = NULL;
1142
c8e5b163 1143 if (writer_task) {
a241ec65
PM
1144 VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task");
1145 kthread_stop(writer_task);
1146 }
1147 writer_task = NULL;
1148
c8e5b163 1149 if (reader_tasks) {
a241ec65 1150 for (i = 0; i < nrealreaders; i++) {
c8e5b163 1151 if (reader_tasks[i]) {
a241ec65
PM
1152 VERBOSE_PRINTK_STRING(
1153 "Stopping rcu_torture_reader task");
1154 kthread_stop(reader_tasks[i]);
1155 }
1156 reader_tasks[i] = NULL;
1157 }
1158 kfree(reader_tasks);
1159 reader_tasks = NULL;
1160 }
1161 rcu_torture_current = NULL;
1162
c8e5b163 1163 if (fakewriter_tasks) {
b772e1dd 1164 for (i = 0; i < nfakewriters; i++) {
c8e5b163 1165 if (fakewriter_tasks[i]) {
b772e1dd
JT
1166 VERBOSE_PRINTK_STRING(
1167 "Stopping rcu_torture_fakewriter task");
1168 kthread_stop(fakewriter_tasks[i]);
1169 }
1170 fakewriter_tasks[i] = NULL;
1171 }
1172 kfree(fakewriter_tasks);
1173 fakewriter_tasks = NULL;
1174 }
1175
c8e5b163 1176 if (stats_task) {
a241ec65
PM
1177 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task");
1178 kthread_stop(stats_task);
1179 }
1180 stats_task = NULL;
1181
bf66f18e
PM
1182 if (fqs_task) {
1183 VERBOSE_PRINTK_STRING("Stopping rcu_torture_fqs task");
1184 kthread_stop(fqs_task);
1185 }
1186 fqs_task = NULL;
1187
a241ec65 1188 /* Wait for all RCU callbacks to fire. */
2326974d
PM
1189
1190 if (cur_ops->cb_barrier != NULL)
1191 cur_ops->cb_barrier();
a241ec65 1192
a241ec65 1193 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */
72e9bb54 1194
c8e5b163 1195 if (cur_ops->cleanup)
72e9bb54 1196 cur_ops->cleanup();
95c38322
PM
1197 if (atomic_read(&n_rcu_torture_error))
1198 rcu_torture_print_module_parms("End of test: FAILURE");
1199 else
1200 rcu_torture_print_module_parms("End of test: SUCCESS");
a241ec65
PM
1201}
1202
6f8bc500 1203static int __init
a241ec65
PM
1204rcu_torture_init(void)
1205{
1206 int i;
1207 int cpu;
1208 int firsterr = 0;
ade5fb81 1209 static struct rcu_torture_ops *torture_ops[] =
d9a3da06
PM
1210 { &rcu_ops, &rcu_sync_ops, &rcu_expedited_ops,
1211 &rcu_bh_ops, &rcu_bh_sync_ops,
804bb837
PM
1212 &srcu_ops, &srcu_expedited_ops,
1213 &sched_ops, &sched_sync_ops, &sched_expedited_ops, };
a241ec65 1214
343e9099
PM
1215 mutex_lock(&fullstop_mutex);
1216
a241ec65 1217 /* Process args and tell the world that the torturer is on the job. */
ade5fb81 1218 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
72e9bb54 1219 cur_ops = torture_ops[i];
ade5fb81 1220 if (strcmp(torture_type, cur_ops->name) == 0)
72e9bb54 1221 break;
72e9bb54 1222 }
ade5fb81 1223 if (i == ARRAY_SIZE(torture_ops)) {
cf886c44 1224 printk(KERN_ALERT "rcu-torture: invalid torture type: \"%s\"\n",
72e9bb54 1225 torture_type);
cf886c44
PM
1226 printk(KERN_ALERT "rcu-torture types:");
1227 for (i = 0; i < ARRAY_SIZE(torture_ops); i++)
1228 printk(KERN_ALERT " %s", torture_ops[i]->name);
1229 printk(KERN_ALERT "\n");
343e9099 1230 mutex_unlock(&fullstop_mutex);
a71fca58 1231 return -EINVAL;
72e9bb54 1232 }
bf66f18e
PM
1233 if (cur_ops->fqs == NULL && fqs_duration != 0) {
1234 printk(KERN_ALERT "rcu-torture: ->fqs NULL and non-zero "
1235 "fqs_duration, fqs disabled.\n");
1236 fqs_duration = 0;
1237 }
c8e5b163 1238 if (cur_ops->init)
72e9bb54
PM
1239 cur_ops->init(); /* no "goto unwind" prior to this point!!! */
1240
a241ec65
PM
1241 if (nreaders >= 0)
1242 nrealreaders = nreaders;
1243 else
1244 nrealreaders = 2 * num_online_cpus();
95c38322 1245 rcu_torture_print_module_parms("Start of test");
c9d557c1 1246 fullstop = FULLSTOP_DONTSTOP;
a241ec65
PM
1247
1248 /* Set up the freelist. */
1249
1250 INIT_LIST_HEAD(&rcu_torture_freelist);
788e770e 1251 for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++) {
996417d2 1252 rcu_tortures[i].rtort_mbtest = 0;
a241ec65
PM
1253 list_add_tail(&rcu_tortures[i].rtort_free,
1254 &rcu_torture_freelist);
1255 }
1256
1257 /* Initialize the statistics so that each run gets its own numbers. */
1258
1259 rcu_torture_current = NULL;
1260 rcu_torture_current_version = 0;
1261 atomic_set(&n_rcu_torture_alloc, 0);
1262 atomic_set(&n_rcu_torture_alloc_fail, 0);
1263 atomic_set(&n_rcu_torture_free, 0);
996417d2
PM
1264 atomic_set(&n_rcu_torture_mberror, 0);
1265 atomic_set(&n_rcu_torture_error, 0);
a241ec65
PM
1266 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
1267 atomic_set(&rcu_torture_wcount[i], 0);
0a945022 1268 for_each_possible_cpu(cpu) {
a241ec65
PM
1269 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1270 per_cpu(rcu_torture_count, cpu)[i] = 0;
1271 per_cpu(rcu_torture_batch, cpu)[i] = 0;
1272 }
1273 }
1274
1275 /* Start up the kthreads. */
1276
1277 VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task");
1278 writer_task = kthread_run(rcu_torture_writer, NULL,
1279 "rcu_torture_writer");
1280 if (IS_ERR(writer_task)) {
1281 firsterr = PTR_ERR(writer_task);
1282 VERBOSE_PRINTK_ERRSTRING("Failed to create writer");
1283 writer_task = NULL;
1284 goto unwind;
1285 }
b772e1dd 1286 fakewriter_tasks = kzalloc(nfakewriters * sizeof(fakewriter_tasks[0]),
a71fca58 1287 GFP_KERNEL);
b772e1dd
JT
1288 if (fakewriter_tasks == NULL) {
1289 VERBOSE_PRINTK_ERRSTRING("out of memory");
1290 firsterr = -ENOMEM;
1291 goto unwind;
1292 }
1293 for (i = 0; i < nfakewriters; i++) {
1294 VERBOSE_PRINTK_STRING("Creating rcu_torture_fakewriter task");
1295 fakewriter_tasks[i] = kthread_run(rcu_torture_fakewriter, NULL,
a71fca58 1296 "rcu_torture_fakewriter");
b772e1dd
JT
1297 if (IS_ERR(fakewriter_tasks[i])) {
1298 firsterr = PTR_ERR(fakewriter_tasks[i]);
1299 VERBOSE_PRINTK_ERRSTRING("Failed to create fakewriter");
1300 fakewriter_tasks[i] = NULL;
1301 goto unwind;
1302 }
1303 }
2860aaba 1304 reader_tasks = kzalloc(nrealreaders * sizeof(reader_tasks[0]),
a241ec65
PM
1305 GFP_KERNEL);
1306 if (reader_tasks == NULL) {
1307 VERBOSE_PRINTK_ERRSTRING("out of memory");
1308 firsterr = -ENOMEM;
1309 goto unwind;
1310 }
1311 for (i = 0; i < nrealreaders; i++) {
1312 VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task");
1313 reader_tasks[i] = kthread_run(rcu_torture_reader, NULL,
1314 "rcu_torture_reader");
1315 if (IS_ERR(reader_tasks[i])) {
1316 firsterr = PTR_ERR(reader_tasks[i]);
1317 VERBOSE_PRINTK_ERRSTRING("Failed to create reader");
1318 reader_tasks[i] = NULL;
1319 goto unwind;
1320 }
1321 }
1322 if (stat_interval > 0) {
1323 VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task");
1324 stats_task = kthread_run(rcu_torture_stats, NULL,
1325 "rcu_torture_stats");
1326 if (IS_ERR(stats_task)) {
1327 firsterr = PTR_ERR(stats_task);
1328 VERBOSE_PRINTK_ERRSTRING("Failed to create stats");
1329 stats_task = NULL;
1330 goto unwind;
1331 }
1332 }
d84f5203
SV
1333 if (test_no_idle_hz) {
1334 rcu_idle_cpu = num_online_cpus() - 1;
73d0a4b1
RR
1335
1336 if (!alloc_cpumask_var(&shuffle_tmp_mask, GFP_KERNEL)) {
1337 firsterr = -ENOMEM;
1338 VERBOSE_PRINTK_ERRSTRING("Failed to alloc mask");
1339 goto unwind;
1340 }
1341
d84f5203
SV
1342 /* Create the shuffler thread */
1343 shuffler_task = kthread_run(rcu_torture_shuffle, NULL,
1344 "rcu_torture_shuffle");
1345 if (IS_ERR(shuffler_task)) {
73d0a4b1 1346 free_cpumask_var(shuffle_tmp_mask);
d84f5203
SV
1347 firsterr = PTR_ERR(shuffler_task);
1348 VERBOSE_PRINTK_ERRSTRING("Failed to create shuffler");
1349 shuffler_task = NULL;
1350 goto unwind;
1351 }
1352 }
d120f65f
PM
1353 if (stutter < 0)
1354 stutter = 0;
1355 if (stutter) {
1356 /* Create the stutter thread */
1357 stutter_task = kthread_run(rcu_torture_stutter, NULL,
1358 "rcu_torture_stutter");
1359 if (IS_ERR(stutter_task)) {
1360 firsterr = PTR_ERR(stutter_task);
1361 VERBOSE_PRINTK_ERRSTRING("Failed to create stutter");
1362 stutter_task = NULL;
1363 goto unwind;
1364 }
1365 }
bf66f18e
PM
1366 if (fqs_duration < 0)
1367 fqs_duration = 0;
1368 if (fqs_duration) {
1369 /* Create the stutter thread */
1370 fqs_task = kthread_run(rcu_torture_fqs, NULL,
1371 "rcu_torture_fqs");
1372 if (IS_ERR(fqs_task)) {
1373 firsterr = PTR_ERR(fqs_task);
1374 VERBOSE_PRINTK_ERRSTRING("Failed to create fqs");
1375 fqs_task = NULL;
1376 goto unwind;
1377 }
1378 }
343e9099
PM
1379 register_reboot_notifier(&rcutorture_nb);
1380 mutex_unlock(&fullstop_mutex);
a241ec65
PM
1381 return 0;
1382
1383unwind:
343e9099 1384 mutex_unlock(&fullstop_mutex);
a241ec65
PM
1385 rcu_torture_cleanup();
1386 return firsterr;
1387}
1388
1389module_init(rcu_torture_init);
1390module_exit(rcu_torture_cleanup);