1 // SPDX-License-Identifier: GPL-2.0
3 * Test module to generate lockups
5 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 #include <linux/kernel.h>
8 #include <linux/module.h>
9 #include <linux/delay.h>
10 #include <linux/sched.h>
11 #include <linux/sched/signal.h>
12 #include <linux/sched/clock.h>
13 #include <linux/cpu.h>
14 #include <linux/nmi.h>
16 #include <linux/uaccess.h>
17 #include <linux/file.h>
19 static unsigned int time_secs
;
20 module_param(time_secs
, uint
, 0600);
21 MODULE_PARM_DESC(time_secs
, "lockup time in seconds, default 0");
23 static unsigned int time_nsecs
;
24 module_param(time_nsecs
, uint
, 0600);
25 MODULE_PARM_DESC(time_nsecs
, "nanoseconds part of lockup time, default 0");
27 static unsigned int cooldown_secs
;
28 module_param(cooldown_secs
, uint
, 0600);
29 MODULE_PARM_DESC(cooldown_secs
, "cooldown time between iterations in seconds, default 0");
31 static unsigned int cooldown_nsecs
;
32 module_param(cooldown_nsecs
, uint
, 0600);
33 MODULE_PARM_DESC(cooldown_nsecs
, "nanoseconds part of cooldown, default 0");
35 static unsigned int iterations
= 1;
36 module_param(iterations
, uint
, 0600);
37 MODULE_PARM_DESC(iterations
, "lockup iterations, default 1");
40 module_param(all_cpus
, bool, 0400);
41 MODULE_PARM_DESC(all_cpus
, "trigger lockup at all cpus at once");
43 static int wait_state
;
44 static char *state
= "R";
45 module_param(state
, charp
, 0400);
46 MODULE_PARM_DESC(state
, "wait in 'R' running (default), 'D' uninterruptible, 'K' killable, 'S' interruptible state");
48 static bool use_hrtimer
;
49 module_param(use_hrtimer
, bool, 0400);
50 MODULE_PARM_DESC(use_hrtimer
, "use high-resolution timer for sleeping");
53 module_param(iowait
, bool, 0400);
54 MODULE_PARM_DESC(iowait
, "account sleep time as iowait");
56 static bool lock_read
;
57 module_param(lock_read
, bool, 0400);
58 MODULE_PARM_DESC(lock_read
, "lock read-write locks for read");
60 static bool lock_single
;
61 module_param(lock_single
, bool, 0400);
62 MODULE_PARM_DESC(lock_single
, "acquire locks only at one cpu");
64 static bool reacquire_locks
;
65 module_param(reacquire_locks
, bool, 0400);
66 MODULE_PARM_DESC(reacquire_locks
, "release and reacquire locks/irq/preempt between iterations");
68 static bool touch_softlockup
;
69 module_param(touch_softlockup
, bool, 0600);
70 MODULE_PARM_DESC(touch_softlockup
, "touch soft-lockup watchdog between iterations");
72 static bool touch_hardlockup
;
73 module_param(touch_hardlockup
, bool, 0600);
74 MODULE_PARM_DESC(touch_hardlockup
, "touch hard-lockup watchdog between iterations");
76 static bool call_cond_resched
;
77 module_param(call_cond_resched
, bool, 0600);
78 MODULE_PARM_DESC(call_cond_resched
, "call cond_resched() between iterations");
80 static bool measure_lock_wait
;
81 module_param(measure_lock_wait
, bool, 0400);
82 MODULE_PARM_DESC(measure_lock_wait
, "measure lock wait time");
84 static unsigned long lock_wait_threshold
= ULONG_MAX
;
85 module_param(lock_wait_threshold
, ulong
, 0400);
86 MODULE_PARM_DESC(lock_wait_threshold
, "print lock wait time longer than this in nanoseconds, default off");
88 static bool test_disable_irq
;
89 module_param_named(disable_irq
, test_disable_irq
, bool, 0400);
90 MODULE_PARM_DESC(disable_irq
, "disable interrupts: generate hard-lockups");
92 static bool disable_softirq
;
93 module_param(disable_softirq
, bool, 0400);
94 MODULE_PARM_DESC(disable_softirq
, "disable bottom-half irq handlers");
96 static bool disable_preempt
;
97 module_param(disable_preempt
, bool, 0400);
98 MODULE_PARM_DESC(disable_preempt
, "disable preemption: generate soft-lockups");
100 static bool lock_rcu
;
101 module_param(lock_rcu
, bool, 0400);
102 MODULE_PARM_DESC(lock_rcu
, "grab rcu_read_lock: generate rcu stalls");
104 static bool lock_mmap_sem
;
105 module_param(lock_mmap_sem
, bool, 0400);
106 MODULE_PARM_DESC(lock_mmap_sem
, "lock mm->mmap_sem: block procfs interfaces");
108 static unsigned long lock_rwsem_ptr
;
109 module_param_unsafe(lock_rwsem_ptr
, ulong
, 0400);
110 MODULE_PARM_DESC(lock_rwsem_ptr
, "lock rw_semaphore at address");
112 static unsigned long lock_mutex_ptr
;
113 module_param_unsafe(lock_mutex_ptr
, ulong
, 0400);
114 MODULE_PARM_DESC(lock_mutex_ptr
, "lock mutex at address");
116 static unsigned long lock_spinlock_ptr
;
117 module_param_unsafe(lock_spinlock_ptr
, ulong
, 0400);
118 MODULE_PARM_DESC(lock_spinlock_ptr
, "lock spinlock at address");
120 static unsigned long lock_rwlock_ptr
;
121 module_param_unsafe(lock_rwlock_ptr
, ulong
, 0400);
122 MODULE_PARM_DESC(lock_rwlock_ptr
, "lock rwlock at address");
124 static unsigned int alloc_pages_nr
;
125 module_param_unsafe(alloc_pages_nr
, uint
, 0600);
126 MODULE_PARM_DESC(alloc_pages_nr
, "allocate and free pages under locks");
128 static unsigned int alloc_pages_order
;
129 module_param(alloc_pages_order
, uint
, 0400);
130 MODULE_PARM_DESC(alloc_pages_order
, "page order to allocate");
132 static gfp_t alloc_pages_gfp
= GFP_KERNEL
;
133 module_param_unsafe(alloc_pages_gfp
, uint
, 0400);
134 MODULE_PARM_DESC(alloc_pages_gfp
, "allocate pages with this gfp_mask, default GFP_KERNEL");
136 static bool alloc_pages_atomic
;
137 module_param(alloc_pages_atomic
, bool, 0400);
138 MODULE_PARM_DESC(alloc_pages_atomic
, "allocate pages with GFP_ATOMIC");
140 static bool reallocate_pages
;
141 module_param(reallocate_pages
, bool, 0400);
142 MODULE_PARM_DESC(reallocate_pages
, "free and allocate pages between iterations");
144 struct file
*test_file
;
145 static struct inode
*test_inode
;
146 static char test_file_path
[256];
147 module_param_string(file_path
, test_file_path
, sizeof(test_file_path
), 0400);
148 MODULE_PARM_DESC(file_path
, "file path to test");
150 static bool test_lock_inode
;
151 module_param_named(lock_inode
, test_lock_inode
, bool, 0400);
152 MODULE_PARM_DESC(lock_inode
, "lock file -> inode -> i_rwsem");
154 static bool test_lock_mapping
;
155 module_param_named(lock_mapping
, test_lock_mapping
, bool, 0400);
156 MODULE_PARM_DESC(lock_mapping
, "lock file -> mapping -> i_mmap_rwsem");
158 static bool test_lock_sb_umount
;
159 module_param_named(lock_sb_umount
, test_lock_sb_umount
, bool, 0400);
160 MODULE_PARM_DESC(lock_sb_umount
, "lock file -> sb -> s_umount");
162 static atomic_t alloc_pages_failed
= ATOMIC_INIT(0);
164 static atomic64_t max_lock_wait
= ATOMIC64_INIT(0);
166 static struct task_struct
*main_task
;
167 static int master_cpu
;
169 static void test_lock(bool master
, bool verbose
)
171 u64
uninitialized_var(wait_start
);
173 if (measure_lock_wait
)
174 wait_start
= local_clock();
176 if (lock_mutex_ptr
&& master
) {
178 pr_notice("lock mutex %ps\n", (void *)lock_mutex_ptr
);
179 mutex_lock((struct mutex
*)lock_mutex_ptr
);
182 if (lock_rwsem_ptr
&& master
) {
184 pr_notice("lock rw_semaphore %ps\n",
185 (void *)lock_rwsem_ptr
);
187 down_read((struct rw_semaphore
*)lock_rwsem_ptr
);
189 down_write((struct rw_semaphore
*)lock_rwsem_ptr
);
192 if (lock_mmap_sem
&& master
) {
194 pr_notice("lock mmap_sem pid=%d\n", main_task
->pid
);
196 mmap_read_lock(main_task
->mm
);
198 mmap_write_lock(main_task
->mm
);
201 if (test_disable_irq
)
213 if (lock_spinlock_ptr
&& master
) {
215 pr_notice("lock spinlock %ps\n",
216 (void *)lock_spinlock_ptr
);
217 spin_lock((spinlock_t
*)lock_spinlock_ptr
);
220 if (lock_rwlock_ptr
&& master
) {
222 pr_notice("lock rwlock %ps\n",
223 (void *)lock_rwlock_ptr
);
225 read_lock((rwlock_t
*)lock_rwlock_ptr
);
227 write_lock((rwlock_t
*)lock_rwlock_ptr
);
230 if (measure_lock_wait
) {
231 s64 cur_wait
= local_clock() - wait_start
;
232 s64 max_wait
= atomic64_read(&max_lock_wait
);
235 if (cur_wait
< max_wait
)
237 max_wait
= atomic64_cmpxchg(&max_lock_wait
,
239 } while (max_wait
!= cur_wait
);
241 if (cur_wait
> lock_wait_threshold
)
242 pr_notice_ratelimited("lock wait %lld ns\n", cur_wait
);
246 static void test_unlock(bool master
, bool verbose
)
248 if (lock_rwlock_ptr
&& master
) {
250 read_unlock((rwlock_t
*)lock_rwlock_ptr
);
252 write_unlock((rwlock_t
*)lock_rwlock_ptr
);
254 pr_notice("unlock rwlock %ps\n",
255 (void *)lock_rwlock_ptr
);
258 if (lock_spinlock_ptr
&& master
) {
259 spin_unlock((spinlock_t
*)lock_spinlock_ptr
);
261 pr_notice("unlock spinlock %ps\n",
262 (void *)lock_spinlock_ptr
);
274 if (test_disable_irq
)
277 if (lock_mmap_sem
&& master
) {
279 mmap_read_unlock(main_task
->mm
);
281 mmap_write_unlock(main_task
->mm
);
283 pr_notice("unlock mmap_sem pid=%d\n", main_task
->pid
);
286 if (lock_rwsem_ptr
&& master
) {
288 up_read((struct rw_semaphore
*)lock_rwsem_ptr
);
290 up_write((struct rw_semaphore
*)lock_rwsem_ptr
);
292 pr_notice("unlock rw_semaphore %ps\n",
293 (void *)lock_rwsem_ptr
);
296 if (lock_mutex_ptr
&& master
) {
297 mutex_unlock((struct mutex
*)lock_mutex_ptr
);
299 pr_notice("unlock mutex %ps\n",
300 (void *)lock_mutex_ptr
);
304 static void test_alloc_pages(struct list_head
*pages
)
309 for (i
= 0; i
< alloc_pages_nr
; i
++) {
310 page
= alloc_pages(alloc_pages_gfp
, alloc_pages_order
);
312 atomic_inc(&alloc_pages_failed
);
315 list_add(&page
->lru
, pages
);
319 static void test_free_pages(struct list_head
*pages
)
321 struct page
*page
, *next
;
323 list_for_each_entry_safe(page
, next
, pages
, lru
)
324 __free_pages(page
, alloc_pages_order
);
325 INIT_LIST_HEAD(pages
);
328 static void test_wait(unsigned int secs
, unsigned int nsecs
)
330 if (wait_state
== TASK_RUNNING
) {
332 mdelay(secs
* MSEC_PER_SEC
);
338 __set_current_state(wait_state
);
342 time
= ns_to_ktime((u64
)secs
* NSEC_PER_SEC
+ nsecs
);
343 schedule_hrtimeout(&time
, HRTIMER_MODE_REL
);
345 schedule_timeout(secs
* HZ
+ nsecs_to_jiffies(nsecs
));
349 static void test_lockup(bool master
)
351 u64 lockup_start
= local_clock();
352 unsigned int iter
= 0;
355 pr_notice("Start on CPU%d\n", raw_smp_processor_id());
357 test_lock(master
, true);
359 test_alloc_pages(&pages
);
361 while (iter
++ < iterations
&& !signal_pending(main_task
)) {
364 current
->in_iowait
= 1;
366 test_wait(time_secs
, time_nsecs
);
369 current
->in_iowait
= 0;
371 if (reallocate_pages
)
372 test_free_pages(&pages
);
375 test_unlock(master
, false);
377 if (touch_softlockup
)
378 touch_softlockup_watchdog();
380 if (touch_hardlockup
)
381 touch_nmi_watchdog();
383 if (call_cond_resched
)
386 test_wait(cooldown_secs
, cooldown_nsecs
);
389 test_lock(master
, false);
391 if (reallocate_pages
)
392 test_alloc_pages(&pages
);
395 pr_notice("Finish on CPU%d in %lld ns\n", raw_smp_processor_id(),
396 local_clock() - lockup_start
);
398 test_free_pages(&pages
);
400 test_unlock(master
, true);
403 DEFINE_PER_CPU(struct work_struct
, test_works
);
405 static void test_work_fn(struct work_struct
*work
)
407 test_lockup(!lock_single
||
408 work
== per_cpu_ptr(&test_works
, master_cpu
));
411 static bool test_kernel_ptr(unsigned long addr
, int size
)
413 void *ptr
= (void *)addr
;
419 /* should be at least readable kernel address */
420 if (access_ok(ptr
, 1) ||
421 access_ok(ptr
+ size
- 1, 1) ||
422 probe_kernel_address(ptr
, buf
) ||
423 probe_kernel_address(ptr
+ size
- 1, buf
)) {
424 pr_err("invalid kernel ptr: %#lx\n", addr
);
431 static bool __maybe_unused
test_magic(unsigned long addr
, int offset
,
432 unsigned int expected
)
434 void *ptr
= (void *)addr
+ offset
;
435 unsigned int magic
= 0;
440 if (probe_kernel_address(ptr
, magic
) || magic
!= expected
) {
441 pr_err("invalid magic at %#lx + %#x = %#x, expected %#x\n",
442 addr
, offset
, magic
, expected
);
449 static int __init
test_lockup_init(void)
451 u64 test_start
= local_clock();
457 wait_state
= TASK_INTERRUPTIBLE
;
460 wait_state
= TASK_UNINTERRUPTIBLE
;
463 wait_state
= TASK_KILLABLE
;
466 wait_state
= TASK_RUNNING
;
469 pr_err("unknown state=%s\n", state
);
473 if (alloc_pages_atomic
)
474 alloc_pages_gfp
= GFP_ATOMIC
;
476 if (test_kernel_ptr(lock_spinlock_ptr
, sizeof(spinlock_t
)) ||
477 test_kernel_ptr(lock_rwlock_ptr
, sizeof(rwlock_t
)) ||
478 test_kernel_ptr(lock_mutex_ptr
, sizeof(struct mutex
)) ||
479 test_kernel_ptr(lock_rwsem_ptr
, sizeof(struct rw_semaphore
)))
482 #ifdef CONFIG_DEBUG_SPINLOCK
483 if (test_magic(lock_spinlock_ptr
,
484 offsetof(spinlock_t
, rlock
.magic
),
486 test_magic(lock_rwlock_ptr
,
487 offsetof(rwlock_t
, magic
),
489 test_magic(lock_mutex_ptr
,
490 offsetof(struct mutex
, wait_lock
.rlock
.magic
),
492 test_magic(lock_rwsem_ptr
,
493 offsetof(struct rw_semaphore
, wait_lock
.magic
),
498 if ((wait_state
!= TASK_RUNNING
||
499 (call_cond_resched
&& !reacquire_locks
) ||
500 (alloc_pages_nr
&& gfpflags_allow_blocking(alloc_pages_gfp
))) &&
501 (test_disable_irq
|| disable_softirq
|| disable_preempt
||
502 lock_rcu
|| lock_spinlock_ptr
|| lock_rwlock_ptr
)) {
503 pr_err("refuse to sleep in atomic context\n");
507 if (lock_mmap_sem
&& !main_task
->mm
) {
508 pr_err("no mm to lock mmap_sem\n");
512 if (test_file_path
[0]) {
513 test_file
= filp_open(test_file_path
, O_RDONLY
, 0);
514 if (IS_ERR(test_file
)) {
515 pr_err("cannot find file_path\n");
518 test_inode
= file_inode(test_file
);
519 } else if (test_lock_inode
||
521 test_lock_sb_umount
) {
522 pr_err("no file to lock\n");
526 if (test_lock_inode
&& test_inode
)
527 lock_rwsem_ptr
= (unsigned long)&test_inode
->i_rwsem
;
529 if (test_lock_mapping
&& test_file
&& test_file
->f_mapping
)
530 lock_rwsem_ptr
= (unsigned long)&test_file
->f_mapping
->i_mmap_rwsem
;
532 if (test_lock_sb_umount
&& test_inode
)
533 lock_rwsem_ptr
= (unsigned long)&test_inode
->i_sb
->s_umount
;
535 pr_notice("START pid=%d time=%u +%u ns cooldown=%u +%u ns iterations=%u state=%s %s%s%s%s%s%s%s%s%s%s%s\n",
536 main_task
->pid
, time_secs
, time_nsecs
,
537 cooldown_secs
, cooldown_nsecs
, iterations
, state
,
538 all_cpus
? "all_cpus " : "",
539 iowait
? "iowait " : "",
540 test_disable_irq
? "disable_irq " : "",
541 disable_softirq
? "disable_softirq " : "",
542 disable_preempt
? "disable_preempt " : "",
543 lock_rcu
? "lock_rcu " : "",
544 lock_read
? "lock_read " : "",
545 touch_softlockup
? "touch_softlockup " : "",
546 touch_hardlockup
? "touch_hardlockup " : "",
547 call_cond_resched
? "call_cond_resched " : "",
548 reacquire_locks
? "reacquire_locks " : "");
551 pr_notice("ALLOCATE PAGES nr=%u order=%u gfp=%pGg %s\n",
552 alloc_pages_nr
, alloc_pages_order
, &alloc_pages_gfp
,
553 reallocate_pages
? "reallocate_pages " : "");
561 master_cpu
= smp_processor_id();
562 for_each_online_cpu(cpu
) {
563 INIT_WORK(per_cpu_ptr(&test_works
, cpu
), test_work_fn
);
564 queue_work_on(cpu
, system_highpri_wq
,
565 per_cpu_ptr(&test_works
, cpu
));
569 for_each_online_cpu(cpu
)
570 flush_work(per_cpu_ptr(&test_works
, cpu
));
577 if (measure_lock_wait
)
578 pr_notice("Maximum lock wait: %lld ns\n",
579 atomic64_read(&max_lock_wait
));
582 pr_notice("Page allocation failed %u times\n",
583 atomic_read(&alloc_pages_failed
));
585 pr_notice("FINISH in %llu ns\n", local_clock() - test_start
);
590 if (signal_pending(main_task
))
595 module_init(test_lockup_init
);
597 MODULE_LICENSE("GPL");
598 MODULE_AUTHOR("Konstantin Khlebnikov <khlebnikov@yandex-team.ru>");
599 MODULE_DESCRIPTION("Test module to generate lockups");