1 // SPDX-License-Identifier: GPL-2.0
4 * Test module for stress and analyze performance of vmalloc allocator.
5 * (C) 2018 Uladzislau Rezki (Sony) <urezki@gmail.com>
7 #include <linux/init.h>
8 #include <linux/kernel.h>
9 #include <linux/module.h>
10 #include <linux/vmalloc.h>
11 #include <linux/random.h>
12 #include <linux/kthread.h>
13 #include <linux/moduleparam.h>
14 #include <linux/completion.h>
15 #include <linux/delay.h>
16 #include <linux/rwsem.h>
18 #include <linux/rcupdate.h>
19 #include <linux/slab.h>
21 #define __param(type, name, init, msg) \
22 static type name = init; \
23 module_param(name, type, 0444); \
24 MODULE_PARM_DESC(name, msg) \
26 __param(int, nr_threads, 0,
27 "Number of workers to perform tests(min: 1 max: USHRT_MAX)");
29 __param(bool, sequential_test_order
, false,
30 "Use sequential stress tests order");
32 __param(int, test_repeat_count
, 1,
33 "Set test repeat counter");
35 __param(int, test_loop_count
, 1000000,
36 "Set test loop counter");
38 __param(int, run_test_mask
, INT_MAX
,
39 "Set tests specified in the mask.\n\n"
40 "\t\tid: 1, name: fix_size_alloc_test\n"
41 "\t\tid: 2, name: full_fit_alloc_test\n"
42 "\t\tid: 4, name: long_busy_list_alloc_test\n"
43 "\t\tid: 8, name: random_size_alloc_test\n"
44 "\t\tid: 16, name: fix_align_alloc_test\n"
45 "\t\tid: 32, name: random_size_align_alloc_test\n"
46 "\t\tid: 64, name: align_shift_alloc_test\n"
47 "\t\tid: 128, name: pcpu_alloc_test\n"
48 "\t\tid: 256, name: kvfree_rcu_1_arg_vmalloc_test\n"
49 "\t\tid: 512, name: kvfree_rcu_2_arg_vmalloc_test\n"
50 /* Add a new test case description here. */
54 * Read write semaphore for synchronization of setup
55 * phase that is done in main thread and workers.
57 static DECLARE_RWSEM(prepare_for_test_rwsem
);
60 * Completion tracking for worker threads.
62 static DECLARE_COMPLETION(test_all_done_comp
);
63 static atomic_t test_n_undone
= ATOMIC_INIT(0);
66 test_report_one_done(void)
68 if (atomic_dec_and_test(&test_n_undone
))
69 complete(&test_all_done_comp
);
72 static int random_size_align_alloc_test(void)
74 unsigned long size
, align
, rnd
;
78 for (i
= 0; i
< test_loop_count
; i
++) {
79 get_random_bytes(&rnd
, sizeof(rnd
));
82 * Maximum 1024 pages, if PAGE_SIZE is 4096.
84 align
= 1 << (rnd
% 23);
89 size
= ((rnd
% 10) + 1) * PAGE_SIZE
;
91 ptr
= __vmalloc_node(size
, align
, GFP_KERNEL
| __GFP_ZERO
, 0,
92 __builtin_return_address(0));
103 * This test case is supposed to be failed.
105 static int align_shift_alloc_test(void)
111 for (i
= 0; i
< BITS_PER_LONG
; i
++) {
112 align
= ((unsigned long) 1) << i
;
114 ptr
= __vmalloc_node(PAGE_SIZE
, align
, GFP_KERNEL
|__GFP_ZERO
, 0,
115 __builtin_return_address(0));
125 static int fix_align_alloc_test(void)
130 for (i
= 0; i
< test_loop_count
; i
++) {
131 ptr
= __vmalloc_node(5 * PAGE_SIZE
, THREAD_ALIGN
<< 1,
132 GFP_KERNEL
| __GFP_ZERO
, 0,
133 __builtin_return_address(0));
143 static int random_size_alloc_test(void)
149 for (i
= 0; i
< test_loop_count
; i
++) {
150 get_random_bytes(&n
, sizeof(i
));
153 p
= vmalloc(n
* PAGE_SIZE
);
165 static int long_busy_list_alloc_test(void)
172 ptr
= vmalloc(sizeof(void *) * 15000);
176 for (i
= 0; i
< 15000; i
++)
177 ptr
[i
] = vmalloc(1 * PAGE_SIZE
);
179 for (i
= 0; i
< test_loop_count
; i
++) {
180 ptr_1
= vmalloc(100 * PAGE_SIZE
);
184 ptr_2
= vmalloc(1 * PAGE_SIZE
);
190 *((__u8
*)ptr_1
) = 0;
191 *((__u8
*)ptr_2
) = 1;
201 for (i
= 0; i
< 15000; i
++)
208 static int full_fit_alloc_test(void)
210 void **ptr
, **junk_ptr
, *tmp
;
215 junk_length
= fls(num_online_cpus());
216 junk_length
*= (32 * 1024 * 1024 / PAGE_SIZE
);
218 ptr
= vmalloc(sizeof(void *) * junk_length
);
222 junk_ptr
= vmalloc(sizeof(void *) * junk_length
);
228 for (i
= 0; i
< junk_length
; i
++) {
229 ptr
[i
] = vmalloc(1 * PAGE_SIZE
);
230 junk_ptr
[i
] = vmalloc(1 * PAGE_SIZE
);
233 for (i
= 0; i
< junk_length
; i
++)
236 for (i
= 0; i
< test_loop_count
; i
++) {
237 tmp
= vmalloc(1 * PAGE_SIZE
);
250 for (i
= 0; i
< junk_length
; i
++)
259 static int fix_size_alloc_test(void)
264 for (i
= 0; i
< test_loop_count
; i
++) {
265 ptr
= vmalloc(3 * PAGE_SIZE
);
279 pcpu_alloc_test(void)
282 #ifndef CONFIG_NEED_PER_CPU_KM
283 void __percpu
**pcpu
;
287 pcpu
= vmalloc(sizeof(void __percpu
*) * 35000);
291 for (i
= 0; i
< 35000; i
++) {
294 get_random_bytes(&r
, sizeof(i
));
295 size
= (r
% (PAGE_SIZE
/ 4)) + 1;
300 get_random_bytes(&r
, sizeof(i
));
301 align
= 1 << ((i
% 11) + 1);
303 pcpu
[i
] = __alloc_percpu(size
, align
);
308 for (i
= 0; i
< 35000; i
++)
309 free_percpu(pcpu
[i
]);
316 struct test_kvfree_rcu
{
318 unsigned char array
[20];
322 kvfree_rcu_1_arg_vmalloc_test(void)
324 struct test_kvfree_rcu
*p
;
327 for (i
= 0; i
< test_loop_count
; i
++) {
328 p
= vmalloc(1 * PAGE_SIZE
);
340 kvfree_rcu_2_arg_vmalloc_test(void)
342 struct test_kvfree_rcu
*p
;
345 for (i
= 0; i
< test_loop_count
; i
++) {
346 p
= vmalloc(1 * PAGE_SIZE
);
357 struct test_case_desc
{
358 const char *test_name
;
359 int (*test_func
)(void);
362 static struct test_case_desc test_case_array
[] = {
363 { "fix_size_alloc_test", fix_size_alloc_test
},
364 { "full_fit_alloc_test", full_fit_alloc_test
},
365 { "long_busy_list_alloc_test", long_busy_list_alloc_test
},
366 { "random_size_alloc_test", random_size_alloc_test
},
367 { "fix_align_alloc_test", fix_align_alloc_test
},
368 { "random_size_align_alloc_test", random_size_align_alloc_test
},
369 { "align_shift_alloc_test", align_shift_alloc_test
},
370 { "pcpu_alloc_test", pcpu_alloc_test
},
371 { "kvfree_rcu_1_arg_vmalloc_test", kvfree_rcu_1_arg_vmalloc_test
},
372 { "kvfree_rcu_2_arg_vmalloc_test", kvfree_rcu_2_arg_vmalloc_test
},
373 /* Add a new test case here. */
376 struct test_case_data
{
382 static struct test_driver
{
383 struct task_struct
*task
;
384 struct test_case_data data
[ARRAY_SIZE(test_case_array
)];
390 static void shuffle_array(int *arr
, int n
)
395 for (i
= n
- 1; i
> 0; i
--) {
396 get_random_bytes(&rnd
, sizeof(rnd
));
408 static int test_func(void *private)
410 struct test_driver
*t
= private;
411 int random_array
[ARRAY_SIZE(test_case_array
)];
416 for (i
= 0; i
< ARRAY_SIZE(test_case_array
); i
++)
419 if (!sequential_test_order
)
420 shuffle_array(random_array
, ARRAY_SIZE(test_case_array
));
423 * Block until initialization is done.
425 down_read(&prepare_for_test_rwsem
);
427 t
->start
= get_cycles();
428 for (i
= 0; i
< ARRAY_SIZE(test_case_array
); i
++) {
429 index
= random_array
[i
];
432 * Skip tests if run_test_mask has been specified.
434 if (!((run_test_mask
& (1 << index
)) >> index
))
438 for (j
= 0; j
< test_repeat_count
; j
++) {
439 if (!test_case_array
[index
].test_func())
440 t
->data
[index
].test_passed
++;
442 t
->data
[index
].test_failed
++;
446 * Take an average time that test took.
448 delta
= (u64
) ktime_us_delta(ktime_get(), kt
);
449 do_div(delta
, (u32
) test_repeat_count
);
451 t
->data
[index
].time
= delta
;
453 t
->stop
= get_cycles();
455 up_read(&prepare_for_test_rwsem
);
456 test_report_one_done();
459 * Wait for the kthread_stop() call.
461 while (!kthread_should_stop())
468 init_test_configurtion(void)
471 * A maximum number of workers is defined as hard-coded
472 * value and set to USHRT_MAX. We add such gap just in
473 * case and for potential heavy stressing.
475 nr_threads
= clamp(nr_threads
, 1, (int) USHRT_MAX
);
477 /* Allocate the space for test instances. */
478 tdriver
= kvcalloc(nr_threads
, sizeof(*tdriver
), GFP_KERNEL
);
482 if (test_repeat_count
<= 0)
483 test_repeat_count
= 1;
485 if (test_loop_count
<= 0)
491 static void do_concurrent_test(void)
496 * Set some basic configurations plus sanity check.
498 ret
= init_test_configurtion();
503 * Put on hold all workers.
505 down_write(&prepare_for_test_rwsem
);
507 for (i
= 0; i
< nr_threads
; i
++) {
508 struct test_driver
*t
= &tdriver
[i
];
510 t
->task
= kthread_run(test_func
, t
, "vmalloc_test/%d", i
);
512 if (!IS_ERR(t
->task
))
514 atomic_inc(&test_n_undone
);
516 pr_err("Failed to start %d kthread\n", i
);
520 * Now let the workers do their job.
522 up_write(&prepare_for_test_rwsem
);
525 * Sleep quiet until all workers are done with 1 second
526 * interval. Since the test can take a lot of time we
527 * can run into a stack trace of the hung task. That is
528 * why we go with completion_timeout and HZ value.
531 ret
= wait_for_completion_timeout(&test_all_done_comp
, HZ
);
534 for (i
= 0; i
< nr_threads
; i
++) {
535 struct test_driver
*t
= &tdriver
[i
];
538 if (!IS_ERR(t
->task
))
539 kthread_stop(t
->task
);
541 for (j
= 0; j
< ARRAY_SIZE(test_case_array
); j
++) {
542 if (!((run_test_mask
& (1 << j
)) >> j
))
546 "Summary: %s passed: %d failed: %d repeat: %d loops: %d avg: %llu usec\n",
547 test_case_array
[j
].test_name
,
548 t
->data
[j
].test_passed
,
549 t
->data
[j
].test_failed
,
550 test_repeat_count
, test_loop_count
,
554 pr_info("All test took worker%d=%lu cycles\n",
555 i
, t
->stop
- t
->start
);
561 static int vmalloc_test_init(void)
563 do_concurrent_test();
564 return -EAGAIN
; /* Fail will directly unload the module */
567 static void vmalloc_test_exit(void)
571 module_init(vmalloc_test_init
)
572 module_exit(vmalloc_test_exit
)
574 MODULE_LICENSE("GPL");
575 MODULE_AUTHOR("Uladzislau Rezki");
576 MODULE_DESCRIPTION("vmalloc test module");