1 /*****************************************************************************\
2 * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
3 * Copyright (C) 2007 The Regents of the University of California.
4 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
5 * Written by Brian Behlendorf <behlendorf1@llnl.gov>.
8 * This file is part of the SPL, Solaris Porting Layer.
9 * For details, see <http://zfsonlinux.org/>.
11 * The SPL is free software; you can redistribute it and/or modify it
12 * under the terms of the GNU General Public License as published by the
13 * Free Software Foundation; either version 2 of the License, or (at your
14 * option) any later version.
16 * The SPL is distributed in the hope that it will be useful, but WITHOUT
17 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 * You should have received a copy of the GNU General Public License along
22 * with the SPL. If not, see <http://www.gnu.org/licenses/>.
23 *****************************************************************************
24 * Solaris Porting LAyer Tests (SPLAT) Kmem Tests.
25 \*****************************************************************************/
28 #include <sys/thread.h>
29 #include "splat-internal.h"
31 #define SPLAT_KMEM_NAME "kmem"
32 #define SPLAT_KMEM_DESC "Kernel Malloc/Slab Tests"
34 #define SPLAT_KMEM_TEST1_ID 0x0101
35 #define SPLAT_KMEM_TEST1_NAME "kmem_alloc"
36 #define SPLAT_KMEM_TEST1_DESC "Memory allocation test (kmem_alloc)"
38 #define SPLAT_KMEM_TEST2_ID 0x0102
39 #define SPLAT_KMEM_TEST2_NAME "kmem_zalloc"
40 #define SPLAT_KMEM_TEST2_DESC "Memory allocation test (kmem_zalloc)"
42 #define SPLAT_KMEM_TEST3_ID 0x0103
43 #define SPLAT_KMEM_TEST3_NAME "vmem_alloc"
44 #define SPLAT_KMEM_TEST3_DESC "Memory allocation test (vmem_alloc)"
46 #define SPLAT_KMEM_TEST4_ID 0x0104
47 #define SPLAT_KMEM_TEST4_NAME "vmem_zalloc"
48 #define SPLAT_KMEM_TEST4_DESC "Memory allocation test (vmem_zalloc)"
50 #define SPLAT_KMEM_TEST5_ID 0x0105
51 #define SPLAT_KMEM_TEST5_NAME "slab_small"
52 #define SPLAT_KMEM_TEST5_DESC "Slab ctor/dtor test (small)"
54 #define SPLAT_KMEM_TEST6_ID 0x0106
55 #define SPLAT_KMEM_TEST6_NAME "slab_large"
56 #define SPLAT_KMEM_TEST6_DESC "Slab ctor/dtor test (large)"
58 #define SPLAT_KMEM_TEST7_ID 0x0107
59 #define SPLAT_KMEM_TEST7_NAME "slab_align"
60 #define SPLAT_KMEM_TEST7_DESC "Slab alignment test"
62 #define SPLAT_KMEM_TEST8_ID 0x0108
63 #define SPLAT_KMEM_TEST8_NAME "slab_reap"
64 #define SPLAT_KMEM_TEST8_DESC "Slab reaping test"
66 #define SPLAT_KMEM_TEST9_ID 0x0109
67 #define SPLAT_KMEM_TEST9_NAME "slab_age"
68 #define SPLAT_KMEM_TEST9_DESC "Slab aging test"
70 #define SPLAT_KMEM_TEST10_ID 0x010a
71 #define SPLAT_KMEM_TEST10_NAME "slab_lock"
72 #define SPLAT_KMEM_TEST10_DESC "Slab locking test"
75 #define SPLAT_KMEM_TEST11_ID 0x010b
76 #define SPLAT_KMEM_TEST11_NAME "slab_overcommit"
77 #define SPLAT_KMEM_TEST11_DESC "Slab memory overcommit test"
80 #define SPLAT_KMEM_TEST13_ID 0x010d
81 #define SPLAT_KMEM_TEST13_NAME "slab_reclaim"
82 #define SPLAT_KMEM_TEST13_DESC "Slab direct memory reclaim test"
84 #define SPLAT_KMEM_ALLOC_COUNT 10
85 #define SPLAT_VMEM_ALLOC_COUNT 10
89 splat_kmem_test1(struct file
*file
, void *arg
)
91 void *ptr
[SPLAT_KMEM_ALLOC_COUNT
];
95 while ((!rc
) && (size
<= (PAGE_SIZE
* 32))) {
98 for (i
= 0; i
< SPLAT_KMEM_ALLOC_COUNT
; i
++) {
99 ptr
[i
] = kmem_alloc(size
, KM_SLEEP
| KM_NODEBUG
);
104 for (i
= 0; i
< SPLAT_KMEM_ALLOC_COUNT
; i
++)
106 kmem_free(ptr
[i
], size
);
108 splat_vprint(file
, SPLAT_KMEM_TEST1_NAME
,
109 "%d byte allocations, %d/%d successful\n",
110 size
, count
, SPLAT_KMEM_ALLOC_COUNT
);
111 if (count
!= SPLAT_KMEM_ALLOC_COUNT
)
121 splat_kmem_test2(struct file
*file
, void *arg
)
123 void *ptr
[SPLAT_KMEM_ALLOC_COUNT
];
124 int size
= PAGE_SIZE
;
125 int i
, j
, count
, rc
= 0;
127 while ((!rc
) && (size
<= (PAGE_SIZE
* 32))) {
130 for (i
= 0; i
< SPLAT_KMEM_ALLOC_COUNT
; i
++) {
131 ptr
[i
] = kmem_zalloc(size
, KM_SLEEP
| KM_NODEBUG
);
136 /* Ensure buffer has been zero filled */
137 for (i
= 0; i
< SPLAT_KMEM_ALLOC_COUNT
; i
++) {
138 for (j
= 0; j
< size
; j
++) {
139 if (((char *)ptr
[i
])[j
] != '\0') {
140 splat_vprint(file
,SPLAT_KMEM_TEST2_NAME
,
141 "%d-byte allocation was "
142 "not zeroed\n", size
);
148 for (i
= 0; i
< SPLAT_KMEM_ALLOC_COUNT
; i
++)
150 kmem_free(ptr
[i
], size
);
152 splat_vprint(file
, SPLAT_KMEM_TEST2_NAME
,
153 "%d byte allocations, %d/%d successful\n",
154 size
, count
, SPLAT_KMEM_ALLOC_COUNT
);
155 if (count
!= SPLAT_KMEM_ALLOC_COUNT
)
165 splat_kmem_test3(struct file
*file
, void *arg
)
167 void *ptr
[SPLAT_VMEM_ALLOC_COUNT
];
168 int size
= PAGE_SIZE
;
169 int i
, count
, rc
= 0;
171 while ((!rc
) && (size
<= (PAGE_SIZE
* 1024))) {
174 for (i
= 0; i
< SPLAT_VMEM_ALLOC_COUNT
; i
++) {
175 ptr
[i
] = vmem_alloc(size
, KM_SLEEP
);
180 for (i
= 0; i
< SPLAT_VMEM_ALLOC_COUNT
; i
++)
182 vmem_free(ptr
[i
], size
);
184 splat_vprint(file
, SPLAT_KMEM_TEST3_NAME
,
185 "%d byte allocations, %d/%d successful\n",
186 size
, count
, SPLAT_VMEM_ALLOC_COUNT
);
187 if (count
!= SPLAT_VMEM_ALLOC_COUNT
)
197 splat_kmem_test4(struct file
*file
, void *arg
)
199 void *ptr
[SPLAT_VMEM_ALLOC_COUNT
];
200 int size
= PAGE_SIZE
;
201 int i
, j
, count
, rc
= 0;
203 while ((!rc
) && (size
<= (PAGE_SIZE
* 1024))) {
206 for (i
= 0; i
< SPLAT_VMEM_ALLOC_COUNT
; i
++) {
207 ptr
[i
] = vmem_zalloc(size
, KM_SLEEP
);
212 /* Ensure buffer has been zero filled */
213 for (i
= 0; i
< SPLAT_VMEM_ALLOC_COUNT
; i
++) {
214 for (j
= 0; j
< size
; j
++) {
215 if (((char *)ptr
[i
])[j
] != '\0') {
216 splat_vprint(file
, SPLAT_KMEM_TEST4_NAME
,
217 "%d-byte allocation was "
218 "not zeroed\n", size
);
224 for (i
= 0; i
< SPLAT_VMEM_ALLOC_COUNT
; i
++)
226 vmem_free(ptr
[i
], size
);
228 splat_vprint(file
, SPLAT_KMEM_TEST4_NAME
,
229 "%d byte allocations, %d/%d successful\n",
230 size
, count
, SPLAT_VMEM_ALLOC_COUNT
);
231 if (count
!= SPLAT_VMEM_ALLOC_COUNT
)
240 #define SPLAT_KMEM_TEST_MAGIC 0x004488CCUL
241 #define SPLAT_KMEM_CACHE_NAME "kmem_test"
242 #define SPLAT_KMEM_OBJ_COUNT 1024
243 #define SPLAT_KMEM_OBJ_RECLAIM 32 /* objects */
244 #define SPLAT_KMEM_THREADS 32
246 #define KCP_FLAG_READY 0x01
248 typedef struct kmem_cache_data
{
249 unsigned long kcd_magic
;
250 struct list_head kcd_node
;
255 typedef struct kmem_cache_thread
{
258 struct list_head kct_list
;
259 } kmem_cache_thread_t
;
261 typedef struct kmem_cache_priv
{
262 unsigned long kcp_magic
;
263 struct file
*kcp_file
;
264 kmem_cache_t
*kcp_cache
;
266 wait_queue_head_t kcp_ctl_waitq
;
267 wait_queue_head_t kcp_thr_waitq
;
270 kmem_cache_thread_t
*kcp_kct
[SPLAT_KMEM_THREADS
];
278 static kmem_cache_priv_t
*
279 splat_kmem_cache_test_kcp_alloc(struct file
*file
, char *name
,
280 int size
, int align
, int alloc
)
282 kmem_cache_priv_t
*kcp
;
284 kcp
= kmem_zalloc(sizeof(kmem_cache_priv_t
), KM_SLEEP
);
288 kcp
->kcp_magic
= SPLAT_KMEM_TEST_MAGIC
;
289 kcp
->kcp_file
= file
;
290 kcp
->kcp_cache
= NULL
;
291 spin_lock_init(&kcp
->kcp_lock
);
292 init_waitqueue_head(&kcp
->kcp_ctl_waitq
);
293 init_waitqueue_head(&kcp
->kcp_thr_waitq
);
295 kcp
->kcp_kct_count
= -1;
296 kcp
->kcp_size
= size
;
297 kcp
->kcp_align
= align
;
299 kcp
->kcp_alloc
= alloc
;
306 splat_kmem_cache_test_kcp_free(kmem_cache_priv_t
*kcp
)
308 kmem_free(kcp
, sizeof(kmem_cache_priv_t
));
311 static kmem_cache_thread_t
*
312 splat_kmem_cache_test_kct_alloc(kmem_cache_priv_t
*kcp
, int id
)
314 kmem_cache_thread_t
*kct
;
316 ASSERTF(id
< SPLAT_KMEM_THREADS
, "id=%d\n", id
);
317 ASSERT(kcp
->kcp_kct
[id
] == NULL
);
319 kct
= kmem_zalloc(sizeof(kmem_cache_thread_t
), KM_SLEEP
);
323 spin_lock_init(&kct
->kct_lock
);
325 INIT_LIST_HEAD(&kct
->kct_list
);
327 spin_lock(&kcp
->kcp_lock
);
328 kcp
->kcp_kct
[id
] = kct
;
329 spin_unlock(&kcp
->kcp_lock
);
335 splat_kmem_cache_test_kct_free(kmem_cache_priv_t
*kcp
,
336 kmem_cache_thread_t
*kct
)
338 spin_lock(&kcp
->kcp_lock
);
339 kcp
->kcp_kct
[kct
->kct_id
] = NULL
;
340 spin_unlock(&kcp
->kcp_lock
);
342 kmem_free(kct
, sizeof(kmem_cache_thread_t
));
346 splat_kmem_cache_test_kcd_free(kmem_cache_priv_t
*kcp
,
347 kmem_cache_thread_t
*kct
)
349 kmem_cache_data_t
*kcd
;
351 spin_lock(&kct
->kct_lock
);
352 while (!list_empty(&kct
->kct_list
)) {
353 kcd
= list_entry(kct
->kct_list
.next
,
354 kmem_cache_data_t
, kcd_node
);
355 list_del(&kcd
->kcd_node
);
356 spin_unlock(&kct
->kct_lock
);
358 kmem_cache_free(kcp
->kcp_cache
, kcd
);
360 spin_lock(&kct
->kct_lock
);
362 spin_unlock(&kct
->kct_lock
);
366 splat_kmem_cache_test_kcd_alloc(kmem_cache_priv_t
*kcp
,
367 kmem_cache_thread_t
*kct
, int count
)
369 kmem_cache_data_t
*kcd
;
372 for (i
= 0; i
< count
; i
++) {
373 kcd
= kmem_cache_alloc(kcp
->kcp_cache
, KM_SLEEP
);
375 splat_kmem_cache_test_kcd_free(kcp
, kct
);
379 spin_lock(&kct
->kct_lock
);
380 list_add_tail(&kcd
->kcd_node
, &kct
->kct_list
);
381 spin_unlock(&kct
->kct_lock
);
388 splat_kmem_cache_test_debug(struct file
*file
, char *name
,
389 kmem_cache_priv_t
*kcp
)
393 splat_vprint(file
, name
, "%s cache objects %d",
394 kcp
->kcp_cache
->skc_name
, kcp
->kcp_count
);
396 if (kcp
->kcp_cache
->skc_flags
& (KMC_KMEM
| KMC_VMEM
)) {
397 splat_vprint(file
, name
, ", slabs %u/%u objs %u/%u",
398 (unsigned)kcp
->kcp_cache
->skc_slab_alloc
,
399 (unsigned)kcp
->kcp_cache
->skc_slab_total
,
400 (unsigned)kcp
->kcp_cache
->skc_obj_alloc
,
401 (unsigned)kcp
->kcp_cache
->skc_obj_total
);
403 if (!(kcp
->kcp_cache
->skc_flags
& KMC_NOMAGAZINE
)) {
404 splat_vprint(file
, name
, "%s", "mags");
406 for_each_online_cpu(j
)
407 splat_print(file
, "%u/%u ",
408 kcp
->kcp_cache
->skc_mag
[j
]->skm_avail
,
409 kcp
->kcp_cache
->skc_mag
[j
]->skm_size
);
413 splat_print(file
, "%s\n", "");
417 splat_kmem_cache_test_constructor(void *ptr
, void *priv
, int flags
)
419 kmem_cache_priv_t
*kcp
= (kmem_cache_priv_t
*)priv
;
420 kmem_cache_data_t
*kcd
= (kmem_cache_data_t
*)ptr
;
423 kcd
->kcd_magic
= kcp
->kcp_magic
;
424 INIT_LIST_HEAD(&kcd
->kcd_node
);
426 memset(kcd
->kcd_buf
, 0xaa, kcp
->kcp_size
- (sizeof *kcd
));
434 splat_kmem_cache_test_destructor(void *ptr
, void *priv
)
436 kmem_cache_priv_t
*kcp
= (kmem_cache_priv_t
*)priv
;
437 kmem_cache_data_t
*kcd
= (kmem_cache_data_t
*)ptr
;
442 memset(kcd
->kcd_buf
, 0xbb, kcp
->kcp_size
- (sizeof *kcd
));
450 * Generic reclaim function which assumes that all objects may
451 * be reclaimed at any time. We free a small percentage of the
452 * objects linked off the kcp or kct[] every time we are called.
455 splat_kmem_cache_test_reclaim(void *priv
)
457 kmem_cache_priv_t
*kcp
= (kmem_cache_priv_t
*)priv
;
458 kmem_cache_thread_t
*kct
;
459 kmem_cache_data_t
*kcd
;
463 ASSERT(kcp
->kcp_magic
== SPLAT_KMEM_TEST_MAGIC
);
465 /* For each kct thread reclaim some objects */
466 spin_lock(&kcp
->kcp_lock
);
467 for (i
= 0; i
< SPLAT_KMEM_THREADS
; i
++) {
468 kct
= kcp
->kcp_kct
[i
];
472 spin_unlock(&kcp
->kcp_lock
);
473 spin_lock(&kct
->kct_lock
);
475 count
= SPLAT_KMEM_OBJ_RECLAIM
;
476 while (count
> 0 && !list_empty(&kct
->kct_list
)) {
477 kcd
= list_entry(kct
->kct_list
.next
,
478 kmem_cache_data_t
, kcd_node
);
479 list_del(&kcd
->kcd_node
);
480 list_add(&kcd
->kcd_node
, &reclaim
);
484 spin_unlock(&kct
->kct_lock
);
485 spin_lock(&kcp
->kcp_lock
);
487 spin_unlock(&kcp
->kcp_lock
);
489 /* Freed outside the spin lock */
490 while (!list_empty(&reclaim
)) {
491 kcd
= list_entry(reclaim
.next
, kmem_cache_data_t
, kcd_node
);
492 list_del(&kcd
->kcd_node
);
493 kmem_cache_free(kcp
->kcp_cache
, kcd
);
500 splat_kmem_cache_test_threads(kmem_cache_priv_t
*kcp
, int threads
)
504 spin_lock(&kcp
->kcp_lock
);
505 rc
= (kcp
->kcp_kct_count
== threads
);
506 spin_unlock(&kcp
->kcp_lock
);
512 splat_kmem_cache_test_flags(kmem_cache_priv_t
*kcp
, int flags
)
516 spin_lock(&kcp
->kcp_lock
);
517 rc
= (kcp
->kcp_flags
& flags
);
518 spin_unlock(&kcp
->kcp_lock
);
524 splat_kmem_cache_test_thread(void *arg
)
526 kmem_cache_priv_t
*kcp
= (kmem_cache_priv_t
*)arg
;
527 kmem_cache_thread_t
*kct
;
530 ASSERT(kcp
->kcp_magic
== SPLAT_KMEM_TEST_MAGIC
);
532 /* Assign thread ids */
533 spin_lock(&kcp
->kcp_lock
);
534 if (kcp
->kcp_kct_count
== -1)
535 kcp
->kcp_kct_count
= 0;
537 id
= kcp
->kcp_kct_count
;
538 kcp
->kcp_kct_count
++;
539 spin_unlock(&kcp
->kcp_lock
);
541 kct
= splat_kmem_cache_test_kct_alloc(kcp
, id
);
547 /* Wait for all threads to have started and report they are ready */
548 if (kcp
->kcp_kct_count
== SPLAT_KMEM_THREADS
)
549 wake_up(&kcp
->kcp_ctl_waitq
);
551 wait_event(kcp
->kcp_thr_waitq
,
552 splat_kmem_cache_test_flags(kcp
, KCP_FLAG_READY
));
554 /* Create and destroy objects */
555 rc
= splat_kmem_cache_test_kcd_alloc(kcp
, kct
, kcp
->kcp_alloc
);
556 splat_kmem_cache_test_kcd_free(kcp
, kct
);
559 splat_kmem_cache_test_kct_free(kcp
, kct
);
561 spin_lock(&kcp
->kcp_lock
);
565 if ((--kcp
->kcp_kct_count
) == 0)
566 wake_up(&kcp
->kcp_ctl_waitq
);
568 spin_unlock(&kcp
->kcp_lock
);
574 splat_kmem_cache_test(struct file
*file
, void *arg
, char *name
,
575 int size
, int align
, int flags
)
577 kmem_cache_priv_t
*kcp
;
578 kmem_cache_data_t
*kcd
= NULL
;
581 kcp
= splat_kmem_cache_test_kcp_alloc(file
, name
, size
, align
, 0);
583 splat_vprint(file
, name
, "Unable to create '%s'\n", "kcp");
588 kmem_cache_create(SPLAT_KMEM_CACHE_NAME
,
589 kcp
->kcp_size
, kcp
->kcp_align
,
590 splat_kmem_cache_test_constructor
,
591 splat_kmem_cache_test_destructor
,
592 NULL
, kcp
, NULL
, flags
);
593 if (!kcp
->kcp_cache
) {
594 splat_vprint(file
, name
,
595 "Unable to create '%s'\n",
596 SPLAT_KMEM_CACHE_NAME
);
601 kcd
= kmem_cache_alloc(kcp
->kcp_cache
, KM_SLEEP
);
603 splat_vprint(file
, name
,
604 "Unable to allocate from '%s'\n",
605 SPLAT_KMEM_CACHE_NAME
);
610 if (!kcd
->kcd_flag
) {
611 splat_vprint(file
, name
,
612 "Failed to run contructor for '%s'\n",
613 SPLAT_KMEM_CACHE_NAME
);
618 if (kcd
->kcd_magic
!= kcp
->kcp_magic
) {
619 splat_vprint(file
, name
,
620 "Failed to pass private data to constructor "
621 "for '%s'\n", SPLAT_KMEM_CACHE_NAME
);
626 max
= kcp
->kcp_count
;
627 kmem_cache_free(kcp
->kcp_cache
, kcd
);
629 /* Destroy the entire cache which will force destructors to
630 * run and we can verify one was called for every object */
631 kmem_cache_destroy(kcp
->kcp_cache
);
632 if (kcp
->kcp_count
) {
633 splat_vprint(file
, name
,
634 "Failed to run destructor on all slab objects "
635 "for '%s'\n", SPLAT_KMEM_CACHE_NAME
);
639 splat_kmem_cache_test_kcp_free(kcp
);
640 splat_vprint(file
, name
,
641 "Successfully ran ctors/dtors for %d elements in '%s'\n",
642 max
, SPLAT_KMEM_CACHE_NAME
);
648 kmem_cache_free(kcp
->kcp_cache
, kcd
);
651 kmem_cache_destroy(kcp
->kcp_cache
);
653 splat_kmem_cache_test_kcp_free(kcp
);
659 splat_kmem_cache_thread_test(struct file
*file
, void *arg
, char *name
,
660 int size
, int alloc
, int max_time
)
662 kmem_cache_priv_t
*kcp
;
664 struct timespec start
, stop
, delta
;
668 kcp
= splat_kmem_cache_test_kcp_alloc(file
, name
, size
, 0, alloc
);
670 splat_vprint(file
, name
, "Unable to create '%s'\n", "kcp");
674 (void)snprintf(cache_name
, 32, "%s-%d-%d",
675 SPLAT_KMEM_CACHE_NAME
, size
, alloc
);
677 kmem_cache_create(cache_name
, kcp
->kcp_size
, 0,
678 splat_kmem_cache_test_constructor
,
679 splat_kmem_cache_test_destructor
,
680 splat_kmem_cache_test_reclaim
,
682 if (!kcp
->kcp_cache
) {
683 splat_vprint(file
, name
, "Unable to create '%s'\n", cache_name
);
688 getnstimeofday(&start
);
690 for (i
= 0; i
< SPLAT_KMEM_THREADS
; i
++) {
691 thr
= thread_create(NULL
, 0,
692 splat_kmem_cache_test_thread
,
693 kcp
, 0, &p0
, TS_RUN
, minclsyspri
);
700 /* Sleep until all threads have started, then set the ready
701 * flag and wake them all up for maximum concurrency. */
702 wait_event(kcp
->kcp_ctl_waitq
,
703 splat_kmem_cache_test_threads(kcp
, SPLAT_KMEM_THREADS
));
705 spin_lock(&kcp
->kcp_lock
);
706 kcp
->kcp_flags
|= KCP_FLAG_READY
;
707 spin_unlock(&kcp
->kcp_lock
);
708 wake_up_all(&kcp
->kcp_thr_waitq
);
710 /* Sleep until all thread have finished */
711 wait_event(kcp
->kcp_ctl_waitq
, splat_kmem_cache_test_threads(kcp
, 0));
713 getnstimeofday(&stop
);
714 delta
= timespec_sub(stop
, start
);
716 splat_vprint(file
, name
,
718 "%lu/%lu/%lu\t%lu/%lu/%lu\n",
719 kcp
->kcp_cache
->skc_name
,
720 delta
.tv_sec
, delta
.tv_nsec
,
721 (unsigned long)kcp
->kcp_cache
->skc_slab_total
,
722 (unsigned long)kcp
->kcp_cache
->skc_slab_max
,
723 (unsigned long)(kcp
->kcp_alloc
*
725 SPL_KMEM_CACHE_OBJ_PER_SLAB
),
726 (unsigned long)kcp
->kcp_cache
->skc_obj_total
,
727 (unsigned long)kcp
->kcp_cache
->skc_obj_max
,
728 (unsigned long)(kcp
->kcp_alloc
*
729 SPLAT_KMEM_THREADS
));
731 if (delta
.tv_sec
>= max_time
)
734 if (!rc
&& kcp
->kcp_rc
)
738 kmem_cache_destroy(kcp
->kcp_cache
);
740 splat_kmem_cache_test_kcp_free(kcp
);
744 /* Validate small object cache behavior for dynamic/kmem/vmem caches */
746 splat_kmem_test5(struct file
*file
, void *arg
)
748 char *name
= SPLAT_KMEM_TEST5_NAME
;
751 /* On slab (default + kmem + vmem) */
752 rc
= splat_kmem_cache_test(file
, arg
, name
, 128, 0, 0);
756 rc
= splat_kmem_cache_test(file
, arg
, name
, 128, 0, KMC_KMEM
);
760 rc
= splat_kmem_cache_test(file
, arg
, name
, 128, 0, KMC_VMEM
);
764 /* Off slab (default + kmem + vmem) */
765 rc
= splat_kmem_cache_test(file
, arg
, name
, 128, 0, KMC_OFFSLAB
);
769 rc
= splat_kmem_cache_test(file
, arg
, name
, 128, 0,
770 KMC_KMEM
| KMC_OFFSLAB
);
774 rc
= splat_kmem_cache_test(file
, arg
, name
, 128, 0,
775 KMC_VMEM
| KMC_OFFSLAB
);
781 * Validate large object cache behavior for dynamic/kmem/vmem caches
784 splat_kmem_test6(struct file
*file
, void *arg
)
786 char *name
= SPLAT_KMEM_TEST6_NAME
;
789 /* On slab (default + kmem + vmem) */
790 rc
= splat_kmem_cache_test(file
, arg
, name
, 256*1024, 0, 0);
794 rc
= splat_kmem_cache_test(file
, arg
, name
, 64*1024, 0, KMC_KMEM
);
798 rc
= splat_kmem_cache_test(file
, arg
, name
, 1024*1024, 0, KMC_VMEM
);
802 /* Off slab (default + kmem + vmem) */
803 rc
= splat_kmem_cache_test(file
, arg
, name
, 256*1024, 0, KMC_OFFSLAB
);
807 rc
= splat_kmem_cache_test(file
, arg
, name
, 64*1024, 0,
808 KMC_KMEM
| KMC_OFFSLAB
);
812 rc
= splat_kmem_cache_test(file
, arg
, name
, 1024*1024, 0,
813 KMC_VMEM
| KMC_OFFSLAB
);
819 * Validate object alignment cache behavior for caches
822 splat_kmem_test7(struct file
*file
, void *arg
)
824 char *name
= SPLAT_KMEM_TEST7_NAME
;
827 for (i
= SPL_KMEM_CACHE_ALIGN
; i
<= PAGE_SIZE
; i
*= 2) {
828 rc
= splat_kmem_cache_test(file
, arg
, name
, 157, i
, 0);
832 rc
= splat_kmem_cache_test(file
, arg
, name
, 157, i
,
842 * Validate kmem_cache_reap() by requesting the slab cache free any objects
843 * it can. For a few reasons this may not immediately result in more free
844 * memory even if objects are freed. First off, due to fragmentation we
845 * may not be able to reclaim any slabs. Secondly, even if we do we fully
846 * clear some slabs we will not want to immediately reclaim all of them
847 * because we may contend with cache allocations and thrash. What we want
848 * to see is the slab size decrease more gradually as it becomes clear they
849 * will not be needed. This should be achievable in less than a minute.
850 * If it takes longer than this something has gone wrong.
853 splat_kmem_test8(struct file
*file
, void *arg
)
855 kmem_cache_priv_t
*kcp
;
856 kmem_cache_thread_t
*kct
;
857 unsigned int spl_kmem_cache_expire_old
;
860 /* Enable cache aging just for this test if it is disabled */
861 spl_kmem_cache_expire_old
= spl_kmem_cache_expire
;
862 spl_kmem_cache_expire
= KMC_EXPIRE_AGE
;
864 kcp
= splat_kmem_cache_test_kcp_alloc(file
, SPLAT_KMEM_TEST8_NAME
,
867 splat_vprint(file
, SPLAT_KMEM_TEST8_NAME
,
868 "Unable to create '%s'\n", "kcp");
874 kmem_cache_create(SPLAT_KMEM_CACHE_NAME
, kcp
->kcp_size
, 0,
875 splat_kmem_cache_test_constructor
,
876 splat_kmem_cache_test_destructor
,
877 splat_kmem_cache_test_reclaim
,
879 if (!kcp
->kcp_cache
) {
880 splat_vprint(file
, SPLAT_KMEM_TEST8_NAME
,
881 "Unable to create '%s'\n", SPLAT_KMEM_CACHE_NAME
);
886 kct
= splat_kmem_cache_test_kct_alloc(kcp
, 0);
888 splat_vprint(file
, SPLAT_KMEM_TEST8_NAME
,
889 "Unable to create '%s'\n", "kct");
894 rc
= splat_kmem_cache_test_kcd_alloc(kcp
, kct
, SPLAT_KMEM_OBJ_COUNT
);
896 splat_vprint(file
, SPLAT_KMEM_TEST8_NAME
, "Unable to "
897 "allocate from '%s'\n", SPLAT_KMEM_CACHE_NAME
);
901 /* Force reclaim every 1/10 a second for 60 seconds. */
902 for (i
= 0; i
< 600; i
++) {
903 kmem_cache_reap_now(kcp
->kcp_cache
);
904 splat_kmem_cache_test_debug(file
, SPLAT_KMEM_TEST8_NAME
, kcp
);
906 if (kcp
->kcp_count
== 0)
909 set_current_state(TASK_INTERRUPTIBLE
);
910 schedule_timeout(HZ
/ 10);
913 if (kcp
->kcp_count
== 0) {
914 splat_vprint(file
, SPLAT_KMEM_TEST8_NAME
,
915 "Successfully created %d objects "
916 "in cache %s and reclaimed them\n",
917 SPLAT_KMEM_OBJ_COUNT
, SPLAT_KMEM_CACHE_NAME
);
919 splat_vprint(file
, SPLAT_KMEM_TEST8_NAME
,
920 "Failed to reclaim %u/%d objects from cache %s\n",
921 (unsigned)kcp
->kcp_count
,
922 SPLAT_KMEM_OBJ_COUNT
, SPLAT_KMEM_CACHE_NAME
);
926 /* Cleanup our mess (for failure case of time expiring) */
927 splat_kmem_cache_test_kcd_free(kcp
, kct
);
929 splat_kmem_cache_test_kct_free(kcp
, kct
);
931 kmem_cache_destroy(kcp
->kcp_cache
);
933 splat_kmem_cache_test_kcp_free(kcp
);
935 spl_kmem_cache_expire
= spl_kmem_cache_expire_old
;
940 /* Test cache aging, we have allocated a large number of objects thus
941 * creating a large number of slabs and then free'd them all. However,
942 * since there should be little memory pressure at the moment those
943 * slabs have not been freed. What we want to see is the slab size
944 * decrease gradually as it becomes clear they will not be be needed.
945 * This should be achievable in less than minute. If it takes longer
946 * than this something has gone wrong.
949 splat_kmem_test9(struct file
*file
, void *arg
)
951 kmem_cache_priv_t
*kcp
;
952 kmem_cache_thread_t
*kct
;
953 unsigned int spl_kmem_cache_expire_old
;
954 int i
, rc
= 0, count
= SPLAT_KMEM_OBJ_COUNT
* 128;
956 /* Enable cache aging just for this test if it is disabled */
957 spl_kmem_cache_expire_old
= spl_kmem_cache_expire
;
958 spl_kmem_cache_expire
= KMC_EXPIRE_AGE
;
960 kcp
= splat_kmem_cache_test_kcp_alloc(file
, SPLAT_KMEM_TEST9_NAME
,
963 splat_vprint(file
, SPLAT_KMEM_TEST9_NAME
,
964 "Unable to create '%s'\n", "kcp");
970 kmem_cache_create(SPLAT_KMEM_CACHE_NAME
, kcp
->kcp_size
, 0,
971 splat_kmem_cache_test_constructor
,
972 splat_kmem_cache_test_destructor
,
974 if (!kcp
->kcp_cache
) {
975 splat_vprint(file
, SPLAT_KMEM_TEST9_NAME
,
976 "Unable to create '%s'\n", SPLAT_KMEM_CACHE_NAME
);
981 kct
= splat_kmem_cache_test_kct_alloc(kcp
, 0);
983 splat_vprint(file
, SPLAT_KMEM_TEST8_NAME
,
984 "Unable to create '%s'\n", "kct");
989 rc
= splat_kmem_cache_test_kcd_alloc(kcp
, kct
, count
);
991 splat_vprint(file
, SPLAT_KMEM_TEST9_NAME
, "Unable to "
992 "allocate from '%s'\n", SPLAT_KMEM_CACHE_NAME
);
996 splat_kmem_cache_test_kcd_free(kcp
, kct
);
998 for (i
= 0; i
< 60; i
++) {
999 splat_kmem_cache_test_debug(file
, SPLAT_KMEM_TEST9_NAME
, kcp
);
1001 if (kcp
->kcp_count
== 0)
1004 set_current_state(TASK_INTERRUPTIBLE
);
1005 schedule_timeout(HZ
);
1008 if (kcp
->kcp_count
== 0) {
1009 splat_vprint(file
, SPLAT_KMEM_TEST9_NAME
,
1010 "Successfully created %d objects "
1011 "in cache %s and reclaimed them\n",
1012 count
, SPLAT_KMEM_CACHE_NAME
);
1014 splat_vprint(file
, SPLAT_KMEM_TEST9_NAME
,
1015 "Failed to reclaim %u/%d objects from cache %s\n",
1016 (unsigned)kcp
->kcp_count
, count
,
1017 SPLAT_KMEM_CACHE_NAME
);
1022 splat_kmem_cache_test_kct_free(kcp
, kct
);
1024 kmem_cache_destroy(kcp
->kcp_cache
);
1026 splat_kmem_cache_test_kcp_free(kcp
);
1028 spl_kmem_cache_expire
= spl_kmem_cache_expire_old
;
1034 * This test creates N threads with a shared kmem cache. They then all
1035 * concurrently allocate and free from the cache to stress the locking and
1036 * concurrent cache performance. If any one test takes longer than 5
1037 * seconds to complete it is treated as a failure and may indicate a
1038 * performance regression. On my test system no one test takes more
1039 * than 1 second to complete so a 5x slowdown likely a problem.
1042 splat_kmem_test10(struct file
*file
, void *arg
)
1044 uint64_t size
, alloc
, rc
= 0;
1046 for (size
= 32; size
<= 1024*1024; size
*= 2) {
1048 splat_vprint(file
, SPLAT_KMEM_TEST10_NAME
, "%-22s %s", "name",
1049 "time (sec)\tslabs \tobjs \thash\n");
1050 splat_vprint(file
, SPLAT_KMEM_TEST10_NAME
, "%-22s %s", "",
1051 " \ttot/max/calc\ttot/max/calc\n");
1053 for (alloc
= 1; alloc
<= 1024; alloc
*= 2) {
1055 /* Skip tests which exceed 1/2 of physical memory. */
1056 if (size
* alloc
* SPLAT_KMEM_THREADS
> physmem
/ 2)
1059 rc
= splat_kmem_cache_thread_test(file
, arg
,
1060 SPLAT_KMEM_TEST10_NAME
, size
, alloc
, 5);
1071 * This test creates N threads with a shared kmem cache which overcommits
1072 * memory by 4x. This makes it impossible for the slab to satify the
1073 * thread requirements without having its reclaim hook run which will
1074 * free objects back for use. This behavior is triggered by the linum VM
1075 * detecting a low memory condition on the node and invoking the shrinkers.
1076 * This should allow all the threads to complete while avoiding deadlock
1077 * and for the most part out of memory events. This is very tough on the
1078 * system so it is possible the test app may get oom'ed. This particular
1079 * test has proven troublesome on 32-bit archs with limited virtual
1080 * address space so it only run on 64-bit systems.
1083 splat_kmem_test11(struct file
*file
, void *arg
)
1085 uint64_t size
, alloc
, rc
;
1088 alloc
= ((4 * physmem
* PAGE_SIZE
) / size
) / SPLAT_KMEM_THREADS
;
1090 splat_vprint(file
, SPLAT_KMEM_TEST11_NAME
, "%-22s %s", "name",
1091 "time (sec)\tslabs \tobjs \thash\n");
1092 splat_vprint(file
, SPLAT_KMEM_TEST11_NAME
, "%-22s %s", "",
1093 " \ttot/max/calc\ttot/max/calc\n");
1095 rc
= splat_kmem_cache_thread_test(file
, arg
,
1096 SPLAT_KMEM_TEST11_NAME
, size
, alloc
, 60);
1102 typedef struct dummy_page
{
1103 struct list_head dp_list
;
1104 char dp_pad
[PAGE_SIZE
- sizeof(struct list_head
)];
1108 * This test is designed to verify that direct reclaim is functioning as
1109 * expected. We allocate a large number of objects thus creating a large
1110 * number of slabs. We then apply memory pressure and expect that the
1111 * direct reclaim path can easily recover those slabs. The registered
1112 * reclaim function will free the objects and the slab shrinker will call
1113 * it repeatedly until at least a single slab can be freed.
1115 * Note it may not be possible to reclaim every last slab via direct reclaim
1116 * without a failure because the shrinker_rwsem may be contended. For this
1117 * reason, quickly reclaiming 3/4 of the slabs is considered a success.
1119 * This should all be possible within 10 seconds. For reference, on a
1120 * system with 2G of memory this test takes roughly 0.2 seconds to run.
1121 * It may take longer on larger memory systems but should still easily
1122 * complete in the alloted 10 seconds.
1125 splat_kmem_test13(struct file
*file
, void *arg
)
1127 kmem_cache_priv_t
*kcp
;
1128 kmem_cache_thread_t
*kct
;
1130 struct list_head list
;
1131 struct timespec start
, stop
, delta
= { 0, 0 };
1132 int size
, count
, slabs
, fails
= 0;
1133 int i
, rc
= 0, max_time
= 10;
1136 count
= ((physmem
* PAGE_SIZE
) / 4 / size
);
1138 kcp
= splat_kmem_cache_test_kcp_alloc(file
, SPLAT_KMEM_TEST13_NAME
,
1141 splat_vprint(file
, SPLAT_KMEM_TEST13_NAME
,
1142 "Unable to create '%s'\n", "kcp");
1148 kmem_cache_create(SPLAT_KMEM_CACHE_NAME
, kcp
->kcp_size
, 0,
1149 splat_kmem_cache_test_constructor
,
1150 splat_kmem_cache_test_destructor
,
1151 splat_kmem_cache_test_reclaim
,
1153 if (!kcp
->kcp_cache
) {
1154 splat_vprint(file
, SPLAT_KMEM_TEST13_NAME
,
1155 "Unable to create '%s'\n", SPLAT_KMEM_CACHE_NAME
);
1160 kct
= splat_kmem_cache_test_kct_alloc(kcp
, 0);
1162 splat_vprint(file
, SPLAT_KMEM_TEST13_NAME
,
1163 "Unable to create '%s'\n", "kct");
1168 rc
= splat_kmem_cache_test_kcd_alloc(kcp
, kct
, count
);
1170 splat_vprint(file
, SPLAT_KMEM_TEST13_NAME
, "Unable to "
1171 "allocate from '%s'\n", SPLAT_KMEM_CACHE_NAME
);
1176 slabs
= kcp
->kcp_cache
->skc_slab_total
;
1177 INIT_LIST_HEAD(&list
);
1178 getnstimeofday(&start
);
1180 /* Apply memory pressure */
1181 while (kcp
->kcp_cache
->skc_slab_total
> (slabs
>> 2)) {
1183 if ((i
% 10000) == 0)
1184 splat_kmem_cache_test_debug(
1185 file
, SPLAT_KMEM_TEST13_NAME
, kcp
);
1187 getnstimeofday(&stop
);
1188 delta
= timespec_sub(stop
, start
);
1189 if (delta
.tv_sec
>= max_time
) {
1190 splat_vprint(file
, SPLAT_KMEM_TEST13_NAME
,
1191 "Failed to reclaim 3/4 of cache in %ds, "
1192 "%u/%u slabs remain\n", max_time
,
1193 (unsigned)kcp
->kcp_cache
->skc_slab_total
,
1199 dp
= (dummy_page_t
*)__get_free_page(GFP_KERNEL
);
1202 splat_vprint(file
, SPLAT_KMEM_TEST13_NAME
,
1203 "Failed (%d) to allocate page with %u "
1204 "slabs still in the cache\n", fails
,
1205 (unsigned)kcp
->kcp_cache
->skc_slab_total
);
1209 list_add(&dp
->dp_list
, &list
);
1214 splat_vprint(file
, SPLAT_KMEM_TEST13_NAME
,
1215 "Successfully created %u slabs and with %d alloc "
1216 "failures reclaimed 3/4 of them in %d.%03ds\n",
1218 (int)delta
.tv_sec
, (int)delta
.tv_nsec
/ 1000000);
1220 /* Release memory pressure pages */
1221 while (!list_empty(&list
)) {
1222 dp
= list_entry(list
.next
, dummy_page_t
, dp_list
);
1223 list_del_init(&dp
->dp_list
);
1224 free_page((unsigned long)dp
);
1227 /* Release remaining kmem cache objects */
1228 splat_kmem_cache_test_kcd_free(kcp
, kct
);
1230 splat_kmem_cache_test_kct_free(kcp
, kct
);
1232 kmem_cache_destroy(kcp
->kcp_cache
);
1234 splat_kmem_cache_test_kcp_free(kcp
);
1240 splat_kmem_init(void)
1242 splat_subsystem_t
*sub
;
1244 sub
= kmalloc(sizeof(*sub
), GFP_KERNEL
);
1248 memset(sub
, 0, sizeof(*sub
));
1249 strncpy(sub
->desc
.name
, SPLAT_KMEM_NAME
, SPLAT_NAME_SIZE
);
1250 strncpy(sub
->desc
.desc
, SPLAT_KMEM_DESC
, SPLAT_DESC_SIZE
);
1251 INIT_LIST_HEAD(&sub
->subsystem_list
);
1252 INIT_LIST_HEAD(&sub
->test_list
);
1253 spin_lock_init(&sub
->test_lock
);
1254 sub
->desc
.id
= SPLAT_SUBSYSTEM_KMEM
;
1256 SPLAT_TEST_INIT(sub
, SPLAT_KMEM_TEST1_NAME
, SPLAT_KMEM_TEST1_DESC
,
1257 SPLAT_KMEM_TEST1_ID
, splat_kmem_test1
);
1258 SPLAT_TEST_INIT(sub
, SPLAT_KMEM_TEST2_NAME
, SPLAT_KMEM_TEST2_DESC
,
1259 SPLAT_KMEM_TEST2_ID
, splat_kmem_test2
);
1260 SPLAT_TEST_INIT(sub
, SPLAT_KMEM_TEST3_NAME
, SPLAT_KMEM_TEST3_DESC
,
1261 SPLAT_KMEM_TEST3_ID
, splat_kmem_test3
);
1262 SPLAT_TEST_INIT(sub
, SPLAT_KMEM_TEST4_NAME
, SPLAT_KMEM_TEST4_DESC
,
1263 SPLAT_KMEM_TEST4_ID
, splat_kmem_test4
);
1264 SPLAT_TEST_INIT(sub
, SPLAT_KMEM_TEST5_NAME
, SPLAT_KMEM_TEST5_DESC
,
1265 SPLAT_KMEM_TEST5_ID
, splat_kmem_test5
);
1266 SPLAT_TEST_INIT(sub
, SPLAT_KMEM_TEST6_NAME
, SPLAT_KMEM_TEST6_DESC
,
1267 SPLAT_KMEM_TEST6_ID
, splat_kmem_test6
);
1268 SPLAT_TEST_INIT(sub
, SPLAT_KMEM_TEST7_NAME
, SPLAT_KMEM_TEST7_DESC
,
1269 SPLAT_KMEM_TEST7_ID
, splat_kmem_test7
);
1270 SPLAT_TEST_INIT(sub
, SPLAT_KMEM_TEST8_NAME
, SPLAT_KMEM_TEST8_DESC
,
1271 SPLAT_KMEM_TEST8_ID
, splat_kmem_test8
);
1272 SPLAT_TEST_INIT(sub
, SPLAT_KMEM_TEST9_NAME
, SPLAT_KMEM_TEST9_DESC
,
1273 SPLAT_KMEM_TEST9_ID
, splat_kmem_test9
);
1274 SPLAT_TEST_INIT(sub
, SPLAT_KMEM_TEST10_NAME
, SPLAT_KMEM_TEST10_DESC
,
1275 SPLAT_KMEM_TEST10_ID
, splat_kmem_test10
);
1277 SPLAT_TEST_INIT(sub
, SPLAT_KMEM_TEST11_NAME
, SPLAT_KMEM_TEST11_DESC
,
1278 SPLAT_KMEM_TEST11_ID
, splat_kmem_test11
);
1280 SPLAT_TEST_INIT(sub
, SPLAT_KMEM_TEST13_NAME
, SPLAT_KMEM_TEST13_DESC
,
1281 SPLAT_KMEM_TEST13_ID
, splat_kmem_test13
);
1287 splat_kmem_fini(splat_subsystem_t
*sub
)
1290 SPLAT_TEST_FINI(sub
, SPLAT_KMEM_TEST13_ID
);
1292 SPLAT_TEST_FINI(sub
, SPLAT_KMEM_TEST11_ID
);
1294 SPLAT_TEST_FINI(sub
, SPLAT_KMEM_TEST10_ID
);
1295 SPLAT_TEST_FINI(sub
, SPLAT_KMEM_TEST9_ID
);
1296 SPLAT_TEST_FINI(sub
, SPLAT_KMEM_TEST8_ID
);
1297 SPLAT_TEST_FINI(sub
, SPLAT_KMEM_TEST7_ID
);
1298 SPLAT_TEST_FINI(sub
, SPLAT_KMEM_TEST6_ID
);
1299 SPLAT_TEST_FINI(sub
, SPLAT_KMEM_TEST5_ID
);
1300 SPLAT_TEST_FINI(sub
, SPLAT_KMEM_TEST4_ID
);
1301 SPLAT_TEST_FINI(sub
, SPLAT_KMEM_TEST3_ID
);
1302 SPLAT_TEST_FINI(sub
, SPLAT_KMEM_TEST2_ID
);
1303 SPLAT_TEST_FINI(sub
, SPLAT_KMEM_TEST1_ID
);
1309 splat_kmem_id(void) {
1310 return SPLAT_SUBSYSTEM_KMEM
;