]> git.proxmox.com Git - mirror_spl.git/blame - module/splat/splat-kmem.c
Remove global memory variables
[mirror_spl.git] / module / splat / splat-kmem.c
CommitLineData
716154c5
BB
1/*****************************************************************************\
2 * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
3 * Copyright (C) 2007 The Regents of the University of California.
4 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
5 * Written by Brian Behlendorf <behlendorf1@llnl.gov>.
715f6251 6 * UCRL-CODE-235197
7 *
716154c5 8 * This file is part of the SPL, Solaris Porting Layer.
3d6af2dd 9 * For details, see <http://zfsonlinux.org/>.
716154c5
BB
10 *
11 * The SPL is free software; you can redistribute it and/or modify it
12 * under the terms of the GNU General Public License as published by the
13 * Free Software Foundation; either version 2 of the License, or (at your
14 * option) any later version.
715f6251 15 *
716154c5 16 * The SPL is distributed in the hope that it will be useful, but WITHOUT
715f6251 17 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 * for more details.
20 *
21 * You should have received a copy of the GNU General Public License along
716154c5
BB
22 * with the SPL. If not, see <http://www.gnu.org/licenses/>.
23 *****************************************************************************
24 * Solaris Porting LAyer Tests (SPLAT) Kmem Tests.
25\*****************************************************************************/
715f6251 26
df870a69
BB
27#include <sys/kmem.h>
28#include <sys/thread.h>
7c50328b 29#include "splat-internal.h"
f1ca4da6 30
7c50328b 31#define SPLAT_KMEM_NAME "kmem"
32#define SPLAT_KMEM_DESC "Kernel Malloc/Slab Tests"
f1ca4da6 33
7c50328b 34#define SPLAT_KMEM_TEST1_ID 0x0101
35#define SPLAT_KMEM_TEST1_NAME "kmem_alloc"
36#define SPLAT_KMEM_TEST1_DESC "Memory allocation test (kmem_alloc)"
f1ca4da6 37
7c50328b 38#define SPLAT_KMEM_TEST2_ID 0x0102
39#define SPLAT_KMEM_TEST2_NAME "kmem_zalloc"
40#define SPLAT_KMEM_TEST2_DESC "Memory allocation test (kmem_zalloc)"
f1ca4da6 41
7c50328b 42#define SPLAT_KMEM_TEST3_ID 0x0103
2fb9b26a 43#define SPLAT_KMEM_TEST3_NAME "vmem_alloc"
44#define SPLAT_KMEM_TEST3_DESC "Memory allocation test (vmem_alloc)"
f1ca4da6 45
7c50328b 46#define SPLAT_KMEM_TEST4_ID 0x0104
2fb9b26a 47#define SPLAT_KMEM_TEST4_NAME "vmem_zalloc"
48#define SPLAT_KMEM_TEST4_DESC "Memory allocation test (vmem_zalloc)"
f1ca4da6 49
79b31f36 50#define SPLAT_KMEM_TEST5_ID 0x0105
ea3e6ca9 51#define SPLAT_KMEM_TEST5_NAME "slab_small"
2fb9b26a 52#define SPLAT_KMEM_TEST5_DESC "Slab ctor/dtor test (small)"
53
54#define SPLAT_KMEM_TEST6_ID 0x0106
ea3e6ca9 55#define SPLAT_KMEM_TEST6_NAME "slab_large"
2fb9b26a 56#define SPLAT_KMEM_TEST6_DESC "Slab ctor/dtor test (large)"
57
58#define SPLAT_KMEM_TEST7_ID 0x0107
ea3e6ca9
BB
59#define SPLAT_KMEM_TEST7_NAME "slab_align"
60#define SPLAT_KMEM_TEST7_DESC "Slab alignment test"
79b31f36 61
44b8f176 62#define SPLAT_KMEM_TEST8_ID 0x0108
ea3e6ca9
BB
63#define SPLAT_KMEM_TEST8_NAME "slab_reap"
64#define SPLAT_KMEM_TEST8_DESC "Slab reaping test"
44b8f176 65
48e0606a 66#define SPLAT_KMEM_TEST9_ID 0x0109
ea3e6ca9
BB
67#define SPLAT_KMEM_TEST9_NAME "slab_age"
68#define SPLAT_KMEM_TEST9_DESC "Slab aging test"
69
70#define SPLAT_KMEM_TEST10_ID 0x010a
71#define SPLAT_KMEM_TEST10_NAME "slab_lock"
72#define SPLAT_KMEM_TEST10_DESC "Slab locking test"
73
11124863 74#if 0
ea3e6ca9
BB
75#define SPLAT_KMEM_TEST11_ID 0x010b
76#define SPLAT_KMEM_TEST11_NAME "slab_overcommit"
77#define SPLAT_KMEM_TEST11_DESC "Slab memory overcommit test"
11124863 78#endif
48e0606a 79
a9a7a01c
PS
80#define SPLAT_KMEM_TEST13_ID 0x010d
81#define SPLAT_KMEM_TEST13_NAME "slab_reclaim"
82#define SPLAT_KMEM_TEST13_DESC "Slab direct memory reclaim test"
83
7c50328b 84#define SPLAT_KMEM_ALLOC_COUNT 10
79b31f36 85#define SPLAT_VMEM_ALLOC_COUNT 10
86
44b8f176 87
f1ca4da6 88static int
7c50328b 89splat_kmem_test1(struct file *file, void *arg)
f1ca4da6 90{
7c50328b 91 void *ptr[SPLAT_KMEM_ALLOC_COUNT];
f1ca4da6 92 int size = PAGE_SIZE;
93 int i, count, rc = 0;
94
79b31f36 95 while ((!rc) && (size <= (PAGE_SIZE * 32))) {
f1ca4da6 96 count = 0;
97
7c50328b 98 for (i = 0; i < SPLAT_KMEM_ALLOC_COUNT; i++) {
23d91792 99 ptr[i] = kmem_alloc(size, KM_SLEEP | KM_NODEBUG);
f1ca4da6 100 if (ptr[i])
101 count++;
102 }
103
7c50328b 104 for (i = 0; i < SPLAT_KMEM_ALLOC_COUNT; i++)
f1ca4da6 105 if (ptr[i])
106 kmem_free(ptr[i], size);
107
7c50328b 108 splat_vprint(file, SPLAT_KMEM_TEST1_NAME,
ea3e6ca9
BB
109 "%d byte allocations, %d/%d successful\n",
110 size, count, SPLAT_KMEM_ALLOC_COUNT);
7c50328b 111 if (count != SPLAT_KMEM_ALLOC_COUNT)
f1ca4da6 112 rc = -ENOMEM;
113
114 size *= 2;
115 }
116
117 return rc;
118}
119
120static int
7c50328b 121splat_kmem_test2(struct file *file, void *arg)
f1ca4da6 122{
7c50328b 123 void *ptr[SPLAT_KMEM_ALLOC_COUNT];
f1ca4da6 124 int size = PAGE_SIZE;
125 int i, j, count, rc = 0;
126
79b31f36 127 while ((!rc) && (size <= (PAGE_SIZE * 32))) {
f1ca4da6 128 count = 0;
129
7c50328b 130 for (i = 0; i < SPLAT_KMEM_ALLOC_COUNT; i++) {
23d91792 131 ptr[i] = kmem_zalloc(size, KM_SLEEP | KM_NODEBUG);
f1ca4da6 132 if (ptr[i])
133 count++;
134 }
135
136 /* Ensure buffer has been zero filled */
7c50328b 137 for (i = 0; i < SPLAT_KMEM_ALLOC_COUNT; i++) {
f1ca4da6 138 for (j = 0; j < size; j++) {
139 if (((char *)ptr[i])[j] != '\0') {
5198ea0e 140 splat_vprint(file,SPLAT_KMEM_TEST2_NAME,
ea3e6ca9
BB
141 "%d-byte allocation was "
142 "not zeroed\n", size);
f1ca4da6 143 rc = -EFAULT;
144 }
145 }
146 }
147
7c50328b 148 for (i = 0; i < SPLAT_KMEM_ALLOC_COUNT; i++)
f1ca4da6 149 if (ptr[i])
150 kmem_free(ptr[i], size);
151
7c50328b 152 splat_vprint(file, SPLAT_KMEM_TEST2_NAME,
ea3e6ca9
BB
153 "%d byte allocations, %d/%d successful\n",
154 size, count, SPLAT_KMEM_ALLOC_COUNT);
7c50328b 155 if (count != SPLAT_KMEM_ALLOC_COUNT)
f1ca4da6 156 rc = -ENOMEM;
157
158 size *= 2;
159 }
160
161 return rc;
162}
163
2fb9b26a 164static int
165splat_kmem_test3(struct file *file, void *arg)
166{
167 void *ptr[SPLAT_VMEM_ALLOC_COUNT];
168 int size = PAGE_SIZE;
169 int i, count, rc = 0;
170
171 while ((!rc) && (size <= (PAGE_SIZE * 1024))) {
172 count = 0;
173
174 for (i = 0; i < SPLAT_VMEM_ALLOC_COUNT; i++) {
175 ptr[i] = vmem_alloc(size, KM_SLEEP);
176 if (ptr[i])
177 count++;
178 }
179
180 for (i = 0; i < SPLAT_VMEM_ALLOC_COUNT; i++)
181 if (ptr[i])
182 vmem_free(ptr[i], size);
183
184 splat_vprint(file, SPLAT_KMEM_TEST3_NAME,
ea3e6ca9
BB
185 "%d byte allocations, %d/%d successful\n",
186 size, count, SPLAT_VMEM_ALLOC_COUNT);
2fb9b26a 187 if (count != SPLAT_VMEM_ALLOC_COUNT)
188 rc = -ENOMEM;
189
190 size *= 2;
191 }
192
193 return rc;
194}
195
196static int
197splat_kmem_test4(struct file *file, void *arg)
198{
199 void *ptr[SPLAT_VMEM_ALLOC_COUNT];
200 int size = PAGE_SIZE;
201 int i, j, count, rc = 0;
202
203 while ((!rc) && (size <= (PAGE_SIZE * 1024))) {
204 count = 0;
205
206 for (i = 0; i < SPLAT_VMEM_ALLOC_COUNT; i++) {
207 ptr[i] = vmem_zalloc(size, KM_SLEEP);
208 if (ptr[i])
209 count++;
210 }
211
212 /* Ensure buffer has been zero filled */
213 for (i = 0; i < SPLAT_VMEM_ALLOC_COUNT; i++) {
214 for (j = 0; j < size; j++) {
215 if (((char *)ptr[i])[j] != '\0') {
216 splat_vprint(file, SPLAT_KMEM_TEST4_NAME,
ea3e6ca9
BB
217 "%d-byte allocation was "
218 "not zeroed\n", size);
2fb9b26a 219 rc = -EFAULT;
220 }
221 }
222 }
223
224 for (i = 0; i < SPLAT_VMEM_ALLOC_COUNT; i++)
225 if (ptr[i])
226 vmem_free(ptr[i], size);
227
228 splat_vprint(file, SPLAT_KMEM_TEST4_NAME,
ea3e6ca9
BB
229 "%d byte allocations, %d/%d successful\n",
230 size, count, SPLAT_VMEM_ALLOC_COUNT);
2fb9b26a 231 if (count != SPLAT_VMEM_ALLOC_COUNT)
232 rc = -ENOMEM;
233
234 size *= 2;
235 }
236
237 return rc;
238}
239
7c50328b 240#define SPLAT_KMEM_TEST_MAGIC 0x004488CCUL
241#define SPLAT_KMEM_CACHE_NAME "kmem_test"
ea3e6ca9 242#define SPLAT_KMEM_OBJ_COUNT 1024
668d2a0d 243#define SPLAT_KMEM_OBJ_RECLAIM 32 /* objects */
ea3e6ca9
BB
244#define SPLAT_KMEM_THREADS 32
245
246#define KCP_FLAG_READY 0x01
f1ca4da6 247
248typedef struct kmem_cache_data {
f1ca4da6 249 unsigned long kcd_magic;
efcd0ca3 250 struct list_head kcd_node;
f1ca4da6 251 int kcd_flag;
2fb9b26a 252 char kcd_buf[0];
f1ca4da6 253} kmem_cache_data_t;
254
ea3e6ca9 255typedef struct kmem_cache_thread {
ea3e6ca9
BB
256 spinlock_t kct_lock;
257 int kct_id;
efcd0ca3 258 struct list_head kct_list;
ea3e6ca9
BB
259} kmem_cache_thread_t;
260
f1ca4da6 261typedef struct kmem_cache_priv {
262 unsigned long kcp_magic;
263 struct file *kcp_file;
264 kmem_cache_t *kcp_cache;
44b8f176 265 spinlock_t kcp_lock;
ea3e6ca9
BB
266 wait_queue_head_t kcp_ctl_waitq;
267 wait_queue_head_t kcp_thr_waitq;
268 int kcp_flags;
269 int kcp_kct_count;
270 kmem_cache_thread_t *kcp_kct[SPLAT_KMEM_THREADS];
2fb9b26a 271 int kcp_size;
48e0606a 272 int kcp_align;
f1ca4da6 273 int kcp_count;
44b8f176 274 int kcp_alloc;
f1ca4da6 275 int kcp_rc;
276} kmem_cache_priv_t;
277
ea3e6ca9
BB
278static kmem_cache_priv_t *
279splat_kmem_cache_test_kcp_alloc(struct file *file, char *name,
efcd0ca3 280 int size, int align, int alloc)
ea3e6ca9
BB
281{
282 kmem_cache_priv_t *kcp;
283
efcd0ca3 284 kcp = kmem_zalloc(sizeof(kmem_cache_priv_t), KM_SLEEP);
ea3e6ca9
BB
285 if (!kcp)
286 return NULL;
287
288 kcp->kcp_magic = SPLAT_KMEM_TEST_MAGIC;
289 kcp->kcp_file = file;
290 kcp->kcp_cache = NULL;
291 spin_lock_init(&kcp->kcp_lock);
292 init_waitqueue_head(&kcp->kcp_ctl_waitq);
293 init_waitqueue_head(&kcp->kcp_thr_waitq);
294 kcp->kcp_flags = 0;
295 kcp->kcp_kct_count = -1;
296 kcp->kcp_size = size;
297 kcp->kcp_align = align;
298 kcp->kcp_count = 0;
299 kcp->kcp_alloc = alloc;
300 kcp->kcp_rc = 0;
ea3e6ca9
BB
301
302 return kcp;
303}
304
305static void
306splat_kmem_cache_test_kcp_free(kmem_cache_priv_t *kcp)
307{
efcd0ca3 308 kmem_free(kcp, sizeof(kmem_cache_priv_t));
ea3e6ca9
BB
309}
310
311static kmem_cache_thread_t *
efcd0ca3 312splat_kmem_cache_test_kct_alloc(kmem_cache_priv_t *kcp, int id)
ea3e6ca9
BB
313{
314 kmem_cache_thread_t *kct;
315
316 ASSERTF(id < SPLAT_KMEM_THREADS, "id=%d\n", id);
efcd0ca3
BB
317 ASSERT(kcp->kcp_kct[id] == NULL);
318
319 kct = kmem_zalloc(sizeof(kmem_cache_thread_t), KM_SLEEP);
ea3e6ca9
BB
320 if (!kct)
321 return NULL;
322
323 spin_lock_init(&kct->kct_lock);
ea3e6ca9 324 kct->kct_id = id;
efcd0ca3
BB
325 INIT_LIST_HEAD(&kct->kct_list);
326
327 spin_lock(&kcp->kcp_lock);
328 kcp->kcp_kct[id] = kct;
329 spin_unlock(&kcp->kcp_lock);
ea3e6ca9
BB
330
331 return kct;
332}
333
334static void
efcd0ca3
BB
335splat_kmem_cache_test_kct_free(kmem_cache_priv_t *kcp,
336 kmem_cache_thread_t *kct)
337{
338 spin_lock(&kcp->kcp_lock);
339 kcp->kcp_kct[kct->kct_id] = NULL;
340 spin_unlock(&kcp->kcp_lock);
341
342 kmem_free(kct, sizeof(kmem_cache_thread_t));
343}
344
345static void
346splat_kmem_cache_test_kcd_free(kmem_cache_priv_t *kcp,
347 kmem_cache_thread_t *kct)
348{
349 kmem_cache_data_t *kcd;
350
351 spin_lock(&kct->kct_lock);
352 while (!list_empty(&kct->kct_list)) {
353 kcd = list_entry(kct->kct_list.next,
354 kmem_cache_data_t, kcd_node);
355 list_del(&kcd->kcd_node);
356 spin_unlock(&kct->kct_lock);
357
358 kmem_cache_free(kcp->kcp_cache, kcd);
359
360 spin_lock(&kct->kct_lock);
361 }
362 spin_unlock(&kct->kct_lock);
363}
364
365static int
366splat_kmem_cache_test_kcd_alloc(kmem_cache_priv_t *kcp,
367 kmem_cache_thread_t *kct, int count)
ea3e6ca9 368{
efcd0ca3
BB
369 kmem_cache_data_t *kcd;
370 int i;
371
372 for (i = 0; i < count; i++) {
373 kcd = kmem_cache_alloc(kcp->kcp_cache, KM_SLEEP);
374 if (kcd == NULL) {
375 splat_kmem_cache_test_kcd_free(kcp, kct);
376 return -ENOMEM;
377 }
378
379 spin_lock(&kct->kct_lock);
380 list_add_tail(&kcd->kcd_node, &kct->kct_list);
381 spin_unlock(&kct->kct_lock);
382 }
383
384 return 0;
ea3e6ca9
BB
385}
386
a9a7a01c
PS
387static void
388splat_kmem_cache_test_debug(struct file *file, char *name,
389 kmem_cache_priv_t *kcp)
390{
391 int j;
392
a073aeb0
BB
393 splat_vprint(file, name, "%s cache objects %d",
394 kcp->kcp_cache->skc_name, kcp->kcp_count);
395
396 if (kcp->kcp_cache->skc_flags & (KMC_KMEM | KMC_VMEM)) {
397 splat_vprint(file, name, ", slabs %u/%u objs %u/%u",
a9a7a01c
PS
398 (unsigned)kcp->kcp_cache->skc_slab_alloc,
399 (unsigned)kcp->kcp_cache->skc_slab_total,
400 (unsigned)kcp->kcp_cache->skc_obj_alloc,
401 (unsigned)kcp->kcp_cache->skc_obj_total);
402
a073aeb0
BB
403 if (!(kcp->kcp_cache->skc_flags & KMC_NOMAGAZINE)) {
404 splat_vprint(file, name, "%s", "mags");
405
406 for_each_online_cpu(j)
407 splat_print(file, "%u/%u ",
408 kcp->kcp_cache->skc_mag[j]->skm_avail,
409 kcp->kcp_cache->skc_mag[j]->skm_size);
410 }
411 }
a9a7a01c
PS
412
413 splat_print(file, "%s\n", "");
414}
415
f1ca4da6 416static int
2fb9b26a 417splat_kmem_cache_test_constructor(void *ptr, void *priv, int flags)
f1ca4da6 418{
f1ca4da6 419 kmem_cache_priv_t *kcp = (kmem_cache_priv_t *)priv;
2fb9b26a 420 kmem_cache_data_t *kcd = (kmem_cache_data_t *)ptr;
f1ca4da6 421
0498e6c5 422 if (kcd && kcp) {
423 kcd->kcd_magic = kcp->kcp_magic;
efcd0ca3 424 INIT_LIST_HEAD(&kcd->kcd_node);
2fb9b26a 425 kcd->kcd_flag = 1;
0498e6c5 426 memset(kcd->kcd_buf, 0xaa, kcp->kcp_size - (sizeof *kcd));
427 kcp->kcp_count++;
f1ca4da6 428 }
429
430 return 0;
431}
432
433static void
2fb9b26a 434splat_kmem_cache_test_destructor(void *ptr, void *priv)
f1ca4da6 435{
f1ca4da6 436 kmem_cache_priv_t *kcp = (kmem_cache_priv_t *)priv;
2fb9b26a 437 kmem_cache_data_t *kcd = (kmem_cache_data_t *)ptr;
f1ca4da6 438
0498e6c5 439 if (kcd && kcp) {
440 kcd->kcd_magic = 0;
2fb9b26a 441 kcd->kcd_flag = 0;
0498e6c5 442 memset(kcd->kcd_buf, 0xbb, kcp->kcp_size - (sizeof *kcd));
443 kcp->kcp_count--;
f1ca4da6 444 }
445
446 return;
447}
448
ea3e6ca9
BB
449/*
450 * Generic reclaim function which assumes that all objects may
451 * be reclaimed at any time. We free a small percentage of the
452 * objects linked off the kcp or kct[] every time we are called.
453 */
454static void
455splat_kmem_cache_test_reclaim(void *priv)
456{
457 kmem_cache_priv_t *kcp = (kmem_cache_priv_t *)priv;
458 kmem_cache_thread_t *kct;
efcd0ca3
BB
459 kmem_cache_data_t *kcd;
460 LIST_HEAD(reclaim);
461 int i, count;
ea3e6ca9
BB
462
463 ASSERT(kcp->kcp_magic == SPLAT_KMEM_TEST_MAGIC);
ea3e6ca9 464
efcd0ca3 465 /* For each kct thread reclaim some objects */
ea3e6ca9 466 spin_lock(&kcp->kcp_lock);
efcd0ca3 467 for (i = 0; i < SPLAT_KMEM_THREADS; i++) {
ea3e6ca9 468 kct = kcp->kcp_kct[i];
efcd0ca3 469 if (!kct)
ea3e6ca9
BB
470 continue;
471
efcd0ca3 472 spin_unlock(&kcp->kcp_lock);
ea3e6ca9 473 spin_lock(&kct->kct_lock);
ea3e6ca9 474
efcd0ca3
BB
475 count = SPLAT_KMEM_OBJ_RECLAIM;
476 while (count > 0 && !list_empty(&kct->kct_list)) {
477 kcd = list_entry(kct->kct_list.next,
478 kmem_cache_data_t, kcd_node);
479 list_del(&kcd->kcd_node);
480 list_add(&kcd->kcd_node, &reclaim);
481 count--;
ea3e6ca9 482 }
efcd0ca3 483
ea3e6ca9 484 spin_unlock(&kct->kct_lock);
efcd0ca3
BB
485 spin_lock(&kcp->kcp_lock);
486 }
487 spin_unlock(&kcp->kcp_lock);
488
489 /* Freed outside the spin lock */
490 while (!list_empty(&reclaim)) {
491 kcd = list_entry(reclaim.next, kmem_cache_data_t, kcd_node);
492 list_del(&kcd->kcd_node);
493 kmem_cache_free(kcp->kcp_cache, kcd);
ea3e6ca9
BB
494 }
495
496 return;
497}
498
499static int
500splat_kmem_cache_test_threads(kmem_cache_priv_t *kcp, int threads)
501{
502 int rc;
503
504 spin_lock(&kcp->kcp_lock);
505 rc = (kcp->kcp_kct_count == threads);
506 spin_unlock(&kcp->kcp_lock);
507
508 return rc;
509}
510
511static int
512splat_kmem_cache_test_flags(kmem_cache_priv_t *kcp, int flags)
513{
514 int rc;
515
516 spin_lock(&kcp->kcp_lock);
517 rc = (kcp->kcp_flags & flags);
518 spin_unlock(&kcp->kcp_lock);
519
520 return rc;
521}
522
523static void
524splat_kmem_cache_test_thread(void *arg)
525{
526 kmem_cache_priv_t *kcp = (kmem_cache_priv_t *)arg;
527 kmem_cache_thread_t *kct;
efcd0ca3 528 int rc = 0, id;
ea3e6ca9
BB
529
530 ASSERT(kcp->kcp_magic == SPLAT_KMEM_TEST_MAGIC);
531
532 /* Assign thread ids */
533 spin_lock(&kcp->kcp_lock);
534 if (kcp->kcp_kct_count == -1)
535 kcp->kcp_kct_count = 0;
536
537 id = kcp->kcp_kct_count;
538 kcp->kcp_kct_count++;
539 spin_unlock(&kcp->kcp_lock);
540
efcd0ca3 541 kct = splat_kmem_cache_test_kct_alloc(kcp, id);
ea3e6ca9
BB
542 if (!kct) {
543 rc = -ENOMEM;
544 goto out;
545 }
546
ea3e6ca9
BB
547 /* Wait for all threads to have started and report they are ready */
548 if (kcp->kcp_kct_count == SPLAT_KMEM_THREADS)
549 wake_up(&kcp->kcp_ctl_waitq);
550
551 wait_event(kcp->kcp_thr_waitq,
552 splat_kmem_cache_test_flags(kcp, KCP_FLAG_READY));
553
efcd0ca3
BB
554 /* Create and destroy objects */
555 rc = splat_kmem_cache_test_kcd_alloc(kcp, kct, kcp->kcp_alloc);
556 splat_kmem_cache_test_kcd_free(kcp, kct);
ea3e6ca9 557out:
efcd0ca3
BB
558 if (kct)
559 splat_kmem_cache_test_kct_free(kcp, kct);
ea3e6ca9 560
efcd0ca3 561 spin_lock(&kcp->kcp_lock);
ea3e6ca9
BB
562 if (!kcp->kcp_rc)
563 kcp->kcp_rc = rc;
564
565 if ((--kcp->kcp_kct_count) == 0)
566 wake_up(&kcp->kcp_ctl_waitq);
567
568 spin_unlock(&kcp->kcp_lock);
569
570 thread_exit();
571}
572
f1ca4da6 573static int
48e0606a 574splat_kmem_cache_test(struct file *file, void *arg, char *name,
ea3e6ca9 575 int size, int align, int flags)
f1ca4da6 576{
ea3e6ca9 577 kmem_cache_priv_t *kcp;
efcd0ca3 578 kmem_cache_data_t *kcd = NULL;
f1ca4da6 579 int rc = 0, max;
580
efcd0ca3 581 kcp = splat_kmem_cache_test_kcp_alloc(file, name, size, align, 0);
ea3e6ca9
BB
582 if (!kcp) {
583 splat_vprint(file, name, "Unable to create '%s'\n", "kcp");
584 return -ENOMEM;
585 }
586
587 kcp->kcp_cache =
588 kmem_cache_create(SPLAT_KMEM_CACHE_NAME,
589 kcp->kcp_size, kcp->kcp_align,
590 splat_kmem_cache_test_constructor,
591 splat_kmem_cache_test_destructor,
592 NULL, kcp, NULL, flags);
593 if (!kcp->kcp_cache) {
2fb9b26a 594 splat_vprint(file, name,
ea3e6ca9 595 "Unable to create '%s'\n",
3f412673 596 SPLAT_KMEM_CACHE_NAME);
ea3e6ca9
BB
597 rc = -ENOMEM;
598 goto out_free;
f1ca4da6 599 }
600
ea3e6ca9 601 kcd = kmem_cache_alloc(kcp->kcp_cache, KM_SLEEP);
f1ca4da6 602 if (!kcd) {
2fb9b26a 603 splat_vprint(file, name,
ea3e6ca9
BB
604 "Unable to allocate from '%s'\n",
605 SPLAT_KMEM_CACHE_NAME);
f1ca4da6 606 rc = -EINVAL;
607 goto out_free;
608 }
609
efcd0ca3 610 if (!kcd->kcd_flag) {
2fb9b26a 611 splat_vprint(file, name,
ea3e6ca9
BB
612 "Failed to run contructor for '%s'\n",
613 SPLAT_KMEM_CACHE_NAME);
f1ca4da6 614 rc = -EINVAL;
615 goto out_free;
616 }
617
efcd0ca3 618 if (kcd->kcd_magic != kcp->kcp_magic) {
2fb9b26a 619 splat_vprint(file, name,
ea3e6ca9
BB
620 "Failed to pass private data to constructor "
621 "for '%s'\n", SPLAT_KMEM_CACHE_NAME);
f1ca4da6 622 rc = -EINVAL;
623 goto out_free;
624 }
625
ea3e6ca9 626 max = kcp->kcp_count;
efcd0ca3 627 kmem_cache_free(kcp->kcp_cache, kcd);
f1ca4da6 628
629 /* Destroy the entire cache which will force destructors to
630 * run and we can verify one was called for every object */
ea3e6ca9
BB
631 kmem_cache_destroy(kcp->kcp_cache);
632 if (kcp->kcp_count) {
2fb9b26a 633 splat_vprint(file, name,
ea3e6ca9
BB
634 "Failed to run destructor on all slab objects "
635 "for '%s'\n", SPLAT_KMEM_CACHE_NAME);
f1ca4da6 636 rc = -EINVAL;
637 }
638
f250d90b 639 splat_kmem_cache_test_kcp_free(kcp);
2fb9b26a 640 splat_vprint(file, name,
ea3e6ca9
BB
641 "Successfully ran ctors/dtors for %d elements in '%s'\n",
642 max, SPLAT_KMEM_CACHE_NAME);
f1ca4da6 643
644 return rc;
645
646out_free:
efcd0ca3
BB
647 if (kcd)
648 kmem_cache_free(kcp->kcp_cache, kcd);
ea3e6ca9
BB
649
650 if (kcp->kcp_cache)
651 kmem_cache_destroy(kcp->kcp_cache);
652
653 splat_kmem_cache_test_kcp_free(kcp);
654
655 return rc;
656}
657
658static int
659splat_kmem_cache_thread_test(struct file *file, void *arg, char *name,
10a4be0f 660 int size, int alloc, int max_time)
ea3e6ca9
BB
661{
662 kmem_cache_priv_t *kcp;
663 kthread_t *thr;
664 struct timespec start, stop, delta;
665 char cache_name[32];
666 int i, rc = 0;
667
efcd0ca3 668 kcp = splat_kmem_cache_test_kcp_alloc(file, name, size, 0, alloc);
ea3e6ca9
BB
669 if (!kcp) {
670 splat_vprint(file, name, "Unable to create '%s'\n", "kcp");
671 return -ENOMEM;
672 }
673
674 (void)snprintf(cache_name, 32, "%s-%d-%d",
675 SPLAT_KMEM_CACHE_NAME, size, alloc);
676 kcp->kcp_cache =
677 kmem_cache_create(cache_name, kcp->kcp_size, 0,
678 splat_kmem_cache_test_constructor,
679 splat_kmem_cache_test_destructor,
680 splat_kmem_cache_test_reclaim,
3c9ce2bf 681 kcp, NULL, 0);
ea3e6ca9
BB
682 if (!kcp->kcp_cache) {
683 splat_vprint(file, name, "Unable to create '%s'\n", cache_name);
684 rc = -ENOMEM;
685 goto out_kcp;
686 }
687
df2c0f18 688 getnstimeofday(&start);
ea3e6ca9
BB
689
690 for (i = 0; i < SPLAT_KMEM_THREADS; i++) {
691 thr = thread_create(NULL, 0,
692 splat_kmem_cache_test_thread,
693 kcp, 0, &p0, TS_RUN, minclsyspri);
694 if (thr == NULL) {
695 rc = -ESRCH;
696 goto out_cache;
697 }
698 }
699
700 /* Sleep until all threads have started, then set the ready
701 * flag and wake them all up for maximum concurrency. */
702 wait_event(kcp->kcp_ctl_waitq,
703 splat_kmem_cache_test_threads(kcp, SPLAT_KMEM_THREADS));
704
705 spin_lock(&kcp->kcp_lock);
706 kcp->kcp_flags |= KCP_FLAG_READY;
707 spin_unlock(&kcp->kcp_lock);
708 wake_up_all(&kcp->kcp_thr_waitq);
709
710 /* Sleep until all thread have finished */
711 wait_event(kcp->kcp_ctl_waitq, splat_kmem_cache_test_threads(kcp, 0));
712
df2c0f18 713 getnstimeofday(&stop);
ea3e6ca9 714 delta = timespec_sub(stop, start);
f1b59d26 715
ea3e6ca9
BB
716 splat_vprint(file, name,
717 "%-22s %2ld.%09ld\t"
718 "%lu/%lu/%lu\t%lu/%lu/%lu\n",
719 kcp->kcp_cache->skc_name,
720 delta.tv_sec, delta.tv_nsec,
721 (unsigned long)kcp->kcp_cache->skc_slab_total,
722 (unsigned long)kcp->kcp_cache->skc_slab_max,
723 (unsigned long)(kcp->kcp_alloc *
724 SPLAT_KMEM_THREADS /
725 SPL_KMEM_CACHE_OBJ_PER_SLAB),
726 (unsigned long)kcp->kcp_cache->skc_obj_total,
727 (unsigned long)kcp->kcp_cache->skc_obj_max,
728 (unsigned long)(kcp->kcp_alloc *
729 SPLAT_KMEM_THREADS));
730
10a4be0f 731 if (delta.tv_sec >= max_time)
ea3e6ca9
BB
732 rc = -ETIME;
733
734 if (!rc && kcp->kcp_rc)
735 rc = kcp->kcp_rc;
736
737out_cache:
738 kmem_cache_destroy(kcp->kcp_cache);
739out_kcp:
740 splat_kmem_cache_test_kcp_free(kcp);
f1ca4da6 741 return rc;
742}
743
a1502d76 744/* Validate small object cache behavior for dynamic/kmem/vmem caches */
2fb9b26a 745static int
746splat_kmem_test5(struct file *file, void *arg)
747{
a1502d76 748 char *name = SPLAT_KMEM_TEST5_NAME;
749 int rc;
750
ceb38728 751 /* On slab (default + kmem + vmem) */
48e0606a 752 rc = splat_kmem_cache_test(file, arg, name, 128, 0, 0);
a1502d76 753 if (rc)
754 return rc;
755
48e0606a 756 rc = splat_kmem_cache_test(file, arg, name, 128, 0, KMC_KMEM);
a1502d76 757 if (rc)
758 return rc;
759
ceb38728
BB
760 rc = splat_kmem_cache_test(file, arg, name, 128, 0, KMC_VMEM);
761 if (rc)
762 return rc;
763
764 /* Off slab (default + kmem + vmem) */
765 rc = splat_kmem_cache_test(file, arg, name, 128, 0, KMC_OFFSLAB);
766 if (rc)
767 return rc;
768
769 rc = splat_kmem_cache_test(file, arg, name, 128, 0,
770 KMC_KMEM | KMC_OFFSLAB);
771 if (rc)
772 return rc;
773
774 rc = splat_kmem_cache_test(file, arg, name, 128, 0,
775 KMC_VMEM | KMC_OFFSLAB);
776
777 return rc;
2fb9b26a 778}
779
efcd0ca3
BB
780/*
781 * Validate large object cache behavior for dynamic/kmem/vmem caches
782 */
2fb9b26a 783static int
784splat_kmem_test6(struct file *file, void *arg)
785{
a1502d76 786 char *name = SPLAT_KMEM_TEST6_NAME;
787 int rc;
788
ceb38728 789 /* On slab (default + kmem + vmem) */
e0dcb22e 790 rc = splat_kmem_cache_test(file, arg, name, 256*1024, 0, 0);
a1502d76 791 if (rc)
792 return rc;
793
e0dcb22e 794 rc = splat_kmem_cache_test(file, arg, name, 64*1024, 0, KMC_KMEM);
a1502d76 795 if (rc)
796 return rc;
797
ceb38728
BB
798 rc = splat_kmem_cache_test(file, arg, name, 1024*1024, 0, KMC_VMEM);
799 if (rc)
800 return rc;
801
802 /* Off slab (default + kmem + vmem) */
803 rc = splat_kmem_cache_test(file, arg, name, 256*1024, 0, KMC_OFFSLAB);
804 if (rc)
805 return rc;
806
807 rc = splat_kmem_cache_test(file, arg, name, 64*1024, 0,
808 KMC_KMEM | KMC_OFFSLAB);
809 if (rc)
810 return rc;
811
812 rc = splat_kmem_cache_test(file, arg, name, 1024*1024, 0,
813 KMC_VMEM | KMC_OFFSLAB);
814
815 return rc;
2fb9b26a 816}
817
efcd0ca3
BB
818/*
819 * Validate object alignment cache behavior for caches
820 */
ea3e6ca9
BB
821static int
822splat_kmem_test7(struct file *file, void *arg)
f1ca4da6 823{
ea3e6ca9
BB
824 char *name = SPLAT_KMEM_TEST7_NAME;
825 int i, rc;
2fb9b26a 826
8b45dda2 827 for (i = SPL_KMEM_CACHE_ALIGN; i <= PAGE_SIZE; i *= 2) {
ea3e6ca9
BB
828 rc = splat_kmem_cache_test(file, arg, name, 157, i, 0);
829 if (rc)
830 return rc;
ceb38728
BB
831
832 rc = splat_kmem_cache_test(file, arg, name, 157, i,
833 KMC_OFFSLAB);
834 if (rc)
835 return rc;
f1ca4da6 836 }
837
ea3e6ca9 838 return rc;
f1ca4da6 839}
840
efcd0ca3
BB
841/*
842 * Validate kmem_cache_reap() by requesting the slab cache free any objects
843 * it can. For a few reasons this may not immediately result in more free
844 * memory even if objects are freed. First off, due to fragmentation we
845 * may not be able to reclaim any slabs. Secondly, even if we do we fully
846 * clear some slabs we will not want to immediately reclaim all of them
847 * because we may contend with cache allocations and thrash. What we want
848 * to see is the slab size decrease more gradually as it becomes clear they
849 * will not be needed. This should be achievable in less than a minute.
850 * If it takes longer than this something has gone wrong.
851 */
f1ca4da6 852static int
ea3e6ca9 853splat_kmem_test8(struct file *file, void *arg)
f1ca4da6 854{
ea3e6ca9 855 kmem_cache_priv_t *kcp;
efcd0ca3 856 kmem_cache_thread_t *kct;
0936c344 857 unsigned int spl_kmem_cache_expire_old;
a9a7a01c 858 int i, rc = 0;
ea3e6ca9 859
0936c344
BB
860 /* Enable cache aging just for this test if it is disabled */
861 spl_kmem_cache_expire_old = spl_kmem_cache_expire;
862 spl_kmem_cache_expire = KMC_EXPIRE_AGE;
863
ea3e6ca9 864 kcp = splat_kmem_cache_test_kcp_alloc(file, SPLAT_KMEM_TEST8_NAME,
efcd0ca3 865 256, 0, 0);
ea3e6ca9
BB
866 if (!kcp) {
867 splat_vprint(file, SPLAT_KMEM_TEST8_NAME,
868 "Unable to create '%s'\n", "kcp");
efcd0ca3
BB
869 rc = -ENOMEM;
870 goto out;
f1ca4da6 871 }
872
ea3e6ca9
BB
873 kcp->kcp_cache =
874 kmem_cache_create(SPLAT_KMEM_CACHE_NAME, kcp->kcp_size, 0,
875 splat_kmem_cache_test_constructor,
876 splat_kmem_cache_test_destructor,
877 splat_kmem_cache_test_reclaim,
878 kcp, NULL, 0);
879 if (!kcp->kcp_cache) {
ea3e6ca9
BB
880 splat_vprint(file, SPLAT_KMEM_TEST8_NAME,
881 "Unable to create '%s'\n", SPLAT_KMEM_CACHE_NAME);
efcd0ca3
BB
882 rc = -ENOMEM;
883 goto out_kcp;
ea3e6ca9 884 }
f1ca4da6 885
efcd0ca3
BB
886 kct = splat_kmem_cache_test_kct_alloc(kcp, 0);
887 if (!kct) {
888 splat_vprint(file, SPLAT_KMEM_TEST8_NAME,
889 "Unable to create '%s'\n", "kct");
890 rc = -ENOMEM;
891 goto out_cache;
892 }
893
894 rc = splat_kmem_cache_test_kcd_alloc(kcp, kct, SPLAT_KMEM_OBJ_COUNT);
895 if (rc) {
896 splat_vprint(file, SPLAT_KMEM_TEST8_NAME, "Unable to "
897 "allocate from '%s'\n", SPLAT_KMEM_CACHE_NAME);
898 goto out_kct;
f1ca4da6 899 }
900
668d2a0d
BB
901 /* Force reclaim every 1/10 a second for 60 seconds. */
902 for (i = 0; i < 600; i++) {
ea3e6ca9 903 kmem_cache_reap_now(kcp->kcp_cache);
a9a7a01c 904 splat_kmem_cache_test_debug(file, SPLAT_KMEM_TEST8_NAME, kcp);
ea3e6ca9 905
a073aeb0 906 if (kcp->kcp_count == 0)
2fb9b26a 907 break;
908
909 set_current_state(TASK_INTERRUPTIBLE);
668d2a0d 910 schedule_timeout(HZ / 10);
2fb9b26a 911 }
912
a073aeb0 913 if (kcp->kcp_count == 0) {
ea3e6ca9 914 splat_vprint(file, SPLAT_KMEM_TEST8_NAME,
2fb9b26a 915 "Successfully created %d objects "
916 "in cache %s and reclaimed them\n",
ea3e6ca9 917 SPLAT_KMEM_OBJ_COUNT, SPLAT_KMEM_CACHE_NAME);
2fb9b26a 918 } else {
ea3e6ca9 919 splat_vprint(file, SPLAT_KMEM_TEST8_NAME,
2fb9b26a 920 "Failed to reclaim %u/%d objects from cache %s\n",
a073aeb0 921 (unsigned)kcp->kcp_count,
ea3e6ca9 922 SPLAT_KMEM_OBJ_COUNT, SPLAT_KMEM_CACHE_NAME);
2fb9b26a 923 rc = -ENOMEM;
924 }
f1ca4da6 925
2fb9b26a 926 /* Cleanup our mess (for failure case of time expiring) */
efcd0ca3
BB
927 splat_kmem_cache_test_kcd_free(kcp, kct);
928out_kct:
929 splat_kmem_cache_test_kct_free(kcp, kct);
930out_cache:
ea3e6ca9 931 kmem_cache_destroy(kcp->kcp_cache);
efcd0ca3 932out_kcp:
ea3e6ca9 933 splat_kmem_cache_test_kcp_free(kcp);
efcd0ca3 934out:
0936c344
BB
935 spl_kmem_cache_expire = spl_kmem_cache_expire_old;
936
f1ca4da6 937 return rc;
938}
939
efcd0ca3
BB
940/* Test cache aging, we have allocated a large number of objects thus
941 * creating a large number of slabs and then free'd them all. However,
942 * since there should be little memory pressure at the moment those
943 * slabs have not been freed. What we want to see is the slab size
944 * decrease gradually as it becomes clear they will not be be needed.
945 * This should be achievable in less than minute. If it takes longer
946 * than this something has gone wrong.
947 */
ea3e6ca9
BB
948static int
949splat_kmem_test9(struct file *file, void *arg)
44b8f176 950{
ea3e6ca9 951 kmem_cache_priv_t *kcp;
efcd0ca3 952 kmem_cache_thread_t *kct;
0936c344 953 unsigned int spl_kmem_cache_expire_old;
a9a7a01c 954 int i, rc = 0, count = SPLAT_KMEM_OBJ_COUNT * 128;
ea3e6ca9 955
0936c344
BB
956 /* Enable cache aging just for this test if it is disabled */
957 spl_kmem_cache_expire_old = spl_kmem_cache_expire;
958 spl_kmem_cache_expire = KMC_EXPIRE_AGE;
959
ea3e6ca9 960 kcp = splat_kmem_cache_test_kcp_alloc(file, SPLAT_KMEM_TEST9_NAME,
efcd0ca3 961 256, 0, 0);
ea3e6ca9
BB
962 if (!kcp) {
963 splat_vprint(file, SPLAT_KMEM_TEST9_NAME,
964 "Unable to create '%s'\n", "kcp");
efcd0ca3
BB
965 rc = -ENOMEM;
966 goto out;
ea3e6ca9 967 }
44b8f176 968
ea3e6ca9
BB
969 kcp->kcp_cache =
970 kmem_cache_create(SPLAT_KMEM_CACHE_NAME, kcp->kcp_size, 0,
971 splat_kmem_cache_test_constructor,
972 splat_kmem_cache_test_destructor,
973 NULL, kcp, NULL, 0);
974 if (!kcp->kcp_cache) {
ea3e6ca9
BB
975 splat_vprint(file, SPLAT_KMEM_TEST9_NAME,
976 "Unable to create '%s'\n", SPLAT_KMEM_CACHE_NAME);
efcd0ca3
BB
977 rc = -ENOMEM;
978 goto out_kcp;
44b8f176 979 }
980
efcd0ca3
BB
981 kct = splat_kmem_cache_test_kct_alloc(kcp, 0);
982 if (!kct) {
983 splat_vprint(file, SPLAT_KMEM_TEST8_NAME,
984 "Unable to create '%s'\n", "kct");
985 rc = -ENOMEM;
986 goto out_cache;
44b8f176 987 }
988
efcd0ca3
BB
989 rc = splat_kmem_cache_test_kcd_alloc(kcp, kct, count);
990 if (rc) {
991 splat_vprint(file, SPLAT_KMEM_TEST9_NAME, "Unable to "
992 "allocate from '%s'\n", SPLAT_KMEM_CACHE_NAME);
993 goto out_kct;
994 }
995
996 splat_kmem_cache_test_kcd_free(kcp, kct);
e9d7a2be 997
ea3e6ca9 998 for (i = 0; i < 60; i++) {
a9a7a01c 999 splat_kmem_cache_test_debug(file, SPLAT_KMEM_TEST9_NAME, kcp);
ea3e6ca9 1000
a073aeb0 1001 if (kcp->kcp_count == 0)
ea3e6ca9 1002 break;
44b8f176 1003
ea3e6ca9
BB
1004 set_current_state(TASK_INTERRUPTIBLE);
1005 schedule_timeout(HZ);
1006 }
44b8f176 1007
a073aeb0 1008 if (kcp->kcp_count == 0) {
ea3e6ca9
BB
1009 splat_vprint(file, SPLAT_KMEM_TEST9_NAME,
1010 "Successfully created %d objects "
1011 "in cache %s and reclaimed them\n",
1012 count, SPLAT_KMEM_CACHE_NAME);
1013 } else {
1014 splat_vprint(file, SPLAT_KMEM_TEST9_NAME,
1015 "Failed to reclaim %u/%d objects from cache %s\n",
a073aeb0 1016 (unsigned)kcp->kcp_count, count,
ea3e6ca9
BB
1017 SPLAT_KMEM_CACHE_NAME);
1018 rc = -ENOMEM;
1019 }
1020
efcd0ca3
BB
1021out_kct:
1022 splat_kmem_cache_test_kct_free(kcp, kct);
1023out_cache:
ea3e6ca9 1024 kmem_cache_destroy(kcp->kcp_cache);
efcd0ca3 1025out_kcp:
ea3e6ca9 1026 splat_kmem_cache_test_kcp_free(kcp);
efcd0ca3 1027out:
0936c344
BB
1028 spl_kmem_cache_expire = spl_kmem_cache_expire_old;
1029
ea3e6ca9 1030 return rc;
44b8f176 1031}
1032
ea3e6ca9
BB
1033/*
1034 * This test creates N threads with a shared kmem cache. They then all
1035 * concurrently allocate and free from the cache to stress the locking and
1036 * concurrent cache performance. If any one test takes longer than 5
1037 * seconds to complete it is treated as a failure and may indicate a
1038 * performance regression. On my test system no one test takes more
1039 * than 1 second to complete so a 5x slowdown likely a problem.
44b8f176 1040 */
1041static int
ea3e6ca9 1042splat_kmem_test10(struct file *file, void *arg)
44b8f176 1043{
e11d6c5f 1044 uint64_t size, alloc, rc = 0;
44b8f176 1045
efcd0ca3 1046 for (size = 32; size <= 1024*1024; size *= 2) {
44b8f176 1047
ea3e6ca9
BB
1048 splat_vprint(file, SPLAT_KMEM_TEST10_NAME, "%-22s %s", "name",
1049 "time (sec)\tslabs \tobjs \thash\n");
1050 splat_vprint(file, SPLAT_KMEM_TEST10_NAME, "%-22s %s", "",
1051 " \ttot/max/calc\ttot/max/calc\n");
44b8f176 1052
ea3e6ca9 1053 for (alloc = 1; alloc <= 1024; alloc *= 2) {
44b8f176 1054
8bbbe46f
BB
1055 /* Skip tests which exceed 1/2 of physical memory. */
1056 if (size * alloc * SPLAT_KMEM_THREADS > physmem / 2)
ea3e6ca9 1057 continue;
7ea1cbf5 1058
ea3e6ca9 1059 rc = splat_kmem_cache_thread_test(file, arg,
10a4be0f 1060 SPLAT_KMEM_TEST10_NAME, size, alloc, 5);
ea3e6ca9
BB
1061 if (rc)
1062 break;
1063 }
44b8f176 1064 }
1065
7ea1cbf5 1066 return rc;
44b8f176 1067}
1068
11124863 1069#if 0
ea3e6ca9
BB
1070/*
1071 * This test creates N threads with a shared kmem cache which overcommits
1072 * memory by 4x. This makes it impossible for the slab to satify the
1073 * thread requirements without having its reclaim hook run which will
1074 * free objects back for use. This behavior is triggered by the linum VM
1075 * detecting a low memory condition on the node and invoking the shrinkers.
1076 * This should allow all the threads to complete while avoiding deadlock
1077 * and for the most part out of memory events. This is very tough on the
4e5691fa
BB
1078 * system so it is possible the test app may get oom'ed. This particular
1079 * test has proven troublesome on 32-bit archs with limited virtual
1080 * address space so it only run on 64-bit systems.
ea3e6ca9 1081 */
fece7c99 1082static int
ea3e6ca9 1083splat_kmem_test11(struct file *file, void *arg)
fece7c99 1084{
ea3e6ca9 1085 uint64_t size, alloc, rc;
fece7c99 1086
efcd0ca3 1087 size = 8 * 1024;
e11d6c5f 1088 alloc = ((4 * physmem * PAGE_SIZE) / size) / SPLAT_KMEM_THREADS;
fece7c99 1089
e11d6c5f 1090 splat_vprint(file, SPLAT_KMEM_TEST11_NAME, "%-22s %s", "name",
ea3e6ca9 1091 "time (sec)\tslabs \tobjs \thash\n");
e11d6c5f 1092 splat_vprint(file, SPLAT_KMEM_TEST11_NAME, "%-22s %s", "",
ea3e6ca9 1093 " \ttot/max/calc\ttot/max/calc\n");
48e0606a 1094
ea3e6ca9 1095 rc = splat_kmem_cache_thread_test(file, arg,
10a4be0f 1096 SPLAT_KMEM_TEST11_NAME, size, alloc, 60);
48e0606a
BB
1097
1098 return rc;
1099}
11124863 1100#endif
48e0606a 1101
a9a7a01c
PS
1102typedef struct dummy_page {
1103 struct list_head dp_list;
1104 char dp_pad[PAGE_SIZE - sizeof(struct list_head)];
1105} dummy_page_t;
1106
1107/*
1108 * This test is designed to verify that direct reclaim is functioning as
1109 * expected. We allocate a large number of objects thus creating a large
1110 * number of slabs. We then apply memory pressure and expect that the
1111 * direct reclaim path can easily recover those slabs. The registered
1112 * reclaim function will free the objects and the slab shrinker will call
1113 * it repeatedly until at least a single slab can be freed.
1114 *
1115 * Note it may not be possible to reclaim every last slab via direct reclaim
1116 * without a failure because the shrinker_rwsem may be contended. For this
1117 * reason, quickly reclaiming 3/4 of the slabs is considered a success.
1118 *
1119 * This should all be possible within 10 seconds. For reference, on a
1120 * system with 2G of memory this test takes roughly 0.2 seconds to run.
1121 * It may take longer on larger memory systems but should still easily
1122 * complete in the alloted 10 seconds.
1123 */
1124static int
1125splat_kmem_test13(struct file *file, void *arg)
1126{
1127 kmem_cache_priv_t *kcp;
efcd0ca3 1128 kmem_cache_thread_t *kct;
a9a7a01c
PS
1129 dummy_page_t *dp;
1130 struct list_head list;
df2c0f18 1131 struct timespec start, stop, delta = { 0, 0 };
a9a7a01c
PS
1132 int size, count, slabs, fails = 0;
1133 int i, rc = 0, max_time = 10;
1134
1135 size = 128 * 1024;
1136 count = ((physmem * PAGE_SIZE) / 4 / size);
1137
1138 kcp = splat_kmem_cache_test_kcp_alloc(file, SPLAT_KMEM_TEST13_NAME,
efcd0ca3 1139 size, 0, 0);
a9a7a01c
PS
1140 if (!kcp) {
1141 splat_vprint(file, SPLAT_KMEM_TEST13_NAME,
1142 "Unable to create '%s'\n", "kcp");
efcd0ca3
BB
1143 rc = -ENOMEM;
1144 goto out;
a9a7a01c
PS
1145 }
1146
1147 kcp->kcp_cache =
1148 kmem_cache_create(SPLAT_KMEM_CACHE_NAME, kcp->kcp_size, 0,
1149 splat_kmem_cache_test_constructor,
1150 splat_kmem_cache_test_destructor,
1151 splat_kmem_cache_test_reclaim,
1152 kcp, NULL, 0);
1153 if (!kcp->kcp_cache) {
a9a7a01c
PS
1154 splat_vprint(file, SPLAT_KMEM_TEST13_NAME,
1155 "Unable to create '%s'\n", SPLAT_KMEM_CACHE_NAME);
efcd0ca3
BB
1156 rc = -ENOMEM;
1157 goto out_kcp;
a9a7a01c
PS
1158 }
1159
efcd0ca3
BB
1160 kct = splat_kmem_cache_test_kct_alloc(kcp, 0);
1161 if (!kct) {
1162 splat_vprint(file, SPLAT_KMEM_TEST13_NAME,
1163 "Unable to create '%s'\n", "kct");
1164 rc = -ENOMEM;
1165 goto out_cache;
1166 }
1167
1168 rc = splat_kmem_cache_test_kcd_alloc(kcp, kct, count);
1169 if (rc) {
1170 splat_vprint(file, SPLAT_KMEM_TEST13_NAME, "Unable to "
1171 "allocate from '%s'\n", SPLAT_KMEM_CACHE_NAME);
1172 goto out_kct;
a9a7a01c
PS
1173 }
1174
1175 i = 0;
1176 slabs = kcp->kcp_cache->skc_slab_total;
1177 INIT_LIST_HEAD(&list);
df2c0f18 1178 getnstimeofday(&start);
a9a7a01c 1179
efcd0ca3 1180 /* Apply memory pressure */
a9a7a01c
PS
1181 while (kcp->kcp_cache->skc_slab_total > (slabs >> 2)) {
1182
1183 if ((i % 10000) == 0)
1184 splat_kmem_cache_test_debug(
1185 file, SPLAT_KMEM_TEST13_NAME, kcp);
1186
df2c0f18
RY
1187 getnstimeofday(&stop);
1188 delta = timespec_sub(stop, start);
a9a7a01c
PS
1189 if (delta.tv_sec >= max_time) {
1190 splat_vprint(file, SPLAT_KMEM_TEST13_NAME,
1191 "Failed to reclaim 3/4 of cache in %ds, "
1192 "%u/%u slabs remain\n", max_time,
1193 (unsigned)kcp->kcp_cache->skc_slab_total,
1194 slabs);
1195 rc = -ETIME;
1196 break;
1197 }
1198
668d2a0d 1199 dp = (dummy_page_t *)__get_free_page(GFP_KERNEL);
a9a7a01c
PS
1200 if (!dp) {
1201 fails++;
1202 splat_vprint(file, SPLAT_KMEM_TEST13_NAME,
1203 "Failed (%d) to allocate page with %u "
1204 "slabs still in the cache\n", fails,
1205 (unsigned)kcp->kcp_cache->skc_slab_total);
1206 continue;
1207 }
1208
1209 list_add(&dp->dp_list, &list);
1210 i++;
1211 }
1212
1213 if (rc == 0)
1214 splat_vprint(file, SPLAT_KMEM_TEST13_NAME,
1215 "Successfully created %u slabs and with %d alloc "
1216 "failures reclaimed 3/4 of them in %d.%03ds\n",
1217 slabs, fails,
1218 (int)delta.tv_sec, (int)delta.tv_nsec / 1000000);
1219
1220 /* Release memory pressure pages */
1221 while (!list_empty(&list)) {
1222 dp = list_entry(list.next, dummy_page_t, dp_list);
1223 list_del_init(&dp->dp_list);
1224 free_page((unsigned long)dp);
1225 }
1226
1227 /* Release remaining kmem cache objects */
efcd0ca3
BB
1228 splat_kmem_cache_test_kcd_free(kcp, kct);
1229out_kct:
1230 splat_kmem_cache_test_kct_free(kcp, kct);
1231out_cache:
a9a7a01c 1232 kmem_cache_destroy(kcp->kcp_cache);
efcd0ca3 1233out_kcp:
a9a7a01c 1234 splat_kmem_cache_test_kcp_free(kcp);
efcd0ca3 1235out:
a9a7a01c
PS
1236 return rc;
1237}
1238
7c50328b 1239splat_subsystem_t *
1240splat_kmem_init(void)
f1ca4da6 1241{
ea3e6ca9 1242 splat_subsystem_t *sub;
f1ca4da6 1243
ea3e6ca9
BB
1244 sub = kmalloc(sizeof(*sub), GFP_KERNEL);
1245 if (sub == NULL)
1246 return NULL;
f1ca4da6 1247
ea3e6ca9
BB
1248 memset(sub, 0, sizeof(*sub));
1249 strncpy(sub->desc.name, SPLAT_KMEM_NAME, SPLAT_NAME_SIZE);
7c50328b 1250 strncpy(sub->desc.desc, SPLAT_KMEM_DESC, SPLAT_DESC_SIZE);
ea3e6ca9 1251 INIT_LIST_HEAD(&sub->subsystem_list);
f1ca4da6 1252 INIT_LIST_HEAD(&sub->test_list);
ea3e6ca9
BB
1253 spin_lock_init(&sub->test_lock);
1254 sub->desc.id = SPLAT_SUBSYSTEM_KMEM;
1255
1256 SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST1_NAME, SPLAT_KMEM_TEST1_DESC,
1257 SPLAT_KMEM_TEST1_ID, splat_kmem_test1);
1258 SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST2_NAME, SPLAT_KMEM_TEST2_DESC,
1259 SPLAT_KMEM_TEST2_ID, splat_kmem_test2);
1260 SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST3_NAME, SPLAT_KMEM_TEST3_DESC,
1261 SPLAT_KMEM_TEST3_ID, splat_kmem_test3);
1262 SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST4_NAME, SPLAT_KMEM_TEST4_DESC,
1263 SPLAT_KMEM_TEST4_ID, splat_kmem_test4);
1264 SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST5_NAME, SPLAT_KMEM_TEST5_DESC,
1265 SPLAT_KMEM_TEST5_ID, splat_kmem_test5);
1266 SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST6_NAME, SPLAT_KMEM_TEST6_DESC,
1267 SPLAT_KMEM_TEST6_ID, splat_kmem_test6);
1268 SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST7_NAME, SPLAT_KMEM_TEST7_DESC,
1269 SPLAT_KMEM_TEST7_ID, splat_kmem_test7);
1270 SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST8_NAME, SPLAT_KMEM_TEST8_DESC,
1271 SPLAT_KMEM_TEST8_ID, splat_kmem_test8);
1272 SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST9_NAME, SPLAT_KMEM_TEST9_DESC,
1273 SPLAT_KMEM_TEST9_ID, splat_kmem_test9);
1274 SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST10_NAME, SPLAT_KMEM_TEST10_DESC,
1275 SPLAT_KMEM_TEST10_ID, splat_kmem_test10);
11124863 1276#if 0
ea3e6ca9
BB
1277 SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST11_NAME, SPLAT_KMEM_TEST11_DESC,
1278 SPLAT_KMEM_TEST11_ID, splat_kmem_test11);
11124863 1279#endif
a9a7a01c
PS
1280 SPLAT_TEST_INIT(sub, SPLAT_KMEM_TEST13_NAME, SPLAT_KMEM_TEST13_DESC,
1281 SPLAT_KMEM_TEST13_ID, splat_kmem_test13);
ea3e6ca9
BB
1282
1283 return sub;
f1ca4da6 1284}
1285
1286void
7c50328b 1287splat_kmem_fini(splat_subsystem_t *sub)
f1ca4da6 1288{
ea3e6ca9 1289 ASSERT(sub);
a9a7a01c 1290 SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST13_ID);
11124863 1291#if 0
ea3e6ca9 1292 SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST11_ID);
11124863 1293#endif
ea3e6ca9
BB
1294 SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST10_ID);
1295 SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST9_ID);
1296 SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST8_ID);
1297 SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST7_ID);
1298 SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST6_ID);
1299 SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST5_ID);
1300 SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST4_ID);
1301 SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST3_ID);
1302 SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST2_ID);
1303 SPLAT_TEST_FINI(sub, SPLAT_KMEM_TEST1_ID);
1304
1305 kfree(sub);
f1ca4da6 1306}
1307
1308int
7c50328b 1309splat_kmem_id(void) {
ea3e6ca9 1310 return SPLAT_SUBSYSTEM_KMEM;
f1ca4da6 1311}