2 * ZPIOS is a heavily modified version of the original PIOS test code.
3 * It is designed to have the test code running in the Linux kernel
4 * against ZFS while still being flexibly controled from user space.
6 * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC.
7 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
8 * Written by Brian Behlendorf <behlendorf1@llnl.gov>.
11 * Original PIOS Test Code
12 * Copyright (C) 2004 Cluster File Systems, Inc.
13 * Written by Peter Braam <braam@clusterfs.com>
14 * Atul Vidwansa <atul@clusterfs.com>
15 * Milind Dumbare <milind@clusterfs.com>
17 * This file is part of ZFS on Linux.
18 * For details, see <http://zfsonlinux.org/>.
20 * ZPIOS is free software; you can redistribute it and/or modify it
21 * under the terms of the GNU General Public License as published by the
22 * Free Software Foundation; either version 2 of the License, or (at your
23 * option) any later version.
25 * ZPIOS is distributed in the hope that it will be useful, but WITHOUT
26 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
27 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
30 * You should have received a copy of the GNU General Public License along
31 * with ZPIOS. If not, see <http://www.gnu.org/licenses/>.
34 #include <sys/zfs_context.h>
37 #include <sys/dsl_destroy.h>
38 #include <linux/cdev.h>
39 #include "zpios-internal.h"
42 static spl_class
*zpios_class
;
43 static spl_device
*zpios_device
;
44 static char *zpios_tag
= "zpios_tag";
47 zpios_upcall(char *path
, char *phase
, run_args_t
*run_args
, int rc
)
50 * This is stack heavy but it should be OK since we are only
51 * making the upcall between tests when the stack is shallow.
53 char id
[16], chunk_size
[16], region_size
[16], thread_count
[16];
54 char region_count
[16], offset
[16], region_noise
[16], chunk_noise
[16];
55 char thread_delay
[16], flags
[16], result
[8];
56 char *argv
[16], *envp
[4];
58 if ((path
== NULL
) || (strlen(path
) == 0))
61 snprintf(id
, 15, "%d", run_args
->id
);
62 snprintf(chunk_size
, 15, "%lu", (long unsigned)run_args
->chunk_size
);
63 snprintf(region_size
, 15, "%lu", (long unsigned) run_args
->region_size
);
64 snprintf(thread_count
, 15, "%u", run_args
->thread_count
);
65 snprintf(region_count
, 15, "%u", run_args
->region_count
);
66 snprintf(offset
, 15, "%lu", (long unsigned)run_args
->offset
);
67 snprintf(region_noise
, 15, "%u", run_args
->region_noise
);
68 snprintf(chunk_noise
, 15, "%u", run_args
->chunk_noise
);
69 snprintf(thread_delay
, 15, "%u", run_args
->thread_delay
);
70 snprintf(flags
, 15, "0x%x", run_args
->flags
);
71 snprintf(result
, 7, "%d", rc
);
73 /* Passing 15 args to registered pre/post upcall */
76 argv
[2] = strlen(run_args
->log
) ? run_args
->log
: "<none>";
78 argv
[4] = run_args
->pool
;
80 argv
[6] = region_size
;
81 argv
[7] = thread_count
;
82 argv
[8] = region_count
;
84 argv
[10] = region_noise
;
85 argv
[11] = chunk_noise
;
86 argv
[12] = thread_delay
;
91 /* Passing environment for user space upcall */
93 envp
[1] = "TERM=linux";
94 envp
[2] = "PATH=/sbin:/usr/sbin:/bin:/usr/bin";
97 return (call_usermodehelper(path
, argv
, envp
, UMH_WAIT_PROC
));
101 zpios_print(struct file
*file
, const char *format
, ...)
103 zpios_info_t
*info
= (zpios_info_t
*)file
->private_data
;
108 ASSERT(info
->info_buffer
);
110 va_start(adx
, format
);
111 spin_lock(&info
->info_lock
);
113 /* Don't allow the kernel to start a write in the red zone */
114 if ((int)(info
->info_head
- info
->info_buffer
) >
115 (info
->info_size
- ZPIOS_INFO_BUFFER_REDZONE
)) {
118 rc
= vsprintf(info
->info_head
, format
, adx
);
120 info
->info_head
+= rc
;
123 spin_unlock(&info
->info_lock
);
130 zpios_dmu_object_create(run_args_t
*run_args
, objset_t
*os
)
136 tx
= dmu_tx_create(os
);
137 dmu_tx_hold_write(tx
, DMU_NEW_OBJECT
, 0, OBJ_SIZE
);
138 rc
= dmu_tx_assign(tx
, TXG_WAIT
);
140 zpios_print(run_args
->file
,
141 "dmu_tx_assign() failed: %d\n", rc
);
146 obj
= dmu_object_alloc(os
, DMU_OT_UINT64_OTHER
, 0, DMU_OT_NONE
, 0, tx
);
147 rc
= dmu_object_set_blocksize(os
, obj
, 128ULL << 10, 0, tx
);
149 zpios_print(run_args
->file
,
150 "dmu_object_set_blocksize() failed: %d\n", rc
);
161 zpios_dmu_object_free(run_args_t
*run_args
, objset_t
*os
, uint64_t obj
)
166 tx
= dmu_tx_create(os
);
167 dmu_tx_hold_free(tx
, obj
, 0, DMU_OBJECT_END
);
168 rc
= dmu_tx_assign(tx
, TXG_WAIT
);
170 zpios_print(run_args
->file
,
171 "dmu_tx_assign() failed: %d\n", rc
);
176 rc
= dmu_object_free(os
, obj
, tx
);
178 zpios_print(run_args
->file
,
179 "dmu_object_free() failed: %d\n", rc
);
190 zpios_dmu_setup(run_args_t
*run_args
)
192 zpios_time_t
*t
= &(run_args
->stats
.cr_time
);
198 (void) zpios_upcall(run_args
->pre
, PHASE_PRE_CREATE
, run_args
, 0);
199 t
->start
= zpios_timespec_now();
201 (void) snprintf(name
, 32, "%s/id_%d", run_args
->pool
, run_args
->id
);
202 rc
= dmu_objset_create(name
, DMU_OST_OTHER
, 0, NULL
, NULL
);
204 zpios_print(run_args
->file
, "Error dmu_objset_create(%s, ...) "
205 "failed: %d\n", name
, rc
);
209 rc
= dmu_objset_own(name
, DMU_OST_OTHER
, 0, zpios_tag
, &os
);
211 zpios_print(run_args
->file
, "Error dmu_objset_own(%s, ...) "
212 "failed: %d\n", name
, rc
);
216 if (!(run_args
->flags
& DMU_FPP
)) {
217 obj
= zpios_dmu_object_create(run_args
, os
);
220 zpios_print(run_args
->file
, "Error zpios_dmu_"
221 "object_create() failed, %d\n", rc
);
226 for (i
= 0; i
< run_args
->region_count
; i
++) {
227 zpios_region_t
*region
;
229 region
= &run_args
->regions
[i
];
230 mutex_init(®ion
->lock
, NULL
, MUTEX_DEFAULT
, NULL
);
232 if (run_args
->flags
& DMU_FPP
) {
233 /* File per process */
235 region
->obj
.obj
= zpios_dmu_object_create(run_args
, os
);
236 ASSERT(region
->obj
.obj
> 0); /* XXX - Handle this */
237 region
->wr_offset
= run_args
->offset
;
238 region
->rd_offset
= run_args
->offset
;
239 region
->init_offset
= run_args
->offset
;
240 region
->max_offset
= run_args
->offset
+
241 run_args
->region_size
;
243 /* Single shared file */
245 region
->obj
.obj
= obj
;
246 region
->wr_offset
= run_args
->offset
* i
;
247 region
->rd_offset
= run_args
->offset
* i
;
248 region
->init_offset
= run_args
->offset
* i
;
249 region
->max_offset
= run_args
->offset
*
250 i
+ run_args
->region_size
;
257 rc2
= dsl_destroy_head(name
);
259 zpios_print(run_args
->file
, "Error dsl_destroy_head"
260 "(%s, ...) failed: %d\n", name
, rc2
);
263 t
->stop
= zpios_timespec_now();
264 t
->delta
= zpios_timespec_sub(t
->stop
, t
->start
);
265 (void) zpios_upcall(run_args
->post
, PHASE_POST_CREATE
, run_args
, rc
);
271 zpios_setup_run(run_args_t
**run_args
, zpios_cmd_t
*kcmd
, struct file
*file
)
276 size
= sizeof (*ra
) + kcmd
->cmd_region_count
* sizeof (zpios_region_t
);
278 ra
= vmem_zalloc(size
, KM_SLEEP
);
280 zpios_print(file
, "Unable to vmem_zalloc() %d bytes "
281 "for regions\n", size
);
286 strncpy(ra
->pool
, kcmd
->cmd_pool
, ZPIOS_NAME_SIZE
- 1);
287 strncpy(ra
->pre
, kcmd
->cmd_pre
, ZPIOS_PATH_SIZE
- 1);
288 strncpy(ra
->post
, kcmd
->cmd_post
, ZPIOS_PATH_SIZE
- 1);
289 strncpy(ra
->log
, kcmd
->cmd_log
, ZPIOS_PATH_SIZE
- 1);
290 ra
->id
= kcmd
->cmd_id
;
291 ra
->chunk_size
= kcmd
->cmd_chunk_size
;
292 ra
->thread_count
= kcmd
->cmd_thread_count
;
293 ra
->region_count
= kcmd
->cmd_region_count
;
294 ra
->region_size
= kcmd
->cmd_region_size
;
295 ra
->offset
= kcmd
->cmd_offset
;
296 ra
->region_noise
= kcmd
->cmd_region_noise
;
297 ra
->chunk_noise
= kcmd
->cmd_chunk_noise
;
298 ra
->thread_delay
= kcmd
->cmd_thread_delay
;
299 ra
->flags
= kcmd
->cmd_flags
;
300 ra
->stats
.wr_data
= 0;
301 ra
->stats
.wr_chunks
= 0;
302 ra
->stats
.rd_data
= 0;
303 ra
->stats
.rd_chunks
= 0;
306 mutex_init(&ra
->lock_work
, NULL
, MUTEX_DEFAULT
, NULL
);
307 mutex_init(&ra
->lock_ctl
, NULL
, MUTEX_DEFAULT
, NULL
);
309 (void) zpios_upcall(ra
->pre
, PHASE_PRE_RUN
, ra
, 0);
311 rc
= zpios_dmu_setup(ra
);
313 mutex_destroy(&ra
->lock_ctl
);
314 mutex_destroy(&ra
->lock_work
);
323 zpios_get_work_item(run_args_t
*run_args
, dmu_obj_t
*obj
, __u64
*offset
,
324 __u32
*chunk_size
, zpios_region_t
**region
, __u32 flags
)
327 unsigned int random_int
;
329 get_random_bytes(&random_int
, sizeof (unsigned int));
331 mutex_enter(&run_args
->lock_work
);
332 i
= run_args
->region_next
;
335 * XXX: I don't much care for this chunk selection mechansim
336 * there's the potential to burn a lot of time here doing nothing
337 * useful while holding the global lock. This could give some
338 * misleading performance results. I'll fix it latter.
340 while (count
< run_args
->region_count
) {
342 zpios_time_t
*rw_time
;
344 j
= i
% run_args
->region_count
;
345 *region
= &(run_args
->regions
[j
]);
347 if (flags
& DMU_WRITE
) {
348 rw_offset
= &((*region
)->wr_offset
);
349 rw_time
= &((*region
)->stats
.wr_time
);
351 rw_offset
= &((*region
)->rd_offset
);
352 rw_time
= &((*region
)->stats
.rd_time
);
355 /* test if region is fully written */
356 if (*rw_offset
+ *chunk_size
> (*region
)->max_offset
) {
360 if (unlikely(rw_time
->stop
.ts_sec
== 0) &&
361 unlikely(rw_time
->stop
.ts_nsec
== 0))
362 rw_time
->stop
= zpios_timespec_now();
367 *offset
= *rw_offset
;
368 *obj
= (*region
)->obj
;
369 *rw_offset
+= *chunk_size
;
371 /* update ctl structure */
372 if (run_args
->region_noise
) {
373 get_random_bytes(&random_int
, sizeof (unsigned int));
374 run_args
->region_next
+=
375 random_int
% run_args
->region_noise
;
377 run_args
->region_next
++;
380 mutex_exit(&run_args
->lock_work
);
384 /* nothing left to do */
385 mutex_exit(&run_args
->lock_work
);
391 zpios_remove_objset(run_args_t
*run_args
)
393 zpios_time_t
*t
= &(run_args
->stats
.rm_time
);
394 zpios_region_t
*region
;
398 (void) zpios_upcall(run_args
->pre
, PHASE_PRE_REMOVE
, run_args
, 0);
399 t
->start
= zpios_timespec_now();
401 (void) snprintf(name
, 32, "%s/id_%d", run_args
->pool
, run_args
->id
);
403 if (run_args
->flags
& DMU_REMOVE
) {
404 if (run_args
->flags
& DMU_FPP
) {
405 for (i
= 0; i
< run_args
->region_count
; i
++) {
406 region
= &run_args
->regions
[i
];
407 rc
= zpios_dmu_object_free(run_args
,
408 region
->obj
.os
, region
->obj
.obj
);
410 zpios_print(run_args
->file
,
411 "Error removing object %d, %d\n",
412 (int)region
->obj
.obj
, rc
);
415 region
= &run_args
->regions
[0];
416 rc
= zpios_dmu_object_free(run_args
,
417 region
->obj
.os
, region
->obj
.obj
);
419 zpios_print(run_args
->file
,
420 "Error removing object %d, %d\n",
421 (int)region
->obj
.obj
, rc
);
425 dmu_objset_disown(run_args
->os
, zpios_tag
);
427 if (run_args
->flags
& DMU_REMOVE
) {
428 rc
= dsl_destroy_head(name
);
430 zpios_print(run_args
->file
, "Error dsl_destroy_head"
431 "(%s, ...) failed: %d\n", name
, rc
);
434 t
->stop
= zpios_timespec_now();
435 t
->delta
= zpios_timespec_sub(t
->stop
, t
->start
);
436 (void) zpios_upcall(run_args
->post
, PHASE_POST_REMOVE
, run_args
, rc
);
440 zpios_cleanup_run(run_args_t
*run_args
)
444 if (run_args
== NULL
)
447 if (run_args
->threads
!= NULL
) {
448 for (i
= 0; i
< run_args
->thread_count
; i
++) {
449 if (run_args
->threads
[i
]) {
450 mutex_destroy(&run_args
->threads
[i
]->lock
);
451 kmem_free(run_args
->threads
[i
],
452 sizeof (thread_data_t
));
456 kmem_free(run_args
->threads
,
457 sizeof (thread_data_t
*) * run_args
->thread_count
);
460 for (i
= 0; i
< run_args
->region_count
; i
++)
461 mutex_destroy(&run_args
->regions
[i
].lock
);
463 mutex_destroy(&run_args
->lock_work
);
464 mutex_destroy(&run_args
->lock_ctl
);
465 size
= run_args
->region_count
* sizeof (zpios_region_t
);
467 vmem_free(run_args
, sizeof (*run_args
) + size
);
471 zpios_dmu_write(run_args_t
*run_args
, objset_t
*os
, uint64_t object
,
472 uint64_t offset
, uint64_t size
, const void *buf
)
475 int rc
, how
= TXG_WAIT
;
478 if (run_args
->flags
& DMU_WRITE_NOWAIT
)
482 tx
= dmu_tx_create(os
);
483 dmu_tx_hold_write(tx
, object
, offset
, size
);
484 rc
= dmu_tx_assign(tx
, how
);
487 if (rc
== ERESTART
&& how
== TXG_NOWAIT
) {
492 zpios_print(run_args
->file
,
493 "Error in dmu_tx_assign(), %d", rc
);
500 // if (run_args->flags & DMU_WRITE_ZC)
501 // flags |= DMU_WRITE_ZEROCOPY;
503 dmu_write(os
, object
, offset
, size
, buf
, tx
);
510 zpios_dmu_read(run_args_t
*run_args
, objset_t
*os
, uint64_t object
,
511 uint64_t offset
, uint64_t size
, void *buf
)
515 // if (run_args->flags & DMU_READ_ZC)
516 // flags |= DMU_READ_ZEROCOPY;
518 if (run_args
->flags
& DMU_READ_NOPF
)
519 flags
|= DMU_READ_NO_PREFETCH
;
521 return (dmu_read(os
, object
, offset
, size
, buf
, flags
));
525 zpios_thread_main(void *data
)
527 thread_data_t
*thr
= (thread_data_t
*)data
;
528 run_args_t
*run_args
= thr
->run_args
;
533 zpios_region_t
*region
;
535 unsigned int random_int
;
536 int chunk_noise
= run_args
->chunk_noise
;
537 int chunk_noise_tmp
= 0;
538 int thread_delay
= run_args
->thread_delay
;
539 int thread_delay_tmp
= 0;
543 get_random_bytes(&random_int
, sizeof (unsigned int));
544 chunk_noise_tmp
= (random_int
% (chunk_noise
* 2))-chunk_noise
;
548 * It's OK to vmem_alloc() this memory because it will be copied
549 * in to the slab and pointers to the slab copy will be setup in
550 * the bio when the IO is submitted. This of course is not ideal
551 * since we want a zero-copy IO path if possible. It would be nice
552 * to have direct access to those slab entries.
554 chunk_size
= run_args
->chunk_size
+ chunk_noise_tmp
;
555 buf
= (char *)vmem_alloc(chunk_size
, KM_SLEEP
);
558 /* Trivial data verification pattern for now. */
559 if (run_args
->flags
& DMU_VERIFY
)
560 memset(buf
, 'z', chunk_size
);
563 mutex_enter(&thr
->lock
);
564 thr
->stats
.wr_time
.start
= zpios_timespec_now();
565 mutex_exit(&thr
->lock
);
567 while (zpios_get_work_item(run_args
, &obj
, &offset
,
568 &chunk_size
, ®ion
, DMU_WRITE
)) {
570 get_random_bytes(&random_int
, sizeof (unsigned int));
571 thread_delay_tmp
= random_int
% thread_delay
;
572 set_current_state(TASK_UNINTERRUPTIBLE
);
573 schedule_timeout(thread_delay_tmp
); /* In jiffies */
576 t
.start
= zpios_timespec_now();
577 rc
= zpios_dmu_write(run_args
, obj
.os
, obj
.obj
,
578 offset
, chunk_size
, buf
);
579 t
.stop
= zpios_timespec_now();
580 t
.delta
= zpios_timespec_sub(t
.stop
, t
.start
);
583 zpios_print(run_args
->file
, "IO error while doing "
584 "dmu_write(): %d\n", rc
);
588 mutex_enter(&thr
->lock
);
589 thr
->stats
.wr_data
+= chunk_size
;
590 thr
->stats
.wr_chunks
++;
591 thr
->stats
.wr_time
.delta
= zpios_timespec_add(
592 thr
->stats
.wr_time
.delta
, t
.delta
);
593 mutex_exit(&thr
->lock
);
595 mutex_enter(®ion
->lock
);
596 region
->stats
.wr_data
+= chunk_size
;
597 region
->stats
.wr_chunks
++;
598 region
->stats
.wr_time
.delta
= zpios_timespec_add(
599 region
->stats
.wr_time
.delta
, t
.delta
);
601 /* First time region was accessed */
602 if (region
->init_offset
== offset
)
603 region
->stats
.wr_time
.start
= t
.start
;
605 mutex_exit(®ion
->lock
);
608 mutex_enter(&run_args
->lock_ctl
);
609 run_args
->threads_done
++;
610 mutex_exit(&run_args
->lock_ctl
);
612 mutex_enter(&thr
->lock
);
614 thr
->stats
.wr_time
.stop
= zpios_timespec_now();
615 mutex_exit(&thr
->lock
);
616 wake_up(&run_args
->waitq
);
618 set_current_state(TASK_UNINTERRUPTIBLE
);
621 /* Check if we should exit */
622 mutex_enter(&thr
->lock
);
624 mutex_exit(&thr
->lock
);
629 mutex_enter(&thr
->lock
);
630 thr
->stats
.rd_time
.start
= zpios_timespec_now();
631 mutex_exit(&thr
->lock
);
633 while (zpios_get_work_item(run_args
, &obj
, &offset
,
634 &chunk_size
, ®ion
, DMU_READ
)) {
636 get_random_bytes(&random_int
, sizeof (unsigned int));
637 thread_delay_tmp
= random_int
% thread_delay
;
638 set_current_state(TASK_UNINTERRUPTIBLE
);
639 schedule_timeout(thread_delay_tmp
); /* In jiffies */
642 if (run_args
->flags
& DMU_VERIFY
)
643 memset(buf
, 0, chunk_size
);
645 t
.start
= zpios_timespec_now();
646 rc
= zpios_dmu_read(run_args
, obj
.os
, obj
.obj
,
647 offset
, chunk_size
, buf
);
648 t
.stop
= zpios_timespec_now();
649 t
.delta
= zpios_timespec_sub(t
.stop
, t
.start
);
652 zpios_print(run_args
->file
, "IO error while doing "
653 "dmu_read(): %d\n", rc
);
657 /* Trivial data verification, expensive! */
658 if (run_args
->flags
& DMU_VERIFY
) {
659 for (i
= 0; i
< chunk_size
; i
++) {
661 zpios_print(run_args
->file
,
662 "IO verify error: %d/%d/%d\n",
663 (int)obj
.obj
, (int)offset
,
670 mutex_enter(&thr
->lock
);
671 thr
->stats
.rd_data
+= chunk_size
;
672 thr
->stats
.rd_chunks
++;
673 thr
->stats
.rd_time
.delta
= zpios_timespec_add(
674 thr
->stats
.rd_time
.delta
, t
.delta
);
675 mutex_exit(&thr
->lock
);
677 mutex_enter(®ion
->lock
);
678 region
->stats
.rd_data
+= chunk_size
;
679 region
->stats
.rd_chunks
++;
680 region
->stats
.rd_time
.delta
= zpios_timespec_add(
681 region
->stats
.rd_time
.delta
, t
.delta
);
683 /* First time region was accessed */
684 if (region
->init_offset
== offset
)
685 region
->stats
.rd_time
.start
= t
.start
;
687 mutex_exit(®ion
->lock
);
690 mutex_enter(&run_args
->lock_ctl
);
691 run_args
->threads_done
++;
692 mutex_exit(&run_args
->lock_ctl
);
694 mutex_enter(&thr
->lock
);
696 thr
->stats
.rd_time
.stop
= zpios_timespec_now();
697 mutex_exit(&thr
->lock
);
698 wake_up(&run_args
->waitq
);
701 vmem_free(buf
, chunk_size
);
704 return (rc
); /* Unreachable, due to do_exit() */
708 zpios_thread_done(run_args_t
*run_args
)
710 ASSERT(run_args
->threads_done
<= run_args
->thread_count
);
711 return (run_args
->threads_done
== run_args
->thread_count
);
715 zpios_threads_run(run_args_t
*run_args
)
717 struct task_struct
*tsk
, **tsks
;
718 thread_data_t
*thr
= NULL
;
719 zpios_time_t
*tt
= &(run_args
->stats
.total_time
);
720 zpios_time_t
*tw
= &(run_args
->stats
.wr_time
);
721 zpios_time_t
*tr
= &(run_args
->stats
.rd_time
);
722 int i
, rc
= 0, tc
= run_args
->thread_count
;
724 tsks
= kmem_zalloc(sizeof (struct task_struct
*) * tc
, KM_SLEEP
);
730 run_args
->threads
= kmem_zalloc(sizeof (thread_data_t
*)*tc
, KM_SLEEP
);
731 if (run_args
->threads
== NULL
) {
736 init_waitqueue_head(&run_args
->waitq
);
737 run_args
->threads_done
= 0;
739 /* Create all the needed threads which will sleep until awoken */
740 for (i
= 0; i
< tc
; i
++) {
741 thr
= kmem_zalloc(sizeof (thread_data_t
), KM_SLEEP
);
748 thr
->run_args
= run_args
;
750 mutex_init(&thr
->lock
, NULL
, MUTEX_DEFAULT
, NULL
);
751 run_args
->threads
[i
] = thr
;
753 tsk
= kthread_create(zpios_thread_main
, (void *)thr
,
754 "%s/%d", "zpios_io", i
);
763 tt
->start
= zpios_timespec_now();
765 /* Wake up all threads for write phase */
766 (void) zpios_upcall(run_args
->pre
, PHASE_PRE_WRITE
, run_args
, 0);
767 for (i
= 0; i
< tc
; i
++)
768 wake_up_process(tsks
[i
]);
770 /* Wait for write phase to complete */
771 tw
->start
= zpios_timespec_now();
772 wait_event(run_args
->waitq
, zpios_thread_done(run_args
));
773 tw
->stop
= zpios_timespec_now();
774 (void) zpios_upcall(run_args
->post
, PHASE_POST_WRITE
, run_args
, rc
);
776 for (i
= 0; i
< tc
; i
++) {
777 thr
= run_args
->threads
[i
];
779 mutex_enter(&thr
->lock
);
784 run_args
->stats
.wr_data
+= thr
->stats
.wr_data
;
785 run_args
->stats
.wr_chunks
+= thr
->stats
.wr_chunks
;
786 mutex_exit(&thr
->lock
);
790 /* Wake up all threads and tell them to exit */
791 for (i
= 0; i
< tc
; i
++) {
792 mutex_enter(&thr
->lock
);
794 mutex_exit(&thr
->lock
);
796 wake_up_process(tsks
[i
]);
801 mutex_enter(&run_args
->lock_ctl
);
802 ASSERT(run_args
->threads_done
== run_args
->thread_count
);
803 run_args
->threads_done
= 0;
804 mutex_exit(&run_args
->lock_ctl
);
806 /* Wake up all threads for read phase */
807 (void) zpios_upcall(run_args
->pre
, PHASE_PRE_READ
, run_args
, 0);
808 for (i
= 0; i
< tc
; i
++)
809 wake_up_process(tsks
[i
]);
811 /* Wait for read phase to complete */
812 tr
->start
= zpios_timespec_now();
813 wait_event(run_args
->waitq
, zpios_thread_done(run_args
));
814 tr
->stop
= zpios_timespec_now();
815 (void) zpios_upcall(run_args
->post
, PHASE_POST_READ
, run_args
, rc
);
817 for (i
= 0; i
< tc
; i
++) {
818 thr
= run_args
->threads
[i
];
820 mutex_enter(&thr
->lock
);
825 run_args
->stats
.rd_data
+= thr
->stats
.rd_data
;
826 run_args
->stats
.rd_chunks
+= thr
->stats
.rd_chunks
;
827 mutex_exit(&thr
->lock
);
830 tt
->stop
= zpios_timespec_now();
831 tt
->delta
= zpios_timespec_sub(tt
->stop
, tt
->start
);
832 tw
->delta
= zpios_timespec_sub(tw
->stop
, tw
->start
);
833 tr
->delta
= zpios_timespec_sub(tr
->stop
, tr
->start
);
836 kmem_free(tsks
, sizeof (struct task_struct
*) * tc
);
838 /* Returns first encountered thread error (if any) */
842 /* Destroy all threads that were created successfully */
843 for (i
= 0; i
< tc
; i
++)
845 (void) kthread_stop(tsks
[i
]);
851 zpios_do_one_run(struct file
*file
, zpios_cmd_t
*kcmd
,
852 int data_size
, void *data
)
854 run_args_t
*run_args
= { 0 };
855 zpios_stats_t
*stats
= (zpios_stats_t
*)data
;
856 int i
, n
, m
, size
, rc
;
858 if ((!kcmd
->cmd_chunk_size
) || (!kcmd
->cmd_region_size
) ||
859 (!kcmd
->cmd_thread_count
) || (!kcmd
->cmd_region_count
)) {
860 zpios_print(file
, "Invalid chunk_size, region_size, "
861 "thread_count, or region_count, %d\n", -EINVAL
);
865 if (!(kcmd
->cmd_flags
& DMU_WRITE
) ||
866 !(kcmd
->cmd_flags
& DMU_READ
)) {
867 zpios_print(file
, "Invalid flags, minimally DMU_WRITE "
868 "and DMU_READ must be set, %d\n", -EINVAL
);
872 if ((kcmd
->cmd_flags
& (DMU_WRITE_ZC
| DMU_READ_ZC
)) &&
873 (kcmd
->cmd_flags
& DMU_VERIFY
)) {
874 zpios_print(file
, "Invalid flags, DMU_*_ZC incompatible "
875 "with DMU_VERIFY, used for performance analysis "
876 "only, %d\n", -EINVAL
);
881 * Opaque data on return contains structs of the following form:
883 * zpios_stat_t stats[];
884 * stats[0] = run_args->stats;
885 * stats[1-N] = threads[N]->stats;
886 * stats[N+1-M] = regions[M]->stats;
888 * Where N is the number of threads, and M is the number of regions.
890 size
= (sizeof (zpios_stats_t
) +
891 (kcmd
->cmd_thread_count
* sizeof (zpios_stats_t
)) +
892 (kcmd
->cmd_region_count
* sizeof (zpios_stats_t
)));
893 if (data_size
< size
) {
894 zpios_print(file
, "Invalid size, command data buffer "
895 "size too small, (%d < %d)\n", data_size
, size
);
899 rc
= zpios_setup_run(&run_args
, kcmd
, file
);
903 rc
= zpios_threads_run(run_args
);
904 zpios_remove_objset(run_args
);
910 m
= 1 + kcmd
->cmd_thread_count
;
911 stats
[0] = run_args
->stats
;
913 for (i
= 0; i
< kcmd
->cmd_thread_count
; i
++)
914 stats
[n
+i
] = run_args
->threads
[i
]->stats
;
916 for (i
= 0; i
< kcmd
->cmd_region_count
; i
++)
917 stats
[m
+i
] = run_args
->regions
[i
].stats
;
921 zpios_cleanup_run(run_args
);
923 (void) zpios_upcall(kcmd
->cmd_post
, PHASE_POST_RUN
, run_args
, 0);
929 zpios_open(struct inode
*inode
, struct file
*file
)
931 unsigned int minor
= iminor(inode
);
934 if (minor
>= ZPIOS_MINORS
)
937 info
= (zpios_info_t
*)kmem_alloc(sizeof (*info
), KM_SLEEP
);
941 spin_lock_init(&info
->info_lock
);
942 info
->info_size
= ZPIOS_INFO_BUFFER_SIZE
;
944 (char *) vmem_alloc(ZPIOS_INFO_BUFFER_SIZE
, KM_SLEEP
);
945 if (info
->info_buffer
== NULL
) {
946 kmem_free(info
, sizeof (*info
));
950 info
->info_head
= info
->info_buffer
;
951 file
->private_data
= (void *)info
;
957 zpios_release(struct inode
*inode
, struct file
*file
)
959 unsigned int minor
= iminor(inode
);
960 zpios_info_t
*info
= (zpios_info_t
*)file
->private_data
;
962 if (minor
>= ZPIOS_MINORS
)
966 ASSERT(info
->info_buffer
);
968 vmem_free(info
->info_buffer
, ZPIOS_INFO_BUFFER_SIZE
);
969 kmem_free(info
, sizeof (*info
));
975 zpios_buffer_clear(struct file
*file
, zpios_cfg_t
*kcfg
, unsigned long arg
)
977 zpios_info_t
*info
= (zpios_info_t
*)file
->private_data
;
980 ASSERT(info
->info_buffer
);
982 spin_lock(&info
->info_lock
);
983 memset(info
->info_buffer
, 0, info
->info_size
);
984 info
->info_head
= info
->info_buffer
;
985 spin_unlock(&info
->info_lock
);
991 zpios_buffer_size(struct file
*file
, zpios_cfg_t
*kcfg
, unsigned long arg
)
993 zpios_info_t
*info
= (zpios_info_t
*)file
->private_data
;
995 int min
, size
, rc
= 0;
998 ASSERT(info
->info_buffer
);
1000 spin_lock(&info
->info_lock
);
1001 if (kcfg
->cfg_arg1
> 0) {
1003 size
= kcfg
->cfg_arg1
;
1004 buf
= (char *)vmem_alloc(size
, KM_SLEEP
);
1010 /* Zero fill and truncate contents when coping buffer */
1011 min
= ((size
< info
->info_size
) ? size
: info
->info_size
);
1012 memset(buf
, 0, size
);
1013 memcpy(buf
, info
->info_buffer
, min
);
1014 vmem_free(info
->info_buffer
, info
->info_size
);
1015 info
->info_size
= size
;
1016 info
->info_buffer
= buf
;
1017 info
->info_head
= info
->info_buffer
;
1020 kcfg
->cfg_rc1
= info
->info_size
;
1022 if (copy_to_user((struct zpios_cfg_t __user
*)arg
,
1023 kcfg
, sizeof (*kcfg
)))
1026 spin_unlock(&info
->info_lock
);
1032 zpios_ioctl_cfg(struct file
*file
, unsigned long arg
)
1037 if (copy_from_user(&kcfg
, (zpios_cfg_t
*)arg
, sizeof (kcfg
)))
1040 if (kcfg
.cfg_magic
!= ZPIOS_CFG_MAGIC
) {
1041 zpios_print(file
, "Bad config magic 0x%x != 0x%x\n",
1042 kcfg
.cfg_magic
, ZPIOS_CFG_MAGIC
);
1046 switch (kcfg
.cfg_cmd
) {
1047 case ZPIOS_CFG_BUFFER_CLEAR
:
1052 rc
= zpios_buffer_clear(file
, &kcfg
, arg
);
1054 case ZPIOS_CFG_BUFFER_SIZE
:
1056 * cfg_arg1 - 0 - query size; >0 resize
1057 * cfg_rc1 - Set to current buffer size
1059 rc
= zpios_buffer_size(file
, &kcfg
, arg
);
1062 zpios_print(file
, "Bad config command %d\n",
1072 zpios_ioctl_cmd(struct file
*file
, unsigned long arg
)
1078 kcmd
= kmem_alloc(sizeof (zpios_cmd_t
), KM_SLEEP
);
1080 zpios_print(file
, "Unable to kmem_alloc() %ld byte for "
1081 "zpios_cmd_t\n", (long int)sizeof (zpios_cmd_t
));
1085 rc
= copy_from_user(kcmd
, (zpios_cfg_t
*)arg
, sizeof (zpios_cmd_t
));
1087 zpios_print(file
, "Unable to copy command structure "
1088 "from user to kernel memory, %d\n", rc
);
1092 if (kcmd
->cmd_magic
!= ZPIOS_CMD_MAGIC
) {
1093 zpios_print(file
, "Bad command magic 0x%x != 0x%x\n",
1094 kcmd
->cmd_magic
, ZPIOS_CFG_MAGIC
);
1099 /* Allocate memory for any opaque data the caller needed to pass on */
1100 if (kcmd
->cmd_data_size
> 0) {
1101 data
= (void *)vmem_alloc(kcmd
->cmd_data_size
, KM_SLEEP
);
1103 zpios_print(file
, "Unable to vmem_alloc() %ld "
1104 "bytes for data buffer\n",
1105 (long)kcmd
->cmd_data_size
);
1110 rc
= copy_from_user(data
, (void *)(arg
+ offsetof(zpios_cmd_t
,
1111 cmd_data_str
)), kcmd
->cmd_data_size
);
1113 zpios_print(file
, "Unable to copy data buffer "
1114 "from user to kernel memory, %d\n", rc
);
1119 rc
= zpios_do_one_run(file
, kcmd
, kcmd
->cmd_data_size
, data
);
1122 /* If the test failed do not print out the stats */
1126 rc
= copy_to_user((void *)(arg
+ offsetof(zpios_cmd_t
,
1127 cmd_data_str
)), data
, kcmd
->cmd_data_size
);
1129 zpios_print(file
, "Unable to copy data buffer "
1130 "from kernel to user memory, %d\n", rc
);
1135 vmem_free(data
, kcmd
->cmd_data_size
);
1138 kmem_free(kcmd
, sizeof (zpios_cmd_t
));
1144 zpios_unlocked_ioctl(struct file
*file
, unsigned int cmd
, unsigned long arg
)
1146 unsigned int minor
= iminor(file
->f_dentry
->d_inode
);
1149 /* Ignore tty ioctls */
1150 if ((cmd
& 0xffffff00) == ((int)'T') << 8)
1153 if (minor
>= ZPIOS_MINORS
)
1158 rc
= zpios_ioctl_cfg(file
, arg
);
1161 rc
= zpios_ioctl_cmd(file
, arg
);
1164 zpios_print(file
, "Bad ioctl command %d\n", cmd
);
1172 #ifdef CONFIG_COMPAT
1173 /* Compatibility handler for ioctls from 32-bit ELF binaries */
1175 zpios_compat_ioctl(struct file
*file
, unsigned int cmd
, unsigned long arg
)
1177 return (zpios_unlocked_ioctl(file
, cmd
, arg
));
1179 #endif /* CONFIG_COMPAT */
1182 * I'm not sure why you would want to write in to this buffer from
1183 * user space since its principle use is to pass test status info
1184 * back to the user space, but I don't see any reason to prevent it.
1187 zpios_write(struct file
*file
, const char __user
*buf
,
1188 size_t count
, loff_t
*ppos
)
1190 unsigned int minor
= iminor(file
->f_dentry
->d_inode
);
1191 zpios_info_t
*info
= (zpios_info_t
*)file
->private_data
;
1194 if (minor
>= ZPIOS_MINORS
)
1198 ASSERT(info
->info_buffer
);
1200 spin_lock(&info
->info_lock
);
1202 /* Write beyond EOF */
1203 if (*ppos
>= info
->info_size
) {
1208 /* Resize count if beyond EOF */
1209 if (*ppos
+ count
> info
->info_size
)
1210 count
= info
->info_size
- *ppos
;
1212 if (copy_from_user(info
->info_buffer
, buf
, count
)) {
1220 spin_unlock(&info
->info_lock
);
1225 zpios_read(struct file
*file
, char __user
*buf
, size_t count
, loff_t
*ppos
)
1227 unsigned int minor
= iminor(file
->f_dentry
->d_inode
);
1228 zpios_info_t
*info
= (zpios_info_t
*)file
->private_data
;
1231 if (minor
>= ZPIOS_MINORS
)
1235 ASSERT(info
->info_buffer
);
1237 spin_lock(&info
->info_lock
);
1239 /* Read beyond EOF */
1240 if (*ppos
>= info
->info_size
)
1243 /* Resize count if beyond EOF */
1244 if (*ppos
+ count
> info
->info_size
)
1245 count
= info
->info_size
- *ppos
;
1247 if (copy_to_user(buf
, info
->info_buffer
+ *ppos
, count
)) {
1255 spin_unlock(&info
->info_lock
);
1259 static loff_t
zpios_seek(struct file
*file
, loff_t offset
, int origin
)
1261 unsigned int minor
= iminor(file
->f_dentry
->d_inode
);
1262 zpios_info_t
*info
= (zpios_info_t
*)file
->private_data
;
1265 if (minor
>= ZPIOS_MINORS
)
1269 ASSERT(info
->info_buffer
);
1271 spin_lock(&info
->info_lock
);
1274 case 0: /* SEEK_SET - No-op just do it */
1276 case 1: /* SEEK_CUR - Seek from current */
1277 offset
= file
->f_pos
+ offset
;
1279 case 2: /* SEEK_END - Seek from end */
1280 offset
= info
->info_size
+ offset
;
1285 file
->f_pos
= offset
;
1286 file
->f_version
= 0;
1290 spin_unlock(&info
->info_lock
);
1295 static struct cdev zpios_cdev
;
1296 static struct file_operations zpios_fops
= {
1297 .owner
= THIS_MODULE
,
1299 .release
= zpios_release
,
1300 .unlocked_ioctl
= zpios_unlocked_ioctl
,
1301 #ifdef CONFIG_COMPAT
1302 .compat_ioctl
= zpios_compat_ioctl
,
1305 .write
= zpios_write
,
1306 .llseek
= zpios_seek
,
1315 dev
= MKDEV(ZPIOS_MAJOR
, 0);
1316 if ((rc
= register_chrdev_region(dev
, ZPIOS_MINORS
, ZPIOS_NAME
)))
1319 /* Support for registering a character driver */
1320 cdev_init(&zpios_cdev
, &zpios_fops
);
1321 zpios_cdev
.owner
= THIS_MODULE
;
1322 kobject_set_name(&zpios_cdev
.kobj
, ZPIOS_NAME
);
1323 if ((rc
= cdev_add(&zpios_cdev
, dev
, ZPIOS_MINORS
))) {
1324 printk(KERN_ERR
"ZPIOS: Error adding cdev, %d\n", rc
);
1325 kobject_put(&zpios_cdev
.kobj
);
1326 unregister_chrdev_region(dev
, ZPIOS_MINORS
);
1330 /* Support for udev make driver info available in sysfs */
1331 zpios_class
= spl_class_create(THIS_MODULE
, ZPIOS_NAME
);
1332 if (IS_ERR(zpios_class
)) {
1333 rc
= PTR_ERR(zpios_class
);
1334 printk(KERN_ERR
"ZPIOS: Error creating zpios class, %d\n", rc
);
1335 cdev_del(&zpios_cdev
);
1336 unregister_chrdev_region(dev
, ZPIOS_MINORS
);
1340 zpios_device
= spl_device_create(zpios_class
, NULL
,
1341 dev
, NULL
, ZPIOS_NAME
);
1345 printk(KERN_ERR
"ZPIOS: Error registering zpios device, %d\n", rc
);
1352 dev_t dev
= MKDEV(ZPIOS_MAJOR
, 0);
1354 spl_device_destroy(zpios_class
, zpios_device
, dev
);
1355 spl_class_destroy(zpios_class
);
1356 cdev_del(&zpios_cdev
);
1357 unregister_chrdev_region(dev
, ZPIOS_MINORS
);
1362 spl_module_init(zpios_init
);
1363 spl_module_exit(zpios_fini
);
1365 MODULE_AUTHOR("LLNL / Sun");
1366 MODULE_DESCRIPTION("Kernel PIOS implementation");
1367 MODULE_LICENSE("GPL");
1368 MODULE_VERSION(ZFS_META_VERSION
"-" ZFS_META_RELEASE
);