4 * Copyright (c) Intel Corporation.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of Intel Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include "spdk/stdinc.h"
36 #include "spdk/bdev.h"
37 #include "spdk/accel_engine.h"
38 #include "spdk/conf.h"
40 #include "spdk/thread.h"
42 #include "spdk/string.h"
43 #include "spdk/queue.h"
44 #include "spdk/util.h"
46 #include "spdk_internal/thread.h"
47 #include "spdk_internal/event.h"
49 #include "config-host.h"
53 /* FreeBSD is missing CLOCK_MONOTONIC_RAW,
54 * so alternative is provided. */
55 #ifndef CLOCK_MONOTONIC_RAW /* Defined in glibc bits/time.h */
56 #define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC
59 struct spdk_fio_options
{
67 struct spdk_fio_request
{
69 struct thread_data
*td
;
72 struct spdk_fio_target
{
73 struct spdk_bdev
*bdev
;
74 struct spdk_bdev_desc
*desc
;
75 struct spdk_io_channel
*ch
;
77 TAILQ_ENTRY(spdk_fio_target
) link
;
80 struct spdk_fio_thread
{
81 struct thread_data
*td
; /* fio thread context */
82 struct spdk_thread
*thread
; /* spdk thread context */
84 TAILQ_HEAD(, spdk_fio_target
) targets
;
85 bool failed
; /* true if the thread failed to initialize */
87 struct io_u
**iocq
; /* io completion queue */
88 unsigned int iocq_count
; /* number of iocq entries filled by last getevents */
89 unsigned int iocq_size
; /* number of iocq entries allocated */
92 static bool g_spdk_env_initialized
= false;
93 static const char *g_json_config_file
= NULL
;
95 static int spdk_fio_init(struct thread_data
*td
);
96 static void spdk_fio_cleanup(struct thread_data
*td
);
97 static size_t spdk_fio_poll_thread(struct spdk_fio_thread
*fio_thread
);
99 /* Default polling timeout (ns) */
100 #define SPDK_FIO_POLLING_TIMEOUT 1000000000ULL
103 spdk_fio_init_thread(struct thread_data
*td
)
105 struct spdk_fio_thread
*fio_thread
;
107 fio_thread
= calloc(1, sizeof(*fio_thread
));
109 SPDK_ERRLOG("failed to allocate thread local context\n");
114 td
->io_ops_data
= fio_thread
;
116 fio_thread
->thread
= spdk_thread_create("fio_thread", NULL
);
117 if (!fio_thread
->thread
) {
119 SPDK_ERRLOG("failed to allocate thread\n");
122 spdk_set_thread(fio_thread
->thread
);
124 fio_thread
->iocq_size
= td
->o
.iodepth
;
125 fio_thread
->iocq
= calloc(fio_thread
->iocq_size
, sizeof(struct io_u
*));
126 assert(fio_thread
->iocq
!= NULL
);
128 TAILQ_INIT(&fio_thread
->targets
);
134 spdk_fio_bdev_close_targets(void *arg
)
136 struct spdk_fio_thread
*fio_thread
= arg
;
137 struct spdk_fio_target
*target
, *tmp
;
139 TAILQ_FOREACH_SAFE(target
, &fio_thread
->targets
, link
, tmp
) {
140 TAILQ_REMOVE(&fio_thread
->targets
, target
, link
);
141 spdk_put_io_channel(target
->ch
);
142 spdk_bdev_close(target
->desc
);
148 spdk_fio_cleanup_thread(struct spdk_fio_thread
*fio_thread
)
150 spdk_thread_send_msg(fio_thread
->thread
, spdk_fio_bdev_close_targets
, fio_thread
);
152 while (!spdk_thread_is_idle(fio_thread
->thread
)) {
153 spdk_fio_poll_thread(fio_thread
);
156 spdk_set_thread(fio_thread
->thread
);
158 spdk_thread_exit(fio_thread
->thread
);
159 while (!spdk_thread_is_exited(fio_thread
->thread
)) {
160 spdk_thread_poll(fio_thread
->thread
, 0, 0);
162 spdk_thread_destroy(fio_thread
->thread
);
163 free(fio_thread
->iocq
);
168 spdk_fio_calc_timeout(struct spdk_fio_thread
*fio_thread
, struct timespec
*ts
)
170 uint64_t timeout
, now
;
172 if (spdk_thread_has_active_pollers(fio_thread
->thread
)) {
176 timeout
= spdk_thread_next_poller_expiration(fio_thread
->thread
);
177 now
= spdk_get_ticks();
180 timeout
= now
+ (SPDK_FIO_POLLING_TIMEOUT
* spdk_get_ticks_hz()) / SPDK_SEC_TO_NSEC
;
184 timeout
= ((timeout
- now
) * SPDK_SEC_TO_NSEC
) / spdk_get_ticks_hz() +
185 ts
->tv_sec
* SPDK_SEC_TO_NSEC
+ ts
->tv_nsec
;
187 ts
->tv_sec
= timeout
/ SPDK_SEC_TO_NSEC
;
188 ts
->tv_nsec
= timeout
% SPDK_SEC_TO_NSEC
;
192 static pthread_t g_init_thread_id
= 0;
193 static pthread_mutex_t g_init_mtx
= PTHREAD_MUTEX_INITIALIZER
;
194 static pthread_cond_t g_init_cond
;
195 static bool g_poll_loop
= true;
198 spdk_fio_bdev_init_done(int rc
, void *cb_arg
)
200 *(bool *)cb_arg
= true;
204 spdk_fio_bdev_init_start(void *arg
)
208 if (g_json_config_file
!= NULL
) {
209 spdk_app_json_config_load(g_json_config_file
, SPDK_DEFAULT_RPC_ADDR
,
210 spdk_fio_bdev_init_done
, done
, true);
212 spdk_subsystem_init(spdk_fio_bdev_init_done
, done
);
217 spdk_fio_bdev_fini_done(void *cb_arg
)
219 *(bool *)cb_arg
= true;
223 spdk_fio_bdev_fini_start(void *arg
)
227 spdk_subsystem_fini(spdk_fio_bdev_fini_done
, done
);
231 spdk_init_thread_poll(void *arg
)
233 struct spdk_fio_options
*eo
= arg
;
234 struct spdk_fio_thread
*fio_thread
;
235 struct spdk_conf
*config
= NULL
;
236 struct spdk_env_opts opts
;
240 struct thread_data td
= {};
242 /* Create a dummy thread data for use on the initialization thread. */
246 /* Parse the SPDK configuration file */
249 if (eo
->conf
&& eo
->json_conf
) {
250 SPDK_ERRLOG("Cannot provide two types of configuration files\n");
253 } else if (eo
->conf
&& strlen(eo
->conf
)) {
254 config
= spdk_conf_allocate();
256 SPDK_ERRLOG("Unable to allocate configuration file\n");
261 rc
= spdk_conf_read(config
, eo
->conf
);
263 SPDK_ERRLOG("Invalid configuration file format\n");
264 spdk_conf_free(config
);
267 if (spdk_conf_first_section(config
) == NULL
) {
268 SPDK_ERRLOG("Invalid configuration file format\n");
269 spdk_conf_free(config
);
273 spdk_conf_set_as_default(config
);
274 } else if (eo
->json_conf
&& strlen(eo
->json_conf
)) {
275 g_json_config_file
= eo
->json_conf
;
277 SPDK_ERRLOG("No configuration file provided\n");
282 /* Initialize the environment library */
283 spdk_env_opts_init(&opts
);
287 opts
.mem_size
= eo
->mem_mb
;
289 opts
.hugepage_single_segments
= eo
->mem_single_seg
;
291 if (spdk_env_init(&opts
) < 0) {
292 SPDK_ERRLOG("Unable to initialize SPDK env\n");
293 spdk_conf_free(config
);
297 spdk_unaffinitize_thread();
299 spdk_thread_lib_init(NULL
, 0);
301 /* Create an SPDK thread temporarily */
302 rc
= spdk_fio_init_thread(&td
);
304 SPDK_ERRLOG("Failed to create initialization thread\n");
308 fio_thread
= td
.io_ops_data
;
310 /* Initialize the bdev layer */
312 spdk_thread_send_msg(fio_thread
->thread
, spdk_fio_bdev_init_start
, &done
);
315 spdk_fio_poll_thread(fio_thread
);
319 * Continue polling until there are no more events.
320 * This handles any final events posted by pollers.
322 while (spdk_fio_poll_thread(fio_thread
) > 0) {};
324 /* Set condition variable */
325 pthread_mutex_lock(&g_init_mtx
);
326 pthread_cond_signal(&g_init_cond
);
328 while (g_poll_loop
) {
329 spdk_fio_poll_thread(fio_thread
);
331 clock_gettime(CLOCK_MONOTONIC
, &ts
);
332 spdk_fio_calc_timeout(fio_thread
, &ts
);
334 rc
= pthread_cond_timedwait(&g_init_cond
, &g_init_mtx
, &ts
);
335 if (rc
!= ETIMEDOUT
) {
340 pthread_mutex_unlock(&g_init_mtx
);
342 /* Finalize the bdev layer */
344 spdk_thread_send_msg(fio_thread
->thread
, spdk_fio_bdev_fini_start
, &done
);
347 spdk_fio_poll_thread(fio_thread
);
348 } while (!done
&& !spdk_thread_is_idle(fio_thread
->thread
));
350 spdk_fio_cleanup_thread(fio_thread
);
360 spdk_fio_init_env(struct thread_data
*td
)
362 pthread_condattr_t attr
;
365 if (pthread_condattr_init(&attr
)) {
366 SPDK_ERRLOG("Unable to initialize condition variable\n");
370 if (pthread_condattr_setclock(&attr
, CLOCK_MONOTONIC
)) {
371 SPDK_ERRLOG("Unable to initialize condition variable\n");
375 if (pthread_cond_init(&g_init_cond
, &attr
)) {
376 SPDK_ERRLOG("Unable to initialize condition variable\n");
381 * Spawn a thread to handle initialization operations and to poll things
382 * like the admin queues periodically.
384 rc
= pthread_create(&g_init_thread_id
, NULL
, &spdk_init_thread_poll
, td
->eo
);
386 SPDK_ERRLOG("Unable to spawn thread to poll admin queue. It won't be polled.\n");
389 /* Wait for background thread to advance past the initialization */
390 pthread_mutex_lock(&g_init_mtx
);
391 pthread_cond_wait(&g_init_cond
, &g_init_mtx
);
392 pthread_mutex_unlock(&g_init_mtx
);
394 pthread_condattr_destroy(&attr
);
398 /* Called for each thread to fill in the 'real_file_size' member for
399 * each file associated with this thread. This is called prior to
400 * the init operation (spdk_fio_init()) below. This call will occur
401 * on the initial start up thread if 'create_serialize' is true, or
402 * on the thread actually associated with 'thread_data' if 'create_serialize'
406 spdk_fio_setup(struct thread_data
*td
)
411 /* we might be running in a daemonized FIO instance where standard
412 * input and output were closed and fds 0, 1, and 2 are reused
413 * for something important by FIO. We can't ensure we won't print
414 * anything (and so will our dependencies, e.g. DPDK), so abort early.
415 * (is_backend is an fio global variable)
419 snprintf(buf
, sizeof(buf
),
420 "SPDK FIO plugin won't work with daemonized FIO server.");
421 fio_server_text_output(FIO_LOG_ERR
, buf
, sizeof(buf
));
425 if (!td
->o
.use_thread
) {
426 SPDK_ERRLOG("must set thread=1 when using spdk plugin\n");
430 if (!g_spdk_env_initialized
) {
431 if (spdk_fio_init_env(td
)) {
432 SPDK_ERRLOG("failed to initialize\n");
436 g_spdk_env_initialized
= true;
439 if (td
->o
.nr_files
== 1 && strcmp(td
->files
[0]->file_name
, "*") == 0) {
440 struct spdk_bdev
*bdev
;
442 /* add all available bdevs as fio targets */
443 for (bdev
= spdk_bdev_first_leaf(); bdev
; bdev
= spdk_bdev_next_leaf(bdev
)) {
444 add_file(td
, spdk_bdev_get_name(bdev
), 0, 1);
448 for_each_file(td
, f
, i
) {
449 struct spdk_bdev
*bdev
;
451 if (strcmp(f
->file_name
, "*") == 0) {
455 bdev
= spdk_bdev_get_by_name(f
->file_name
);
457 SPDK_ERRLOG("Unable to find bdev with name %s\n", f
->file_name
);
461 f
->real_file_size
= spdk_bdev_get_num_blocks(bdev
) *
462 spdk_bdev_get_block_size(bdev
);
470 spdk_fio_bdev_open(void *arg
)
472 struct thread_data
*td
= arg
;
473 struct spdk_fio_thread
*fio_thread
;
478 fio_thread
= td
->io_ops_data
;
480 for_each_file(td
, f
, i
) {
481 struct spdk_fio_target
*target
;
483 if (strcmp(f
->file_name
, "*") == 0) {
487 target
= calloc(1, sizeof(*target
));
489 SPDK_ERRLOG("Unable to allocate memory for I/O target.\n");
490 fio_thread
->failed
= true;
494 target
->bdev
= spdk_bdev_get_by_name(f
->file_name
);
496 SPDK_ERRLOG("Unable to find bdev with name %s\n", f
->file_name
);
498 fio_thread
->failed
= true;
502 rc
= spdk_bdev_open(target
->bdev
, true, NULL
, NULL
, &target
->desc
);
504 SPDK_ERRLOG("Unable to open bdev %s\n", f
->file_name
);
506 fio_thread
->failed
= true;
510 target
->ch
= spdk_bdev_get_io_channel(target
->desc
);
512 SPDK_ERRLOG("Unable to get I/O channel for bdev.\n");
513 spdk_bdev_close(target
->desc
);
515 fio_thread
->failed
= true;
519 f
->engine_data
= target
;
521 TAILQ_INSERT_TAIL(&fio_thread
->targets
, target
, link
);
525 /* Called for each thread, on that thread, shortly after the thread
529 spdk_fio_init(struct thread_data
*td
)
531 struct spdk_fio_thread
*fio_thread
;
533 spdk_fio_init_thread(td
);
535 fio_thread
= td
->io_ops_data
;
536 fio_thread
->failed
= false;
538 spdk_thread_send_msg(fio_thread
->thread
, spdk_fio_bdev_open
, td
);
540 while (spdk_fio_poll_thread(fio_thread
) > 0) {}
542 if (fio_thread
->failed
) {
550 spdk_fio_cleanup(struct thread_data
*td
)
552 struct spdk_fio_thread
*fio_thread
= td
->io_ops_data
;
554 spdk_fio_cleanup_thread(fio_thread
);
555 td
->io_ops_data
= NULL
;
559 spdk_fio_open(struct thread_data
*td
, struct fio_file
*f
)
566 spdk_fio_close(struct thread_data
*td
, struct fio_file
*f
)
572 spdk_fio_iomem_alloc(struct thread_data
*td
, size_t total_mem
)
574 td
->orig_buffer
= spdk_dma_zmalloc(total_mem
, 0x1000, NULL
);
575 return td
->orig_buffer
== NULL
;
579 spdk_fio_iomem_free(struct thread_data
*td
)
581 spdk_dma_free(td
->orig_buffer
);
585 spdk_fio_io_u_init(struct thread_data
*td
, struct io_u
*io_u
)
587 struct spdk_fio_request
*fio_req
;
589 io_u
->engine_data
= NULL
;
591 fio_req
= calloc(1, sizeof(*fio_req
));
592 if (fio_req
== NULL
) {
598 io_u
->engine_data
= fio_req
;
604 spdk_fio_io_u_free(struct thread_data
*td
, struct io_u
*io_u
)
606 struct spdk_fio_request
*fio_req
= io_u
->engine_data
;
609 assert(fio_req
->io
== io_u
);
611 io_u
->engine_data
= NULL
;
616 spdk_fio_completion_cb(struct spdk_bdev_io
*bdev_io
,
620 struct spdk_fio_request
*fio_req
= cb_arg
;
621 struct thread_data
*td
= fio_req
->td
;
622 struct spdk_fio_thread
*fio_thread
= td
->io_ops_data
;
624 assert(fio_thread
->iocq_count
< fio_thread
->iocq_size
);
625 fio_req
->io
->error
= success
? 0 : EIO
;
626 fio_thread
->iocq
[fio_thread
->iocq_count
++] = fio_req
->io
;
628 spdk_bdev_free_io(bdev_io
);
631 #if FIO_IOOPS_VERSION >= 24
632 typedef enum fio_q_status fio_q_status_t
;
634 typedef int fio_q_status_t
;
637 static fio_q_status_t
638 spdk_fio_queue(struct thread_data
*td
, struct io_u
*io_u
)
641 struct spdk_fio_request
*fio_req
= io_u
->engine_data
;
642 struct spdk_fio_target
*target
= io_u
->file
->engine_data
;
644 assert(fio_req
->td
== td
);
647 SPDK_ERRLOG("Unable to look up correct I/O target.\n");
648 fio_req
->io
->error
= ENODEV
;
649 return FIO_Q_COMPLETED
;
652 switch (io_u
->ddir
) {
654 rc
= spdk_bdev_read(target
->desc
, target
->ch
,
655 io_u
->buf
, io_u
->offset
, io_u
->xfer_buflen
,
656 spdk_fio_completion_cb
, fio_req
);
659 rc
= spdk_bdev_write(target
->desc
, target
->ch
,
660 io_u
->buf
, io_u
->offset
, io_u
->xfer_buflen
,
661 spdk_fio_completion_cb
, fio_req
);
664 rc
= spdk_bdev_unmap(target
->desc
, target
->ch
,
665 io_u
->offset
, io_u
->xfer_buflen
,
666 spdk_fio_completion_cb
, fio_req
);
678 fio_req
->io
->error
= abs(rc
);
679 return FIO_Q_COMPLETED
;
686 spdk_fio_event(struct thread_data
*td
, int event
)
688 struct spdk_fio_thread
*fio_thread
= td
->io_ops_data
;
691 assert((unsigned)event
< fio_thread
->iocq_count
);
692 return fio_thread
->iocq
[event
];
696 spdk_fio_poll_thread(struct spdk_fio_thread
*fio_thread
)
698 return spdk_thread_poll(fio_thread
->thread
, 0, 0);
702 spdk_fio_getevents(struct thread_data
*td
, unsigned int min
,
703 unsigned int max
, const struct timespec
*t
)
705 struct spdk_fio_thread
*fio_thread
= td
->io_ops_data
;
706 struct timespec t0
, t1
;
707 uint64_t timeout
= 0;
710 timeout
= t
->tv_sec
* SPDK_SEC_TO_NSEC
+ t
->tv_nsec
;
711 clock_gettime(CLOCK_MONOTONIC_RAW
, &t0
);
714 fio_thread
->iocq_count
= 0;
717 spdk_fio_poll_thread(fio_thread
);
719 if (fio_thread
->iocq_count
>= min
) {
720 return fio_thread
->iocq_count
;
724 clock_gettime(CLOCK_MONOTONIC_RAW
, &t1
);
725 uint64_t elapse
= ((t1
.tv_sec
- t0
.tv_sec
) * SPDK_SEC_TO_NSEC
)
726 + t1
.tv_nsec
- t0
.tv_nsec
;
727 if (elapse
> timeout
) {
733 return fio_thread
->iocq_count
;
737 spdk_fio_invalidate(struct thread_data
*td
, struct fio_file
*f
)
739 /* TODO: This should probably send a flush to the device, but for now just return successful. */
743 static struct fio_option options
[] = {
746 .lname
= "SPDK configuration file",
747 .type
= FIO_OPT_STR_STORE
,
748 .off1
= offsetof(struct spdk_fio_options
, conf
),
749 .help
= "A SPDK configuration file",
750 .category
= FIO_OPT_C_ENGINE
,
751 .group
= FIO_OPT_G_INVALID
,
754 .name
= "spdk_json_conf",
755 .lname
= "SPDK JSON configuration file",
756 .type
= FIO_OPT_STR_STORE
,
757 .off1
= offsetof(struct spdk_fio_options
, json_conf
),
758 .help
= "A SPDK JSON configuration file",
759 .category
= FIO_OPT_C_ENGINE
,
760 .group
= FIO_OPT_G_INVALID
,
764 .lname
= "SPDK memory in MB",
766 .off1
= offsetof(struct spdk_fio_options
, mem_mb
),
767 .help
= "Amount of memory in MB to allocate for SPDK",
768 .category
= FIO_OPT_C_ENGINE
,
769 .group
= FIO_OPT_G_INVALID
,
772 .name
= "spdk_single_seg",
773 .lname
= "SPDK switch to create just a single hugetlbfs file",
774 .type
= FIO_OPT_BOOL
,
775 .off1
= offsetof(struct spdk_fio_options
, mem_single_seg
),
776 .help
= "If set to 1, SPDK will use just a single hugetlbfs file",
777 .category
= FIO_OPT_C_ENGINE
,
778 .group
= FIO_OPT_G_INVALID
,
785 /* FIO imports this structure using dlsym */
786 struct ioengine_ops ioengine
= {
788 .version
= FIO_IOOPS_VERSION
,
789 .flags
= FIO_RAWIO
| FIO_NOEXTEND
| FIO_NODISKUTIL
| FIO_MEMALIGN
,
790 .setup
= spdk_fio_setup
,
791 .init
= spdk_fio_init
,
792 /* .prep = unused, */
793 .queue
= spdk_fio_queue
,
794 /* .commit = unused, */
795 .getevents
= spdk_fio_getevents
,
796 .event
= spdk_fio_event
,
797 /* .errdetails = unused, */
798 /* .cancel = unused, */
799 .cleanup
= spdk_fio_cleanup
,
800 .open_file
= spdk_fio_open
,
801 .close_file
= spdk_fio_close
,
802 .invalidate
= spdk_fio_invalidate
,
803 /* .unlink_file = unused, */
804 /* .get_file_size = unused, */
805 /* .terminate = unused, */
806 .iomem_alloc
= spdk_fio_iomem_alloc
,
807 .iomem_free
= spdk_fio_iomem_free
,
808 .io_u_init
= spdk_fio_io_u_init
,
809 .io_u_free
= spdk_fio_io_u_free
,
810 .option_struct_size
= sizeof(struct spdk_fio_options
),
814 static void fio_init
spdk_fio_register(void)
816 register_ioengine(&ioengine
);
820 spdk_fio_finish_env(void)
822 pthread_mutex_lock(&g_init_mtx
);
824 pthread_cond_signal(&g_init_cond
);
825 pthread_mutex_unlock(&g_init_mtx
);
826 pthread_join(g_init_thread_id
, NULL
);
828 spdk_thread_lib_fini();
831 static void fio_exit
spdk_fio_unregister(void)
833 if (g_spdk_env_initialized
) {
834 spdk_fio_finish_env();
835 g_spdk_env_initialized
= false;
837 unregister_ioengine(&ioengine
);