]>
Commit | Line | Data |
---|---|---|
11fdf7f2 TL |
1 | /*- |
2 | * BSD LICENSE | |
3 | * | |
4 | * Copyright (c) Intel Corporation. | |
5 | * All rights reserved. | |
6 | * | |
7 | * Redistribution and use in source and binary forms, with or without | |
8 | * modification, are permitted provided that the following conditions | |
9 | * are met: | |
10 | * | |
11 | * * Redistributions of source code must retain the above copyright | |
12 | * notice, this list of conditions and the following disclaimer. | |
13 | * * Redistributions in binary form must reproduce the above copyright | |
14 | * notice, this list of conditions and the following disclaimer in | |
15 | * the documentation and/or other materials provided with the | |
16 | * distribution. | |
17 | * * Neither the name of Intel Corporation nor the names of its | |
18 | * contributors may be used to endorse or promote products derived | |
19 | * from this software without specific prior written permission. | |
20 | * | |
21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
32 | */ | |
33 | ||
34 | #include "spdk/stdinc.h" | |
35 | ||
36 | #include "spdk/bdev.h" | |
f67539c2 | 37 | #include "spdk/accel_engine.h" |
11fdf7f2 TL |
38 | #include "spdk/conf.h" |
39 | #include "spdk/env.h" | |
40 | #include "spdk/thread.h" | |
41 | #include "spdk/log.h" | |
42 | #include "spdk/string.h" | |
43 | #include "spdk/queue.h" | |
9f95a23c TL |
44 | #include "spdk/util.h" |
45 | ||
46 | #include "spdk_internal/thread.h" | |
f67539c2 | 47 | #include "spdk_internal/event.h" |
11fdf7f2 TL |
48 | |
49 | #include "config-host.h" | |
50 | #include "fio.h" | |
51 | #include "optgroup.h" | |
52 | ||
f67539c2 TL |
53 | /* FreeBSD is missing CLOCK_MONOTONIC_RAW, |
54 | * so alternative is provided. */ | |
55 | #ifndef CLOCK_MONOTONIC_RAW /* Defined in glibc bits/time.h */ | |
56 | #define CLOCK_MONOTONIC_RAW CLOCK_MONOTONIC | |
57 | #endif | |
58 | ||
11fdf7f2 TL |
59 | struct spdk_fio_options { |
60 | void *pad; | |
61 | char *conf; | |
f67539c2 | 62 | char *json_conf; |
11fdf7f2 TL |
63 | unsigned mem_mb; |
64 | bool mem_single_seg; | |
65 | }; | |
66 | ||
11fdf7f2 TL |
67 | struct spdk_fio_request { |
68 | struct io_u *io; | |
69 | struct thread_data *td; | |
70 | }; | |
71 | ||
72 | struct spdk_fio_target { | |
73 | struct spdk_bdev *bdev; | |
74 | struct spdk_bdev_desc *desc; | |
75 | struct spdk_io_channel *ch; | |
76 | ||
77 | TAILQ_ENTRY(spdk_fio_target) link; | |
78 | }; | |
79 | ||
80 | struct spdk_fio_thread { | |
81 | struct thread_data *td; /* fio thread context */ | |
82 | struct spdk_thread *thread; /* spdk thread context */ | |
11fdf7f2 TL |
83 | |
84 | TAILQ_HEAD(, spdk_fio_target) targets; | |
9f95a23c | 85 | bool failed; /* true if the thread failed to initialize */ |
11fdf7f2 | 86 | |
9f95a23c TL |
87 | struct io_u **iocq; /* io completion queue */ |
88 | unsigned int iocq_count; /* number of iocq entries filled by last getevents */ | |
89 | unsigned int iocq_size; /* number of iocq entries allocated */ | |
11fdf7f2 TL |
90 | }; |
91 | ||
11fdf7f2 | 92 | static bool g_spdk_env_initialized = false; |
f67539c2 | 93 | static const char *g_json_config_file = NULL; |
11fdf7f2 TL |
94 | |
95 | static int spdk_fio_init(struct thread_data *td); | |
96 | static void spdk_fio_cleanup(struct thread_data *td); | |
97 | static size_t spdk_fio_poll_thread(struct spdk_fio_thread *fio_thread); | |
98 | ||
9f95a23c TL |
99 | /* Default polling timeout (ns) */ |
100 | #define SPDK_FIO_POLLING_TIMEOUT 1000000000ULL | |
101 | ||
102 | static int | |
103 | spdk_fio_init_thread(struct thread_data *td) | |
11fdf7f2 | 104 | { |
9f95a23c | 105 | struct spdk_fio_thread *fio_thread; |
11fdf7f2 | 106 | |
9f95a23c TL |
107 | fio_thread = calloc(1, sizeof(*fio_thread)); |
108 | if (!fio_thread) { | |
109 | SPDK_ERRLOG("failed to allocate thread local context\n"); | |
110 | return -1; | |
111 | } | |
11fdf7f2 | 112 | |
9f95a23c TL |
113 | fio_thread->td = td; |
114 | td->io_ops_data = fio_thread; | |
11fdf7f2 | 115 | |
9f95a23c TL |
116 | fio_thread->thread = spdk_thread_create("fio_thread", NULL); |
117 | if (!fio_thread->thread) { | |
118 | free(fio_thread); | |
119 | SPDK_ERRLOG("failed to allocate thread\n"); | |
120 | return -1; | |
11fdf7f2 | 121 | } |
9f95a23c | 122 | spdk_set_thread(fio_thread->thread); |
11fdf7f2 | 123 | |
9f95a23c TL |
124 | fio_thread->iocq_size = td->o.iodepth; |
125 | fio_thread->iocq = calloc(fio_thread->iocq_size, sizeof(struct io_u *)); | |
126 | assert(fio_thread->iocq != NULL); | |
127 | ||
128 | TAILQ_INIT(&fio_thread->targets); | |
129 | ||
130 | return 0; | |
11fdf7f2 TL |
131 | } |
132 | ||
9f95a23c TL |
133 | static void |
134 | spdk_fio_bdev_close_targets(void *arg) | |
11fdf7f2 | 135 | { |
9f95a23c TL |
136 | struct spdk_fio_thread *fio_thread = arg; |
137 | struct spdk_fio_target *target, *tmp; | |
11fdf7f2 | 138 | |
9f95a23c TL |
139 | TAILQ_FOREACH_SAFE(target, &fio_thread->targets, link, tmp) { |
140 | TAILQ_REMOVE(&fio_thread->targets, target, link); | |
141 | spdk_put_io_channel(target->ch); | |
142 | spdk_bdev_close(target->desc); | |
143 | free(target); | |
11fdf7f2 | 144 | } |
11fdf7f2 TL |
145 | } |
146 | ||
147 | static void | |
9f95a23c | 148 | spdk_fio_cleanup_thread(struct spdk_fio_thread *fio_thread) |
11fdf7f2 | 149 | { |
9f95a23c | 150 | spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_close_targets, fio_thread); |
11fdf7f2 | 151 | |
9f95a23c TL |
152 | while (!spdk_thread_is_idle(fio_thread->thread)) { |
153 | spdk_fio_poll_thread(fio_thread); | |
154 | } | |
11fdf7f2 | 155 | |
9f95a23c | 156 | spdk_set_thread(fio_thread->thread); |
11fdf7f2 | 157 | |
9f95a23c | 158 | spdk_thread_exit(fio_thread->thread); |
f67539c2 TL |
159 | while (!spdk_thread_is_exited(fio_thread->thread)) { |
160 | spdk_thread_poll(fio_thread->thread, 0, 0); | |
161 | } | |
9f95a23c TL |
162 | spdk_thread_destroy(fio_thread->thread); |
163 | free(fio_thread->iocq); | |
164 | free(fio_thread); | |
11fdf7f2 TL |
165 | } |
166 | ||
9f95a23c TL |
167 | static void |
168 | spdk_fio_calc_timeout(struct spdk_fio_thread *fio_thread, struct timespec *ts) | |
11fdf7f2 | 169 | { |
9f95a23c | 170 | uint64_t timeout, now; |
11fdf7f2 | 171 | |
9f95a23c TL |
172 | if (spdk_thread_has_active_pollers(fio_thread->thread)) { |
173 | return; | |
11fdf7f2 TL |
174 | } |
175 | ||
9f95a23c TL |
176 | timeout = spdk_thread_next_poller_expiration(fio_thread->thread); |
177 | now = spdk_get_ticks(); | |
11fdf7f2 | 178 | |
9f95a23c TL |
179 | if (timeout == 0) { |
180 | timeout = now + (SPDK_FIO_POLLING_TIMEOUT * spdk_get_ticks_hz()) / SPDK_SEC_TO_NSEC; | |
11fdf7f2 TL |
181 | } |
182 | ||
9f95a23c TL |
183 | if (timeout > now) { |
184 | timeout = ((timeout - now) * SPDK_SEC_TO_NSEC) / spdk_get_ticks_hz() + | |
185 | ts->tv_sec * SPDK_SEC_TO_NSEC + ts->tv_nsec; | |
186 | ||
187 | ts->tv_sec = timeout / SPDK_SEC_TO_NSEC; | |
188 | ts->tv_nsec = timeout % SPDK_SEC_TO_NSEC; | |
11fdf7f2 | 189 | } |
9f95a23c | 190 | } |
11fdf7f2 | 191 | |
9f95a23c TL |
192 | static pthread_t g_init_thread_id = 0; |
193 | static pthread_mutex_t g_init_mtx = PTHREAD_MUTEX_INITIALIZER; | |
194 | static pthread_cond_t g_init_cond; | |
195 | static bool g_poll_loop = true; | |
11fdf7f2 | 196 | |
9f95a23c | 197 | static void |
f67539c2 | 198 | spdk_fio_bdev_init_done(int rc, void *cb_arg) |
9f95a23c TL |
199 | { |
200 | *(bool *)cb_arg = true; | |
201 | } | |
11fdf7f2 | 202 | |
9f95a23c TL |
203 | static void |
204 | spdk_fio_bdev_init_start(void *arg) | |
205 | { | |
206 | bool *done = arg; | |
11fdf7f2 | 207 | |
f67539c2 TL |
208 | if (g_json_config_file != NULL) { |
209 | spdk_app_json_config_load(g_json_config_file, SPDK_DEFAULT_RPC_ADDR, | |
210 | spdk_fio_bdev_init_done, done, true); | |
211 | } else { | |
212 | spdk_subsystem_init(spdk_fio_bdev_init_done, done); | |
213 | } | |
11fdf7f2 TL |
214 | } |
215 | ||
9f95a23c TL |
216 | static void |
217 | spdk_fio_bdev_fini_done(void *cb_arg) | |
11fdf7f2 | 218 | { |
9f95a23c TL |
219 | *(bool *)cb_arg = true; |
220 | } | |
11fdf7f2 | 221 | |
9f95a23c TL |
222 | static void |
223 | spdk_fio_bdev_fini_start(void *arg) | |
224 | { | |
225 | bool *done = arg; | |
11fdf7f2 | 226 | |
f67539c2 | 227 | spdk_subsystem_fini(spdk_fio_bdev_fini_done, done); |
11fdf7f2 TL |
228 | } |
229 | ||
9f95a23c TL |
230 | static void * |
231 | spdk_init_thread_poll(void *arg) | |
11fdf7f2 | 232 | { |
9f95a23c | 233 | struct spdk_fio_options *eo = arg; |
11fdf7f2 | 234 | struct spdk_fio_thread *fio_thread; |
f67539c2 | 235 | struct spdk_conf *config = NULL; |
11fdf7f2 | 236 | struct spdk_env_opts opts; |
9f95a23c TL |
237 | bool done; |
238 | int rc; | |
239 | struct timespec ts; | |
240 | struct thread_data td = {}; | |
241 | ||
242 | /* Create a dummy thread data for use on the initialization thread. */ | |
243 | td.o.iodepth = 32; | |
244 | td.eo = eo; | |
11fdf7f2 TL |
245 | |
246 | /* Parse the SPDK configuration file */ | |
9f95a23c | 247 | eo = arg; |
11fdf7f2 | 248 | |
f67539c2 TL |
249 | if (eo->conf && eo->json_conf) { |
250 | SPDK_ERRLOG("Cannot provide two types of configuration files\n"); | |
251 | rc = EINVAL; | |
9f95a23c | 252 | goto err_exit; |
f67539c2 TL |
253 | } else if (eo->conf && strlen(eo->conf)) { |
254 | config = spdk_conf_allocate(); | |
255 | if (!config) { | |
256 | SPDK_ERRLOG("Unable to allocate configuration file\n"); | |
257 | rc = ENOMEM; | |
258 | goto err_exit; | |
259 | } | |
11fdf7f2 | 260 | |
f67539c2 TL |
261 | rc = spdk_conf_read(config, eo->conf); |
262 | if (rc != 0) { | |
263 | SPDK_ERRLOG("Invalid configuration file format\n"); | |
264 | spdk_conf_free(config); | |
265 | goto err_exit; | |
266 | } | |
267 | if (spdk_conf_first_section(config) == NULL) { | |
268 | SPDK_ERRLOG("Invalid configuration file format\n"); | |
269 | spdk_conf_free(config); | |
270 | rc = EINVAL; | |
271 | goto err_exit; | |
272 | } | |
273 | spdk_conf_set_as_default(config); | |
274 | } else if (eo->json_conf && strlen(eo->json_conf)) { | |
275 | g_json_config_file = eo->json_conf; | |
276 | } else { | |
277 | SPDK_ERRLOG("No configuration file provided\n"); | |
9f95a23c TL |
278 | rc = EINVAL; |
279 | goto err_exit; | |
11fdf7f2 | 280 | } |
11fdf7f2 TL |
281 | |
282 | /* Initialize the environment library */ | |
283 | spdk_env_opts_init(&opts); | |
284 | opts.name = "fio"; | |
285 | ||
286 | if (eo->mem_mb) { | |
287 | opts.mem_size = eo->mem_mb; | |
288 | } | |
289 | opts.hugepage_single_segments = eo->mem_single_seg; | |
290 | ||
291 | if (spdk_env_init(&opts) < 0) { | |
292 | SPDK_ERRLOG("Unable to initialize SPDK env\n"); | |
293 | spdk_conf_free(config); | |
9f95a23c TL |
294 | rc = EINVAL; |
295 | goto err_exit; | |
11fdf7f2 TL |
296 | } |
297 | spdk_unaffinitize_thread(); | |
298 | ||
9f95a23c TL |
299 | spdk_thread_lib_init(NULL, 0); |
300 | ||
11fdf7f2 | 301 | /* Create an SPDK thread temporarily */ |
9f95a23c | 302 | rc = spdk_fio_init_thread(&td); |
11fdf7f2 TL |
303 | if (rc < 0) { |
304 | SPDK_ERRLOG("Failed to create initialization thread\n"); | |
9f95a23c | 305 | goto err_exit; |
11fdf7f2 TL |
306 | } |
307 | ||
9f95a23c | 308 | fio_thread = td.io_ops_data; |
11fdf7f2 TL |
309 | |
310 | /* Initialize the bdev layer */ | |
9f95a23c TL |
311 | done = false; |
312 | spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_init_start, &done); | |
11fdf7f2 | 313 | |
11fdf7f2 TL |
314 | do { |
315 | spdk_fio_poll_thread(fio_thread); | |
316 | } while (!done); | |
317 | ||
318 | /* | |
319 | * Continue polling until there are no more events. | |
320 | * This handles any final events posted by pollers. | |
321 | */ | |
9f95a23c TL |
322 | while (spdk_fio_poll_thread(fio_thread) > 0) {}; |
323 | ||
324 | /* Set condition variable */ | |
325 | pthread_mutex_lock(&g_init_mtx); | |
326 | pthread_cond_signal(&g_init_cond); | |
327 | ||
328 | while (g_poll_loop) { | |
329 | spdk_fio_poll_thread(fio_thread); | |
330 | ||
331 | clock_gettime(CLOCK_MONOTONIC, &ts); | |
332 | spdk_fio_calc_timeout(fio_thread, &ts); | |
333 | ||
334 | rc = pthread_cond_timedwait(&g_init_cond, &g_init_mtx, &ts); | |
335 | if (rc != ETIMEDOUT) { | |
336 | break; | |
337 | } | |
338 | } | |
339 | ||
340 | pthread_mutex_unlock(&g_init_mtx); | |
341 | ||
342 | /* Finalize the bdev layer */ | |
343 | done = false; | |
344 | spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_fini_start, &done); | |
345 | ||
11fdf7f2 | 346 | do { |
9f95a23c TL |
347 | spdk_fio_poll_thread(fio_thread); |
348 | } while (!done && !spdk_thread_is_idle(fio_thread->thread)); | |
349 | ||
350 | spdk_fio_cleanup_thread(fio_thread); | |
351 | ||
352 | pthread_exit(NULL); | |
353 | ||
354 | err_exit: | |
355 | exit(rc); | |
356 | return NULL; | |
357 | } | |
358 | ||
359 | static int | |
360 | spdk_fio_init_env(struct thread_data *td) | |
361 | { | |
362 | pthread_condattr_t attr; | |
363 | int rc = -1; | |
364 | ||
365 | if (pthread_condattr_init(&attr)) { | |
366 | SPDK_ERRLOG("Unable to initialize condition variable\n"); | |
367 | return -1; | |
368 | } | |
369 | ||
370 | if (pthread_condattr_setclock(&attr, CLOCK_MONOTONIC)) { | |
371 | SPDK_ERRLOG("Unable to initialize condition variable\n"); | |
372 | goto out; | |
373 | } | |
374 | ||
375 | if (pthread_cond_init(&g_init_cond, &attr)) { | |
376 | SPDK_ERRLOG("Unable to initialize condition variable\n"); | |
377 | goto out; | |
378 | } | |
11fdf7f2 TL |
379 | |
380 | /* | |
9f95a23c TL |
381 | * Spawn a thread to handle initialization operations and to poll things |
382 | * like the admin queues periodically. | |
11fdf7f2 | 383 | */ |
9f95a23c | 384 | rc = pthread_create(&g_init_thread_id, NULL, &spdk_init_thread_poll, td->eo); |
11fdf7f2 TL |
385 | if (rc != 0) { |
386 | SPDK_ERRLOG("Unable to spawn thread to poll admin queue. It won't be polled.\n"); | |
387 | } | |
388 | ||
9f95a23c TL |
389 | /* Wait for background thread to advance past the initialization */ |
390 | pthread_mutex_lock(&g_init_mtx); | |
391 | pthread_cond_wait(&g_init_cond, &g_init_mtx); | |
392 | pthread_mutex_unlock(&g_init_mtx); | |
393 | out: | |
394 | pthread_condattr_destroy(&attr); | |
395 | return rc; | |
11fdf7f2 TL |
396 | } |
397 | ||
398 | /* Called for each thread to fill in the 'real_file_size' member for | |
399 | * each file associated with this thread. This is called prior to | |
400 | * the init operation (spdk_fio_init()) below. This call will occur | |
401 | * on the initial start up thread if 'create_serialize' is true, or | |
402 | * on the thread actually associated with 'thread_data' if 'create_serialize' | |
403 | * is false. | |
404 | */ | |
405 | static int | |
406 | spdk_fio_setup(struct thread_data *td) | |
407 | { | |
408 | unsigned int i; | |
409 | struct fio_file *f; | |
410 | ||
f67539c2 TL |
411 | /* we might be running in a daemonized FIO instance where standard |
412 | * input and output were closed and fds 0, 1, and 2 are reused | |
413 | * for something important by FIO. We can't ensure we won't print | |
414 | * anything (and so will our dependencies, e.g. DPDK), so abort early. | |
415 | * (is_backend is an fio global variable) | |
416 | */ | |
417 | if (is_backend) { | |
418 | char buf[1024]; | |
419 | snprintf(buf, sizeof(buf), | |
420 | "SPDK FIO plugin won't work with daemonized FIO server."); | |
421 | fio_server_text_output(FIO_LOG_ERR, buf, sizeof(buf)); | |
422 | return -1; | |
423 | } | |
424 | ||
11fdf7f2 TL |
425 | if (!td->o.use_thread) { |
426 | SPDK_ERRLOG("must set thread=1 when using spdk plugin\n"); | |
427 | return -1; | |
428 | } | |
429 | ||
430 | if (!g_spdk_env_initialized) { | |
431 | if (spdk_fio_init_env(td)) { | |
432 | SPDK_ERRLOG("failed to initialize\n"); | |
433 | return -1; | |
434 | } | |
435 | ||
436 | g_spdk_env_initialized = true; | |
437 | } | |
438 | ||
f67539c2 TL |
439 | if (td->o.nr_files == 1 && strcmp(td->files[0]->file_name, "*") == 0) { |
440 | struct spdk_bdev *bdev; | |
441 | ||
442 | /* add all available bdevs as fio targets */ | |
443 | for (bdev = spdk_bdev_first_leaf(); bdev; bdev = spdk_bdev_next_leaf(bdev)) { | |
444 | add_file(td, spdk_bdev_get_name(bdev), 0, 1); | |
445 | } | |
446 | } | |
447 | ||
11fdf7f2 TL |
448 | for_each_file(td, f, i) { |
449 | struct spdk_bdev *bdev; | |
450 | ||
f67539c2 TL |
451 | if (strcmp(f->file_name, "*") == 0) { |
452 | continue; | |
453 | } | |
454 | ||
11fdf7f2 TL |
455 | bdev = spdk_bdev_get_by_name(f->file_name); |
456 | if (!bdev) { | |
457 | SPDK_ERRLOG("Unable to find bdev with name %s\n", f->file_name); | |
458 | return -1; | |
459 | } | |
460 | ||
461 | f->real_file_size = spdk_bdev_get_num_blocks(bdev) * | |
462 | spdk_bdev_get_block_size(bdev); | |
463 | ||
464 | } | |
465 | ||
466 | return 0; | |
467 | } | |
468 | ||
9f95a23c TL |
469 | static void |
470 | spdk_fio_bdev_open(void *arg) | |
11fdf7f2 | 471 | { |
9f95a23c | 472 | struct thread_data *td = arg; |
11fdf7f2 TL |
473 | struct spdk_fio_thread *fio_thread; |
474 | unsigned int i; | |
475 | struct fio_file *f; | |
476 | int rc; | |
477 | ||
11fdf7f2 TL |
478 | fio_thread = td->io_ops_data; |
479 | ||
480 | for_each_file(td, f, i) { | |
481 | struct spdk_fio_target *target; | |
482 | ||
f67539c2 TL |
483 | if (strcmp(f->file_name, "*") == 0) { |
484 | continue; | |
485 | } | |
486 | ||
11fdf7f2 TL |
487 | target = calloc(1, sizeof(*target)); |
488 | if (!target) { | |
489 | SPDK_ERRLOG("Unable to allocate memory for I/O target.\n"); | |
9f95a23c TL |
490 | fio_thread->failed = true; |
491 | return; | |
11fdf7f2 TL |
492 | } |
493 | ||
494 | target->bdev = spdk_bdev_get_by_name(f->file_name); | |
495 | if (!target->bdev) { | |
496 | SPDK_ERRLOG("Unable to find bdev with name %s\n", f->file_name); | |
497 | free(target); | |
9f95a23c TL |
498 | fio_thread->failed = true; |
499 | return; | |
11fdf7f2 TL |
500 | } |
501 | ||
502 | rc = spdk_bdev_open(target->bdev, true, NULL, NULL, &target->desc); | |
503 | if (rc) { | |
504 | SPDK_ERRLOG("Unable to open bdev %s\n", f->file_name); | |
505 | free(target); | |
9f95a23c TL |
506 | fio_thread->failed = true; |
507 | return; | |
11fdf7f2 TL |
508 | } |
509 | ||
510 | target->ch = spdk_bdev_get_io_channel(target->desc); | |
511 | if (!target->ch) { | |
512 | SPDK_ERRLOG("Unable to get I/O channel for bdev.\n"); | |
513 | spdk_bdev_close(target->desc); | |
514 | free(target); | |
9f95a23c TL |
515 | fio_thread->failed = true; |
516 | return; | |
11fdf7f2 TL |
517 | } |
518 | ||
519 | f->engine_data = target; | |
520 | ||
521 | TAILQ_INSERT_TAIL(&fio_thread->targets, target, link); | |
522 | } | |
11fdf7f2 TL |
523 | } |
524 | ||
9f95a23c TL |
525 | /* Called for each thread, on that thread, shortly after the thread |
526 | * starts. | |
527 | */ | |
528 | static int | |
529 | spdk_fio_init(struct thread_data *td) | |
11fdf7f2 | 530 | { |
9f95a23c | 531 | struct spdk_fio_thread *fio_thread; |
11fdf7f2 | 532 | |
9f95a23c TL |
533 | spdk_fio_init_thread(td); |
534 | ||
535 | fio_thread = td->io_ops_data; | |
536 | fio_thread->failed = false; | |
537 | ||
538 | spdk_thread_send_msg(fio_thread->thread, spdk_fio_bdev_open, td); | |
11fdf7f2 TL |
539 | |
540 | while (spdk_fio_poll_thread(fio_thread) > 0) {} | |
541 | ||
9f95a23c TL |
542 | if (fio_thread->failed) { |
543 | return -1; | |
544 | } | |
545 | ||
546 | return 0; | |
11fdf7f2 TL |
547 | } |
548 | ||
549 | static void | |
550 | spdk_fio_cleanup(struct thread_data *td) | |
551 | { | |
552 | struct spdk_fio_thread *fio_thread = td->io_ops_data; | |
553 | ||
554 | spdk_fio_cleanup_thread(fio_thread); | |
555 | td->io_ops_data = NULL; | |
556 | } | |
557 | ||
558 | static int | |
559 | spdk_fio_open(struct thread_data *td, struct fio_file *f) | |
560 | { | |
561 | ||
562 | return 0; | |
563 | } | |
564 | ||
565 | static int | |
566 | spdk_fio_close(struct thread_data *td, struct fio_file *f) | |
567 | { | |
568 | return 0; | |
569 | } | |
570 | ||
571 | static int | |
572 | spdk_fio_iomem_alloc(struct thread_data *td, size_t total_mem) | |
573 | { | |
574 | td->orig_buffer = spdk_dma_zmalloc(total_mem, 0x1000, NULL); | |
575 | return td->orig_buffer == NULL; | |
576 | } | |
577 | ||
578 | static void | |
579 | spdk_fio_iomem_free(struct thread_data *td) | |
580 | { | |
581 | spdk_dma_free(td->orig_buffer); | |
582 | } | |
583 | ||
584 | static int | |
585 | spdk_fio_io_u_init(struct thread_data *td, struct io_u *io_u) | |
586 | { | |
587 | struct spdk_fio_request *fio_req; | |
588 | ||
f67539c2 TL |
589 | io_u->engine_data = NULL; |
590 | ||
11fdf7f2 TL |
591 | fio_req = calloc(1, sizeof(*fio_req)); |
592 | if (fio_req == NULL) { | |
593 | return 1; | |
594 | } | |
595 | fio_req->io = io_u; | |
596 | fio_req->td = td; | |
597 | ||
598 | io_u->engine_data = fio_req; | |
599 | ||
600 | return 0; | |
601 | } | |
602 | ||
603 | static void | |
604 | spdk_fio_io_u_free(struct thread_data *td, struct io_u *io_u) | |
605 | { | |
606 | struct spdk_fio_request *fio_req = io_u->engine_data; | |
607 | ||
608 | if (fio_req) { | |
609 | assert(fio_req->io == io_u); | |
610 | free(fio_req); | |
611 | io_u->engine_data = NULL; | |
612 | } | |
613 | } | |
614 | ||
615 | static void | |
616 | spdk_fio_completion_cb(struct spdk_bdev_io *bdev_io, | |
617 | bool success, | |
618 | void *cb_arg) | |
619 | { | |
620 | struct spdk_fio_request *fio_req = cb_arg; | |
621 | struct thread_data *td = fio_req->td; | |
622 | struct spdk_fio_thread *fio_thread = td->io_ops_data; | |
623 | ||
624 | assert(fio_thread->iocq_count < fio_thread->iocq_size); | |
625 | fio_req->io->error = success ? 0 : EIO; | |
626 | fio_thread->iocq[fio_thread->iocq_count++] = fio_req->io; | |
627 | ||
628 | spdk_bdev_free_io(bdev_io); | |
629 | } | |
630 | ||
631 | #if FIO_IOOPS_VERSION >= 24 | |
632 | typedef enum fio_q_status fio_q_status_t; | |
633 | #else | |
634 | typedef int fio_q_status_t; | |
635 | #endif | |
636 | ||
637 | static fio_q_status_t | |
638 | spdk_fio_queue(struct thread_data *td, struct io_u *io_u) | |
639 | { | |
640 | int rc = 1; | |
641 | struct spdk_fio_request *fio_req = io_u->engine_data; | |
642 | struct spdk_fio_target *target = io_u->file->engine_data; | |
643 | ||
644 | assert(fio_req->td == td); | |
645 | ||
646 | if (!target) { | |
647 | SPDK_ERRLOG("Unable to look up correct I/O target.\n"); | |
648 | fio_req->io->error = ENODEV; | |
649 | return FIO_Q_COMPLETED; | |
650 | } | |
651 | ||
652 | switch (io_u->ddir) { | |
653 | case DDIR_READ: | |
654 | rc = spdk_bdev_read(target->desc, target->ch, | |
655 | io_u->buf, io_u->offset, io_u->xfer_buflen, | |
656 | spdk_fio_completion_cb, fio_req); | |
657 | break; | |
658 | case DDIR_WRITE: | |
659 | rc = spdk_bdev_write(target->desc, target->ch, | |
660 | io_u->buf, io_u->offset, io_u->xfer_buflen, | |
661 | spdk_fio_completion_cb, fio_req); | |
662 | break; | |
663 | case DDIR_TRIM: | |
664 | rc = spdk_bdev_unmap(target->desc, target->ch, | |
665 | io_u->offset, io_u->xfer_buflen, | |
666 | spdk_fio_completion_cb, fio_req); | |
667 | break; | |
668 | default: | |
669 | assert(false); | |
670 | break; | |
671 | } | |
672 | ||
673 | if (rc == -ENOMEM) { | |
674 | return FIO_Q_BUSY; | |
675 | } | |
676 | ||
677 | if (rc != 0) { | |
678 | fio_req->io->error = abs(rc); | |
679 | return FIO_Q_COMPLETED; | |
680 | } | |
681 | ||
682 | return FIO_Q_QUEUED; | |
683 | } | |
684 | ||
685 | static struct io_u * | |
686 | spdk_fio_event(struct thread_data *td, int event) | |
687 | { | |
688 | struct spdk_fio_thread *fio_thread = td->io_ops_data; | |
689 | ||
690 | assert(event >= 0); | |
691 | assert((unsigned)event < fio_thread->iocq_count); | |
692 | return fio_thread->iocq[event]; | |
693 | } | |
694 | ||
695 | static size_t | |
696 | spdk_fio_poll_thread(struct spdk_fio_thread *fio_thread) | |
697 | { | |
9f95a23c | 698 | return spdk_thread_poll(fio_thread->thread, 0, 0); |
11fdf7f2 TL |
699 | } |
700 | ||
701 | static int | |
702 | spdk_fio_getevents(struct thread_data *td, unsigned int min, | |
703 | unsigned int max, const struct timespec *t) | |
704 | { | |
705 | struct spdk_fio_thread *fio_thread = td->io_ops_data; | |
706 | struct timespec t0, t1; | |
707 | uint64_t timeout = 0; | |
708 | ||
709 | if (t) { | |
9f95a23c | 710 | timeout = t->tv_sec * SPDK_SEC_TO_NSEC + t->tv_nsec; |
11fdf7f2 TL |
711 | clock_gettime(CLOCK_MONOTONIC_RAW, &t0); |
712 | } | |
713 | ||
714 | fio_thread->iocq_count = 0; | |
715 | ||
716 | for (;;) { | |
717 | spdk_fio_poll_thread(fio_thread); | |
718 | ||
719 | if (fio_thread->iocq_count >= min) { | |
720 | return fio_thread->iocq_count; | |
721 | } | |
722 | ||
723 | if (t) { | |
724 | clock_gettime(CLOCK_MONOTONIC_RAW, &t1); | |
9f95a23c | 725 | uint64_t elapse = ((t1.tv_sec - t0.tv_sec) * SPDK_SEC_TO_NSEC) |
11fdf7f2 TL |
726 | + t1.tv_nsec - t0.tv_nsec; |
727 | if (elapse > timeout) { | |
728 | break; | |
729 | } | |
730 | } | |
731 | } | |
732 | ||
733 | return fio_thread->iocq_count; | |
734 | } | |
735 | ||
736 | static int | |
737 | spdk_fio_invalidate(struct thread_data *td, struct fio_file *f) | |
738 | { | |
739 | /* TODO: This should probably send a flush to the device, but for now just return successful. */ | |
740 | return 0; | |
741 | } | |
742 | ||
743 | static struct fio_option options[] = { | |
744 | { | |
745 | .name = "spdk_conf", | |
746 | .lname = "SPDK configuration file", | |
747 | .type = FIO_OPT_STR_STORE, | |
748 | .off1 = offsetof(struct spdk_fio_options, conf), | |
749 | .help = "A SPDK configuration file", | |
750 | .category = FIO_OPT_C_ENGINE, | |
751 | .group = FIO_OPT_G_INVALID, | |
752 | }, | |
f67539c2 TL |
753 | { |
754 | .name = "spdk_json_conf", | |
755 | .lname = "SPDK JSON configuration file", | |
756 | .type = FIO_OPT_STR_STORE, | |
757 | .off1 = offsetof(struct spdk_fio_options, json_conf), | |
758 | .help = "A SPDK JSON configuration file", | |
759 | .category = FIO_OPT_C_ENGINE, | |
760 | .group = FIO_OPT_G_INVALID, | |
761 | }, | |
11fdf7f2 TL |
762 | { |
763 | .name = "spdk_mem", | |
764 | .lname = "SPDK memory in MB", | |
765 | .type = FIO_OPT_INT, | |
766 | .off1 = offsetof(struct spdk_fio_options, mem_mb), | |
767 | .help = "Amount of memory in MB to allocate for SPDK", | |
768 | .category = FIO_OPT_C_ENGINE, | |
769 | .group = FIO_OPT_G_INVALID, | |
770 | }, | |
771 | { | |
772 | .name = "spdk_single_seg", | |
773 | .lname = "SPDK switch to create just a single hugetlbfs file", | |
774 | .type = FIO_OPT_BOOL, | |
775 | .off1 = offsetof(struct spdk_fio_options, mem_single_seg), | |
776 | .help = "If set to 1, SPDK will use just a single hugetlbfs file", | |
777 | .category = FIO_OPT_C_ENGINE, | |
778 | .group = FIO_OPT_G_INVALID, | |
779 | }, | |
780 | { | |
781 | .name = NULL, | |
782 | }, | |
783 | }; | |
784 | ||
785 | /* FIO imports this structure using dlsym */ | |
786 | struct ioengine_ops ioengine = { | |
787 | .name = "spdk_bdev", | |
788 | .version = FIO_IOOPS_VERSION, | |
789 | .flags = FIO_RAWIO | FIO_NOEXTEND | FIO_NODISKUTIL | FIO_MEMALIGN, | |
790 | .setup = spdk_fio_setup, | |
791 | .init = spdk_fio_init, | |
9f95a23c | 792 | /* .prep = unused, */ |
11fdf7f2 | 793 | .queue = spdk_fio_queue, |
9f95a23c | 794 | /* .commit = unused, */ |
11fdf7f2 TL |
795 | .getevents = spdk_fio_getevents, |
796 | .event = spdk_fio_event, | |
9f95a23c TL |
797 | /* .errdetails = unused, */ |
798 | /* .cancel = unused, */ | |
11fdf7f2 TL |
799 | .cleanup = spdk_fio_cleanup, |
800 | .open_file = spdk_fio_open, | |
801 | .close_file = spdk_fio_close, | |
802 | .invalidate = spdk_fio_invalidate, | |
9f95a23c TL |
803 | /* .unlink_file = unused, */ |
804 | /* .get_file_size = unused, */ | |
805 | /* .terminate = unused, */ | |
11fdf7f2 TL |
806 | .iomem_alloc = spdk_fio_iomem_alloc, |
807 | .iomem_free = spdk_fio_iomem_free, | |
808 | .io_u_init = spdk_fio_io_u_init, | |
809 | .io_u_free = spdk_fio_io_u_free, | |
810 | .option_struct_size = sizeof(struct spdk_fio_options), | |
811 | .options = options, | |
812 | }; | |
813 | ||
814 | static void fio_init spdk_fio_register(void) | |
815 | { | |
816 | register_ioengine(&ioengine); | |
817 | } | |
818 | ||
11fdf7f2 TL |
819 | static void |
820 | spdk_fio_finish_env(void) | |
821 | { | |
9f95a23c TL |
822 | pthread_mutex_lock(&g_init_mtx); |
823 | g_poll_loop = false; | |
824 | pthread_cond_signal(&g_init_cond); | |
825 | pthread_mutex_unlock(&g_init_mtx); | |
826 | pthread_join(g_init_thread_id, NULL); | |
11fdf7f2 | 827 | |
9f95a23c | 828 | spdk_thread_lib_fini(); |
11fdf7f2 TL |
829 | } |
830 | ||
831 | static void fio_exit spdk_fio_unregister(void) | |
832 | { | |
833 | if (g_spdk_env_initialized) { | |
834 | spdk_fio_finish_env(); | |
835 | g_spdk_env_initialized = false; | |
836 | } | |
837 | unregister_ioengine(&ioengine); | |
838 | } |