ceph/src/spdk/lib/bdev/split/vbdev_split.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) Intel Corporation.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 /*
  35  * This is a simple example of a virtual block device that takes a single
  36  * bdev and slices it into multiple smaller bdevs.
  37  */
  38
  39 #include "vbdev_split.h"
  40
  41 #include "spdk/rpc.h"
  42 #include "spdk/conf.h"
  43 #include "spdk/endian.h"
  44 #include "spdk/string.h"
  45 #include "spdk/thread.h"
  46 #include "spdk/util.h"
  47
  48 #include "spdk/bdev_module.h"
  49 #include "spdk_internal/log.h"
  50
  51 struct spdk_vbdev_split_config {
  52         char *base_bdev;
  53         unsigned split_count;
  54         uint64_t split_size_mb;
  55
  56         struct spdk_bdev_part_base *split_base;
  57         bool removed;
  58
  59         TAILQ_ENTRY(spdk_vbdev_split_config) tailq;
  60 };
  61
  62 static TAILQ_HEAD(, spdk_vbdev_split_config) g_split_config = TAILQ_HEAD_INITIALIZER(
  63                         g_split_config);
  64 static SPDK_BDEV_PART_TAILQ g_split_disks = TAILQ_HEAD_INITIALIZER(g_split_disks);
  65
  66 struct vbdev_split_channel {
  67         struct spdk_bdev_part_channel   part_ch;
  68 };
  69
  70 struct vbdev_split_bdev_io {
  71         struct spdk_io_channel *ch;
  72         struct spdk_bdev_io *bdev_io;
  73
  74         /* for bdev_io_wait */
  75         struct spdk_bdev_io_wait_entry bdev_io_wait;
  76 };
  77
  78 static void vbdev_split_del_config(struct spdk_vbdev_split_config *cfg);
  79
  80 static int vbdev_split_init(void);
  81 static void vbdev_split_fini(void);
  82 static void vbdev_split_examine(struct spdk_bdev *bdev);
  83 static int vbdev_split_config_json(struct spdk_json_write_ctx *w);
  84 static int vbdev_split_get_ctx_size(void);
  85
  86 static void
  87 vbdev_split_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io);
  88
  89 static struct spdk_bdev_module split_if = {
  90         .name = "split",
  91         .module_init = vbdev_split_init,
  92         .module_fini = vbdev_split_fini,
  93         .get_ctx_size = vbdev_split_get_ctx_size,
  94         .examine_config = vbdev_split_examine,
  95         .config_json = vbdev_split_config_json,
  96 };
  97
  98 SPDK_BDEV_MODULE_REGISTER(&split_if)
  99
 100 static void
 101 vbdev_split_base_free(void *ctx)
 102 {
 103         struct spdk_vbdev_split_config *cfg = ctx;
 104
 105         cfg->split_base = NULL;
 106         if (cfg->removed) {
 107                 vbdev_split_del_config(cfg);
 108         }
 109 }
 110
 111 static int
 112 vbdev_split_destruct(void *ctx)
 113 {
 114         struct spdk_bdev_part *part = ctx;
 115
 116         return spdk_bdev_part_free(part);
 117 }
 118
 119 static void
 120 vbdev_split_base_bdev_hotremove_cb(void *_base_bdev)
 121 {
 122         spdk_bdev_part_base_hotremove(_base_bdev, &g_split_disks);
 123 }
 124
 125 static void
 126 vbdev_split_resubmit_io(void *arg)
 127 {
 128         struct vbdev_split_bdev_io *split_io = (struct vbdev_split_bdev_io *)arg;
 129
 130         vbdev_split_submit_request(split_io->ch, split_io->bdev_io);
 131 }
 132
 133 static void
 134 vbdev_split_queue_io(struct vbdev_split_bdev_io *split_io)
 135 {
 136         int rc;
 137
 138         split_io->bdev_io_wait.bdev = split_io->bdev_io->bdev;
 139         split_io->bdev_io_wait.cb_fn = vbdev_split_resubmit_io;
 140         split_io->bdev_io_wait.cb_arg = split_io;
 141
 142         rc = spdk_bdev_queue_io_wait(split_io->bdev_io->bdev,
 143                                      split_io->ch, &split_io->bdev_io_wait);
 144         if (rc != 0) {
 145                 SPDK_ERRLOG("Queue io failed in vbdev_split_queue_io, rc=%d\n", rc);
 146                 spdk_bdev_io_complete(split_io->bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
 147         }
 148 }
 149
 150 static void
 151 vbdev_split_submit_request(struct spdk_io_channel *_ch, struct spdk_bdev_io *bdev_io)
 152 {
 153         struct vbdev_split_channel *ch = spdk_io_channel_get_ctx(_ch);
 154         struct vbdev_split_bdev_io *io_ctx = (struct vbdev_split_bdev_io *)bdev_io->driver_ctx;
 155         int rc;
 156
 157         rc = spdk_bdev_part_submit_request(&ch->part_ch, bdev_io);
 158         if (rc) {
 159                 if (rc == -ENOMEM) {
 160                         SPDK_DEBUGLOG(SPDK_LOG_VBDEV_SPLIT, "split: no memory, queue io.\n");
 161                         io_ctx->ch = _ch;
 162                         io_ctx->bdev_io = bdev_io;
 163                         vbdev_split_queue_io(io_ctx);
 164                 } else {
 165                         SPDK_ERRLOG("split: error on io submission, rc=%d.\n", rc);
 166                         spdk_bdev_io_complete(bdev_io, SPDK_BDEV_IO_STATUS_FAILED);
 167                 }
 168         }
 169 }
 170
 171 static int
 172 vbdev_split_dump_info_json(void *ctx, struct spdk_json_write_ctx *w)
 173 {
 174         struct spdk_bdev_part *part = ctx;
 175         struct spdk_bdev *split_base_bdev = spdk_bdev_part_get_base_bdev(part);
 176         uint64_t offset_blocks = spdk_bdev_part_get_offset_blocks(part);
 177
 178         spdk_json_write_name(w, "split");
 179         spdk_json_write_object_begin(w);
 180
 181         spdk_json_write_name(w, "base_bdev");
 182         spdk_json_write_string(w, spdk_bdev_get_name(split_base_bdev));
 183         spdk_json_write_name(w, "offset_blocks");
 184         spdk_json_write_uint64(w, offset_blocks);
 185
 186         spdk_json_write_object_end(w);
 187
 188         return 0;
 189 }
 190
 191 static void
 192 vbdev_split_write_config_json(struct spdk_bdev *bdev, struct spdk_json_write_ctx *w)
 193 {
 194         /* No config per bdev needed */
 195 }
 196
 197 static struct spdk_bdev_fn_table vbdev_split_fn_table = {
 198         .destruct               = vbdev_split_destruct,
 199         .submit_request         = vbdev_split_submit_request,
 200         .dump_info_json         = vbdev_split_dump_info_json,
 201         .write_config_json      = vbdev_split_write_config_json
 202 };
 203
 204 static int
 205 vbdev_split_create(struct spdk_vbdev_split_config *cfg)
 206 {
 207         uint64_t split_size_blocks, offset_blocks;
 208         uint64_t split_count, max_split_count;
 209         uint64_t mb = 1024 * 1024;
 210         uint64_t i;
 211         int rc;
 212         char *name;
 213         struct spdk_bdev *base_bdev;
 214         struct spdk_bdev *split_base_bdev;
 215         struct bdev_part_tailq *split_base_tailq;
 216
 217         assert(cfg->split_count > 0);
 218
 219         base_bdev = spdk_bdev_get_by_name(cfg->base_bdev);
 220         if (!base_bdev) {
 221                 return -ENODEV;
 222         }
 223
 224         if (cfg->split_size_mb) {
 225                 if (((cfg->split_size_mb * mb) % base_bdev->blocklen) != 0) {
 226                         SPDK_ERRLOG("Split size %" PRIu64 " MB is not possible with block size "
 227                                     "%" PRIu32 "\n",
 228                                     cfg->split_size_mb, base_bdev->blocklen);
 229                         return -EINVAL;
 230                 }
 231                 split_size_blocks = (cfg->split_size_mb * mb) / base_bdev->blocklen;
 232                 SPDK_DEBUGLOG(SPDK_LOG_VBDEV_SPLIT, "Split size %" PRIu64 " MB specified by user\n",
 233                               cfg->split_size_mb);
 234         } else {
 235                 split_size_blocks = base_bdev->blockcnt / cfg->split_count;
 236                 SPDK_DEBUGLOG(SPDK_LOG_VBDEV_SPLIT, "Split size not specified by user\n");
 237         }
 238
 239         max_split_count = base_bdev->blockcnt / split_size_blocks;
 240         split_count = cfg->split_count;
 241         if (split_count > max_split_count) {
 242                 SPDK_WARNLOG("Split count %" PRIu64 " is greater than maximum possible split count "
 243                              "%" PRIu64 " - clamping\n", split_count, max_split_count);
 244                 split_count = max_split_count;
 245         }
 246
 247         SPDK_DEBUGLOG(SPDK_LOG_VBDEV_SPLIT, "base_bdev: %s split_count: %" PRIu64
 248                       " split_size_blocks: %" PRIu64 "\n",
 249                       spdk_bdev_get_name(base_bdev), split_count, split_size_blocks);
 250
 251         cfg->split_base = spdk_bdev_part_base_construct(base_bdev,
 252                           vbdev_split_base_bdev_hotremove_cb,
 253                           &split_if, &vbdev_split_fn_table,
 254                           &g_split_disks, vbdev_split_base_free, cfg,
 255                           sizeof(struct vbdev_split_channel), NULL, NULL);
 256         if (!cfg->split_base) {
 257                 SPDK_ERRLOG("Cannot construct bdev part base\n");
 258                 return -ENOMEM;
 259         }
 260
 261         offset_blocks = 0;
 262         for (i = 0; i < split_count; i++) {
 263                 struct spdk_bdev_part *d;
 264
 265                 d = calloc(1, sizeof(*d));
 266                 if (d == NULL) {
 267                         SPDK_ERRLOG("could not allocate bdev part\n");
 268                         rc = -ENOMEM;
 269                         goto err;
 270                 }
 271
 272                 name = spdk_sprintf_alloc("%sp%" PRIu64, cfg->base_bdev, i);
 273                 if (!name) {
 274                         SPDK_ERRLOG("could not allocate name\n");
 275                         free(d);
 276                         rc = -ENOMEM;
 277                         goto err;
 278                 }
 279
 280                 rc = spdk_bdev_part_construct(d, cfg->split_base, name, offset_blocks, split_size_blocks,
 281                                               "Split Disk");
 282                 free(name);
 283                 if (rc) {
 284                         SPDK_ERRLOG("could not construct bdev part\n");
 285                         /* spdk_bdev_part_construct will free name if it fails */
 286                         free(d);
 287                         rc = -ENOMEM;
 288                         goto err;
 289                 }
 290
 291                 offset_blocks += split_size_blocks;
 292         }
 293
 294         return 0;
 295 err:
 296         split_base_bdev = spdk_bdev_part_base_get_bdev(cfg->split_base);
 297         split_base_tailq = spdk_bdev_part_base_get_tailq(cfg->split_base);
 298         cfg->removed = true;
 299         spdk_bdev_part_base_hotremove(split_base_bdev, split_base_tailq);
 300         return rc;
 301 }
 302
 303 static void
 304 vbdev_split_del_config(struct spdk_vbdev_split_config *cfg)
 305 {
 306         TAILQ_REMOVE(&g_split_config, cfg, tailq);
 307         free(cfg->base_bdev);
 308         free(cfg);
 309 }
 310
 311 static void
 312 vbdev_split_destruct_config(struct spdk_vbdev_split_config *cfg)
 313 {
 314         struct spdk_bdev *split_base_bdev;
 315         struct bdev_part_tailq *split_base_tailq;
 316
 317         cfg->removed = true;
 318         if (cfg->split_base != NULL) {
 319                 split_base_bdev = spdk_bdev_part_base_get_bdev(cfg->split_base);
 320                 split_base_tailq = spdk_bdev_part_base_get_tailq(cfg->split_base);
 321                 spdk_bdev_part_base_hotremove(split_base_bdev, split_base_tailq);
 322         } else {
 323                 vbdev_split_del_config(cfg);
 324         }
 325 }
 326
 327 static void
 328 vbdev_split_clear_config(void)
 329 {
 330         struct spdk_vbdev_split_config *cfg, *tmp_cfg;
 331
 332         TAILQ_FOREACH_SAFE(cfg, &g_split_config, tailq, tmp_cfg) {
 333                 vbdev_split_destruct_config(cfg);
 334         }
 335 }
 336
 337 static struct spdk_vbdev_split_config *
 338 vbdev_split_config_find_by_base_name(const char *base_bdev_name)
 339 {
 340         struct spdk_vbdev_split_config *cfg;
 341
 342         TAILQ_FOREACH(cfg, &g_split_config, tailq) {
 343                 if (strcmp(cfg->base_bdev, base_bdev_name) == 0) {
 344                         return cfg;
 345                 }
 346         }
 347
 348         return NULL;
 349 }
 350
 351 static int
 352 vbdev_split_add_config(const char *base_bdev_name, unsigned split_count, uint64_t split_size,
 353                        struct spdk_vbdev_split_config **config)
 354 {
 355         struct spdk_vbdev_split_config *cfg;
 356         assert(base_bdev_name);
 357
 358         if (base_bdev_name == NULL) {
 359                 SPDK_ERRLOG("Split bdev config: no base bdev provided.");
 360                 return -EINVAL;
 361         }
 362
 363         if (split_count == 0) {
 364                 SPDK_ERRLOG("Split bdev config: split_count can't be 0.");
 365                 return -EINVAL;
 366         }
 367
 368         /* Check if we already have 'base_bdev_name' registered in config */
 369         cfg = vbdev_split_config_find_by_base_name(base_bdev_name);
 370         if (cfg) {
 371                 SPDK_ERRLOG("Split bdev config for base bdev '%s' already exist.", base_bdev_name);
 372                 return -EEXIST;
 373         }
 374
 375         cfg = calloc(1, sizeof(*cfg));
 376         if (!cfg) {
 377                 SPDK_ERRLOG("calloc(): Out of memory");
 378                 return -ENOMEM;
 379         }
 380
 381         cfg->base_bdev = strdup(base_bdev_name);
 382         if (!cfg->base_bdev) {
 383                 SPDK_ERRLOG("strdup(): Out of memory");
 384                 free(cfg);
 385                 return -ENOMEM;
 386         }
 387
 388         cfg->split_count = split_count;
 389         cfg->split_size_mb = split_size;
 390         TAILQ_INSERT_TAIL(&g_split_config, cfg, tailq);
 391         if (config) {
 392                 *config = cfg;
 393         }
 394
 395         return 0;
 396 }
 397
 398 static int
 399 vbdev_split_init(void)
 400 {
 401
 402         struct spdk_conf_section *sp;
 403         const char *base_bdev_name;
 404         const char *split_count_str;
 405         const char *split_size_str;
 406         int rc, i, split_count, split_size;
 407
 408         sp = spdk_conf_find_section(NULL, "Split");
 409         if (sp == NULL) {
 410                 return 0;
 411         }
 412
 413         for (i = 0; ; i++) {
 414                 if (!spdk_conf_section_get_nval(sp, "Split", i)) {
 415                         break;
 416                 }
 417
 418                 base_bdev_name = spdk_conf_section_get_nmval(sp, "Split", i, 0);
 419                 if (!base_bdev_name) {
 420                         SPDK_ERRLOG("Split configuration missing bdev name\n");
 421                         rc = -EINVAL;
 422                         goto err;
 423                 }
 424
 425                 split_count_str = spdk_conf_section_get_nmval(sp, "Split", i, 1);
 426                 if (!split_count_str) {
 427                         SPDK_ERRLOG("Split configuration missing split count\n");
 428                         rc = -EINVAL;
 429                         goto err;
 430                 }
 431
 432                 split_count = atoi(split_count_str);
 433                 if (split_count < 1) {
 434                         SPDK_ERRLOG("Invalid Split count %d\n", split_count);
 435                         rc = -EINVAL;
 436                         goto err;
 437                 }
 438
 439                 /* Optional split size in MB */
 440                 split_size = 0;
 441                 split_size_str = spdk_conf_section_get_nmval(sp, "Split", i, 2);
 442                 if (split_size_str) {
 443                         split_size = atoi(split_size_str);
 444                         if (split_size <= 0) {
 445                                 SPDK_ERRLOG("Invalid Split size %d\n", split_size);
 446                                 rc = -EINVAL;
 447                                 goto err;
 448                         }
 449                 }
 450
 451                 rc = vbdev_split_add_config(base_bdev_name, split_count, split_size, NULL);
 452                 if (rc != 0) {
 453                         goto err;
 454                 }
 455         }
 456
 457         return 0;
 458 err:
 459         vbdev_split_clear_config();
 460         return rc;
 461 }
 462
 463 static void
 464 vbdev_split_fini(void)
 465 {
 466         vbdev_split_clear_config();
 467 }
 468
 469 static void
 470 vbdev_split_examine(struct spdk_bdev *bdev)
 471 {
 472         struct spdk_vbdev_split_config *cfg = vbdev_split_config_find_by_base_name(bdev->name);
 473
 474         if (cfg != NULL && cfg->removed == false) {
 475                 assert(cfg->split_base == NULL);
 476
 477                 if (vbdev_split_create(cfg)) {
 478                         SPDK_ERRLOG("could not split bdev %s\n", bdev->name);
 479                 }
 480         }
 481         spdk_bdev_module_examine_done(&split_if);
 482 }
 483
 484 static int
 485 vbdev_split_config_json(struct spdk_json_write_ctx *w)
 486 {
 487         struct spdk_vbdev_split_config *cfg;
 488
 489         TAILQ_FOREACH(cfg, &g_split_config, tailq) {
 490                 spdk_json_write_object_begin(w);
 491
 492                 spdk_json_write_named_string(w, "method", "construct_split_vbdev");
 493
 494                 spdk_json_write_named_object_begin(w, "params");
 495                 spdk_json_write_named_string(w, "base_bdev", cfg->base_bdev);
 496                 spdk_json_write_named_uint32(w, "split_count", cfg->split_count);
 497                 spdk_json_write_named_uint64(w, "split_size_mb", cfg->split_size_mb);
 498                 spdk_json_write_object_end(w);
 499
 500                 spdk_json_write_object_end(w);
 501         }
 502
 503         return 0;
 504 }
 505
 506 int
 507 create_vbdev_split(const char *base_bdev_name, unsigned split_count, uint64_t split_size_mb)
 508 {
 509         int rc;
 510         struct spdk_vbdev_split_config *cfg;
 511
 512         rc = vbdev_split_add_config(base_bdev_name, split_count, split_size_mb, &cfg);
 513         if (rc) {
 514                 return rc;
 515         }
 516
 517         rc = vbdev_split_create(cfg);
 518         if (rc == -ENODEV) {
 519                 /* It is ok if base bdev does not exist yet. */
 520                 rc = 0;
 521         }
 522
 523         return rc;
 524 }
 525
 526 int
 527 spdk_vbdev_split_destruct(const char *base_bdev_name)
 528 {
 529         struct spdk_vbdev_split_config *cfg = vbdev_split_config_find_by_base_name(base_bdev_name);
 530
 531         if (!cfg) {
 532                 SPDK_ERRLOG("Split configuration for '%s' not found\n", base_bdev_name);
 533                 return -ENOENT;
 534         }
 535
 536         vbdev_split_destruct_config(cfg);
 537         return 0;
 538 }
 539
 540 struct spdk_bdev_part_base *
 541 spdk_vbdev_split_get_part_base(struct spdk_bdev *bdev)
 542 {
 543         struct spdk_vbdev_split_config *cfg;
 544
 545         cfg = vbdev_split_config_find_by_base_name(spdk_bdev_get_name(bdev));
 546
 547         if (cfg == NULL) {
 548                 return NULL;
 549         }
 550
 551         return cfg->split_base;
 552 }
 553
 554 /*
 555  * During init we'll be asked how much memory we'd like passed to us
 556  * in bev_io structures as context. Here's where we specify how
 557  * much context we want per IO.
 558  */
 559 static int
 560 vbdev_split_get_ctx_size(void)
 561 {
 562         return sizeof(struct vbdev_split_bdev_io);
 563 }
 564
 565 SPDK_LOG_REGISTER_COMPONENT("vbdev_split", SPDK_LOG_VBDEV_SPLIT)