ceph/src/spdk/examples/nvme/reserve/reserve.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) Intel Corporation.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include "spdk/stdinc.h"
  35
  36 #include "spdk/endian.h"
  37 #include "spdk/nvme.h"
  38 #include "spdk/env.h"
  39 #include "spdk/log.h"
  40
  41 #define MAX_DEVS 64
  42
  43 struct dev {
  44         struct spdk_pci_addr                    pci_addr;
  45         struct spdk_nvme_ctrlr                  *ctrlr;
  46         char                                    name[100];
  47 };
  48
  49 static struct dev devs[MAX_DEVS];
  50 static int num_devs = 0;
  51
  52 #define foreach_dev(iter) \
  53         for (iter = devs; iter - devs < num_devs; iter++)
  54
  55 static int outstanding_commands;
  56 static int reserve_command_result;
  57 static bool get_host_id_successful;
  58
  59 #define HOST_ID         0xABABABABCDCDCDCD
  60 #define EXT_HOST_ID     ((uint8_t[]){0x0f, 0x97, 0xcd, 0x74, 0x8c, 0x80, 0x41, 0x42, \
  61                                      0x99, 0x0f, 0x65, 0xc4, 0xf0, 0x39, 0x24, 0x20})
  62
  63 #define CR_KEY          0xDEADBEAF5A5A5A5B
  64
  65 static void
  66 get_feature_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
  67 {
  68         if (spdk_nvme_cpl_is_error(cpl)) {
  69                 fprintf(stdout, "Get Features - Host Identifier failed\n");
  70                 get_host_id_successful = false;
  71         } else {
  72                 get_host_id_successful = true;
  73         }
  74         outstanding_commands--;
  75 }
  76
  77 static int
  78 get_host_identifier(struct spdk_nvme_ctrlr *ctrlr)
  79 {
  80         int ret;
  81         uint8_t host_id[16];
  82         uint32_t host_id_size;
  83         uint32_t cdw11;
  84
  85         if (spdk_nvme_ctrlr_get_data(ctrlr)->ctratt.host_id_exhid_supported) {
  86                 host_id_size = 16;
  87                 cdw11 = 1;
  88                 printf("Using 128-bit extended host identifier\n");
  89         } else {
  90                 host_id_size = 8;
  91                 cdw11 = 0;
  92                 printf("Using 64-bit host identifier\n");
  93         }
  94
  95         outstanding_commands = 0;
  96         ret = spdk_nvme_ctrlr_cmd_get_feature(ctrlr, SPDK_NVME_FEAT_HOST_IDENTIFIER, cdw11, host_id,
  97                                               host_id_size,
  98                                               get_feature_completion, NULL);
  99         if (ret) {
 100                 fprintf(stdout, "Get Feature: Failed\n");
 101                 return -1;
 102         }
 103
 104         outstanding_commands++;
 105         get_host_id_successful = false;
 106
 107         while (outstanding_commands) {
 108                 spdk_nvme_ctrlr_process_admin_completions(ctrlr);
 109         }
 110
 111         if (get_host_id_successful) {
 112                 spdk_log_dump(stdout, "Get Feature: Host Identifier:", host_id, host_id_size);
 113         }
 114
 115         return 0;
 116 }
 117
 118 static void
 119 reservation_ns_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
 120 {
 121         if (spdk_nvme_cpl_is_error(cpl)) {
 122                 reserve_command_result = -1;
 123         } else {
 124                 reserve_command_result = 0;
 125         }
 126
 127         outstanding_commands--;
 128 }
 129
 130 static int
 131 reservation_ns_register(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
 132                         uint32_t ns_id)
 133 {
 134         int ret;
 135         struct spdk_nvme_reservation_register_data rr_data;
 136         struct spdk_nvme_ns *ns;
 137
 138         ns = spdk_nvme_ctrlr_get_ns(ctrlr, ns_id);
 139
 140         rr_data.crkey = CR_KEY;
 141         rr_data.nrkey = CR_KEY;
 142
 143         outstanding_commands = 0;
 144         reserve_command_result = -1;
 145
 146         ret = spdk_nvme_ns_cmd_reservation_register(ns, qpair, &rr_data, true,
 147                         SPDK_NVME_RESERVE_REGISTER_KEY,
 148                         SPDK_NVME_RESERVE_PTPL_NO_CHANGES,
 149                         reservation_ns_completion, NULL);
 150         if (ret) {
 151                 fprintf(stderr, "Reservation Register Failed\n");
 152                 return -1;
 153         }
 154
 155         outstanding_commands++;
 156         while (outstanding_commands) {
 157                 spdk_nvme_qpair_process_completions(qpair, 100);
 158         }
 159
 160         if (reserve_command_result) {
 161                 fprintf(stderr, "Reservation Register Failed\n");
 162         }
 163
 164         return 0;
 165 }
 166
 167 static int
 168 reservation_ns_report(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, uint32_t ns_id)
 169 {
 170         int ret, i;
 171         uint8_t *payload;
 172         struct spdk_nvme_reservation_status_data *status;
 173         struct spdk_nvme_registered_ctrlr_data *cdata;
 174         struct spdk_nvme_ns *ns;
 175
 176         ns = spdk_nvme_ctrlr_get_ns(ctrlr, ns_id);
 177
 178         outstanding_commands = 0;
 179         reserve_command_result = -1;
 180
 181         payload = spdk_dma_zmalloc(0x1000, 0x1000, NULL);
 182         if (!payload) {
 183                 fprintf(stderr, "DMA Buffer Allocation Failed\n");
 184                 return -1;
 185         }
 186
 187         ret = spdk_nvme_ns_cmd_reservation_report(ns, qpair, payload, 0x1000,
 188                         reservation_ns_completion, NULL);
 189         if (ret) {
 190                 fprintf(stderr, "Reservation Report Failed\n");
 191                 spdk_dma_free(payload);
 192                 return -1;
 193         }
 194
 195         outstanding_commands++;
 196         while (outstanding_commands) {
 197                 spdk_nvme_qpair_process_completions(qpair, 100);
 198         }
 199
 200         if (reserve_command_result) {
 201                 fprintf(stderr, "Reservation Report Failed\n");
 202                 spdk_dma_free(payload);
 203                 return 0;
 204         }
 205
 206         status = (struct spdk_nvme_reservation_status_data *)payload;
 207         fprintf(stdout, "Reservation Generation Counter                  %u\n", status->gen);
 208         fprintf(stdout, "Reservation type                                %u\n", status->rtype);
 209         fprintf(stdout, "Reservation Number of Registered Controllers    %u\n", status->regctl);
 210         fprintf(stdout, "Reservation Persist Through Power Loss State    %u\n", status->ptpls);
 211         for (i = 0; i < status->regctl; i++) {
 212                 cdata = (struct spdk_nvme_registered_ctrlr_data *)(payload +
 213                                 sizeof(struct spdk_nvme_reservation_status_data) +
 214                                 sizeof(struct spdk_nvme_registered_ctrlr_data) * i);
 215                 fprintf(stdout, "Controller ID                           %u\n", cdata->cntlid);
 216                 fprintf(stdout, "Controller Reservation Status           %u\n", cdata->rcsts.status);
 217                 fprintf(stdout, "Controller Host ID                      0x%"PRIx64"\n", cdata->hostid);
 218                 fprintf(stdout, "Controller Reservation Key              0x%"PRIx64"\n", cdata->rkey);
 219         }
 220
 221         spdk_dma_free(payload);
 222         return 0;
 223 }
 224
 225 static int
 226 reservation_ns_acquire(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, uint32_t ns_id)
 227 {
 228         int ret;
 229         struct spdk_nvme_reservation_acquire_data cdata;
 230         struct spdk_nvme_ns *ns;
 231
 232         ns = spdk_nvme_ctrlr_get_ns(ctrlr, ns_id);
 233         cdata.crkey = CR_KEY;
 234         cdata.prkey = 0;
 235
 236         outstanding_commands = 0;
 237         reserve_command_result = -1;
 238
 239         ret = spdk_nvme_ns_cmd_reservation_acquire(ns, qpair, &cdata,
 240                         false,
 241                         SPDK_NVME_RESERVE_ACQUIRE,
 242                         SPDK_NVME_RESERVE_WRITE_EXCLUSIVE,
 243                         reservation_ns_completion, NULL);
 244         if (ret) {
 245                 fprintf(stderr, "Reservation Acquire Failed\n");
 246                 return -1;
 247         }
 248
 249         outstanding_commands++;
 250         while (outstanding_commands) {
 251                 spdk_nvme_qpair_process_completions(qpair, 100);
 252         }
 253
 254         if (reserve_command_result) {
 255                 fprintf(stderr, "Reservation Acquire Failed\n");
 256         }
 257
 258         return 0;
 259 }
 260
 261 static int
 262 reservation_ns_release(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, uint32_t ns_id)
 263 {
 264         int ret;
 265         struct spdk_nvme_reservation_key_data cdata;
 266         struct spdk_nvme_ns *ns;
 267
 268         ns = spdk_nvme_ctrlr_get_ns(ctrlr, ns_id);
 269         cdata.crkey = CR_KEY;
 270
 271         outstanding_commands = 0;
 272         reserve_command_result = -1;
 273
 274         ret = spdk_nvme_ns_cmd_reservation_release(ns, qpair, &cdata,
 275                         false,
 276                         SPDK_NVME_RESERVE_RELEASE,
 277                         SPDK_NVME_RESERVE_WRITE_EXCLUSIVE,
 278                         reservation_ns_completion, NULL);
 279         if (ret) {
 280                 fprintf(stderr, "Reservation Release Failed\n");
 281                 return -1;
 282         }
 283
 284         outstanding_commands++;
 285         while (outstanding_commands) {
 286                 spdk_nvme_qpair_process_completions(qpair, 100);
 287         }
 288
 289         if (reserve_command_result) {
 290                 fprintf(stderr, "Reservation Release Failed\n");
 291         }
 292
 293         return 0;
 294 }
 295
 296 static void
 297 reserve_controller(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
 298                    const struct spdk_pci_addr *pci_addr)
 299 {
 300         const struct spdk_nvme_ctrlr_data       *cdata;
 301
 302         cdata = spdk_nvme_ctrlr_get_data(ctrlr);
 303
 304         printf("=====================================================\n");
 305         printf("NVMe Controller at PCI bus %d, device %d, function %d\n",
 306                pci_addr->bus, pci_addr->dev, pci_addr->func);
 307         printf("=====================================================\n");
 308
 309         printf("Reservations:                %s\n",
 310                cdata->oncs.reservations ? "Supported" : "Not Supported");
 311
 312         if (!cdata->oncs.reservations) {
 313                 return;
 314         }
 315
 316         get_host_identifier(ctrlr);
 317
 318         /* tested 1 namespace */
 319         reservation_ns_register(ctrlr, qpair, 1);
 320         reservation_ns_acquire(ctrlr, qpair, 1);
 321         reservation_ns_report(ctrlr, qpair, 1);
 322         reservation_ns_release(ctrlr, qpair, 1);
 323 }
 324
 325 static bool
 326 probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
 327          struct spdk_nvme_ctrlr_opts *opts)
 328 {
 329         /*
 330          * Provide both 64-bit and 128-bit host identifiers.
 331          *
 332          * The NVMe library will choose which one to use based on whether the controller
 333          * supports extended host identifiers.
 334          */
 335         to_le64(opts->host_id, HOST_ID);
 336         memcpy(opts->extended_host_id, EXT_HOST_ID, sizeof(opts->extended_host_id));
 337
 338         return true;
 339 }
 340
 341 static void
 342 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
 343           struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
 344 {
 345         struct dev *dev;
 346
 347         /* add to dev list */
 348         dev = &devs[num_devs++];
 349         spdk_pci_addr_parse(&dev->pci_addr, trid->traddr);
 350         dev->ctrlr = ctrlr;
 351 }
 352
 353 int main(int argc, char **argv)
 354 {
 355         struct dev              *iter;
 356         int                     rc, i;
 357         struct spdk_env_opts    opts;
 358
 359         spdk_env_opts_init(&opts);
 360         opts.name = "reserve";
 361         opts.core_mask = "0x1";
 362         opts.shm_id = 0;
 363         if (spdk_env_init(&opts) < 0) {
 364                 fprintf(stderr, "Unable to initialize SPDK env\n");
 365                 return 1;
 366         }
 367
 368         if (spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, NULL) != 0) {
 369                 fprintf(stderr, "spdk_nvme_probe() failed\n");
 370                 return 1;
 371         }
 372
 373         rc = 0;
 374
 375         foreach_dev(iter) {
 376                 struct spdk_nvme_qpair *qpair;
 377
 378                 qpair = spdk_nvme_ctrlr_alloc_io_qpair(iter->ctrlr, NULL, 0);
 379                 if (!qpair) {
 380                         fprintf(stderr, "spdk_nvme_ctrlr_alloc_io_qpair() failed\n");
 381                         rc = 1;
 382                 } else {
 383                         reserve_controller(iter->ctrlr, qpair, &iter->pci_addr);
 384                 }
 385         }
 386
 387         printf("Cleaning up...\n");
 388
 389         for (i = 0; i < num_devs; i++) {
 390                 struct dev *dev = &devs[i];
 391                 spdk_nvme_detach(dev->ctrlr);
 392         }
 393
 394         return rc;
 395 }