ceph/src/spdk/examples/nvme/reserve/reserve.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (c) Intel Corporation.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdbool.h>
  35 #include <stdio.h>
  36 #include <stdlib.h>
  37 #include <unistd.h>
  38 #include <inttypes.h>
  39
  40 #include "spdk/nvme.h"
  41 #include "spdk/env.h"
  42
  43 #define MAX_DEVS 64
  44
  45 struct dev {
  46         struct spdk_pci_addr                    pci_addr;
  47         struct spdk_nvme_ctrlr                  *ctrlr;
  48         char                                    name[100];
  49 };
  50
  51 static struct dev devs[MAX_DEVS];
  52 static int num_devs = 0;
  53
  54 #define foreach_dev(iter) \
  55         for (iter = devs; iter - devs < num_devs; iter++)
  56
  57 static int outstanding_commands;
  58 static int reserve_command_result;
  59 static int set_feature_result;
  60
  61 struct feature {
  62         uint32_t result;
  63         bool valid;
  64 };
  65
  66 static struct feature features[256];
  67
  68 #define HOST_ID         0xABABABABCDCDCDCD
  69 #define CR_KEY          0xDEADBEAF5A5A5A5B
  70
  71 static void
  72 get_feature_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
  73 {
  74         struct feature *feature = cb_arg;
  75         int fid = feature - features;
  76
  77         if (spdk_nvme_cpl_is_error(cpl)) {
  78                 fprintf(stdout, "get_feature(0x%02X) failed\n", fid);
  79         } else {
  80                 feature->result = cpl->cdw0;
  81                 feature->valid = true;
  82         }
  83         outstanding_commands--;
  84 }
  85
  86 static void
  87 set_feature_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
  88 {
  89         struct feature *feature = cb_arg;
  90         int fid = feature - features;
  91
  92         if (spdk_nvme_cpl_is_error(cpl)) {
  93                 fprintf(stdout, "set_feature(0x%02X) failed\n", fid);
  94                 set_feature_result = -1;
  95         } else {
  96                 set_feature_result = 0;
  97         }
  98         outstanding_commands--;
  99 }
 100
 101 static int
 102 get_host_identifier(struct spdk_nvme_ctrlr *ctrlr)
 103 {
 104         int ret;
 105         uint64_t host_id;
 106         struct spdk_nvme_cmd cmd = {};
 107
 108         cmd.opc = SPDK_NVME_OPC_GET_FEATURES;
 109         cmd.cdw10 = SPDK_NVME_FEAT_HOST_IDENTIFIER;
 110
 111         outstanding_commands = 0;
 112         ret = spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, &host_id, sizeof(host_id),
 113                                             get_feature_completion, &features[SPDK_NVME_FEAT_HOST_IDENTIFIER]);
 114         if (ret) {
 115                 fprintf(stdout, "Get Feature: Failed\n");
 116                 return -1;
 117         }
 118
 119         outstanding_commands++;
 120
 121         while (outstanding_commands) {
 122                 spdk_nvme_ctrlr_process_admin_completions(ctrlr);
 123         }
 124
 125         if (features[SPDK_NVME_FEAT_HOST_IDENTIFIER].valid) {
 126                 fprintf(stdout, "Get Feature: Host Identifier 0x%" PRIx64 "\n", host_id);
 127         }
 128
 129         return 0;
 130 }
 131
 132 static int
 133 set_host_identifier(struct spdk_nvme_ctrlr *ctrlr)
 134 {
 135         int ret;
 136         uint64_t host_id;
 137         struct spdk_nvme_cmd cmd = {};
 138
 139         cmd.opc = SPDK_NVME_OPC_SET_FEATURES;
 140         cmd.cdw10 = SPDK_NVME_FEAT_HOST_IDENTIFIER;
 141
 142         host_id = HOST_ID;
 143
 144         outstanding_commands = 0;
 145         set_feature_result = -1;
 146
 147         fprintf(stdout, "Set Feature: Host Identifier 0x%" PRIx64 "\n", host_id);
 148         ret = spdk_nvme_ctrlr_cmd_admin_raw(ctrlr, &cmd, &host_id, sizeof(host_id),
 149                                             set_feature_completion, &features[SPDK_NVME_FEAT_HOST_IDENTIFIER]);
 150         if (ret) {
 151                 fprintf(stdout, "Set Feature: Failed\n");
 152                 return -1;
 153         }
 154
 155         outstanding_commands++;
 156
 157         while (outstanding_commands) {
 158                 spdk_nvme_ctrlr_process_admin_completions(ctrlr);
 159         }
 160
 161         if (set_feature_result)
 162                 fprintf(stdout, "Set Feature: Host Identifier Failed\n");
 163
 164         return 0;
 165 }
 166
 167 static void
 168 reservation_ns_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
 169 {
 170         if (spdk_nvme_cpl_is_error(cpl)) {
 171                 reserve_command_result = -1;
 172         } else {
 173                 reserve_command_result = 0;
 174         }
 175
 176         outstanding_commands--;
 177 }
 178
 179 static int
 180 reservation_ns_register(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
 181                         uint16_t ns_id)
 182 {
 183         int ret;
 184         struct spdk_nvme_reservation_register_data rr_data;
 185         struct spdk_nvme_ns *ns;
 186
 187         ns = spdk_nvme_ctrlr_get_ns(ctrlr, ns_id);
 188
 189         rr_data.crkey = CR_KEY;
 190         rr_data.nrkey = CR_KEY;
 191
 192         outstanding_commands = 0;
 193         reserve_command_result = -1;
 194
 195         ret = spdk_nvme_ns_cmd_reservation_register(ns, qpair, &rr_data, true,
 196                         SPDK_NVME_RESERVE_REGISTER_KEY,
 197                         SPDK_NVME_RESERVE_PTPL_NO_CHANGES,
 198                         reservation_ns_completion, NULL);
 199         if (ret) {
 200                 fprintf(stderr, "Reservation Register Failed\n");
 201                 return -1;
 202         }
 203
 204         outstanding_commands++;
 205         while (outstanding_commands) {
 206                 spdk_nvme_qpair_process_completions(qpair, 100);
 207         }
 208
 209         if (reserve_command_result)
 210                 fprintf(stderr, "Reservation Register Failed\n");
 211
 212         return 0;
 213 }
 214
 215 static int
 216 reservation_ns_report(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, uint16_t ns_id)
 217 {
 218         int ret, i;
 219         uint8_t payload[0x1000];
 220         struct spdk_nvme_reservation_status_data *status;
 221         struct spdk_nvme_reservation_ctrlr_data *cdata;
 222         struct spdk_nvme_ns *ns;
 223
 224         ns = spdk_nvme_ctrlr_get_ns(ctrlr, ns_id);
 225
 226         outstanding_commands = 0;
 227         reserve_command_result = -1;
 228
 229         ret = spdk_nvme_ns_cmd_reservation_report(ns, qpair, payload, 0x1000,
 230                         reservation_ns_completion, NULL);
 231         if (ret) {
 232                 fprintf(stderr, "Reservation Report Failed\n");
 233                 return -1;
 234         }
 235
 236         outstanding_commands++;
 237         while (outstanding_commands) {
 238                 spdk_nvme_qpair_process_completions(qpair, 100);
 239         }
 240
 241         if (reserve_command_result) {
 242                 fprintf(stderr, "Reservation Report Failed\n");
 243                 return 0;
 244         }
 245
 246         status = (struct spdk_nvme_reservation_status_data *)payload;
 247         fprintf(stdout, "Reservation Generation Counter                  %u\n", status->generation);
 248         fprintf(stdout, "Reservation type                                %u\n", status->type);
 249         fprintf(stdout, "Reservation Number of Registered Controllers    %u\n", status->nr_regctl);
 250         fprintf(stdout, "Reservation Persist Through Power Loss State    %u\n", status->ptpl_state);
 251         for (i = 0; i < status->nr_regctl; i++) {
 252                 cdata = (struct spdk_nvme_reservation_ctrlr_data *)(payload + sizeof(struct
 253                                 spdk_nvme_reservation_status_data) * (i + 1));
 254                 fprintf(stdout, "Controller ID                           %u\n", cdata->ctrlr_id);
 255                 fprintf(stdout, "Controller Reservation Status           %u\n", cdata->rcsts.status);
 256                 fprintf(stdout, "Controller Host ID                      0x%"PRIx64"\n", cdata->host_id);
 257                 fprintf(stdout, "Controller Reservation Key              0x%"PRIx64"\n", cdata->key);
 258         }
 259
 260         return 0;
 261 }
 262
 263 static int
 264 reservation_ns_acquire(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, uint16_t ns_id)
 265 {
 266         int ret;
 267         struct spdk_nvme_reservation_acquire_data cdata;
 268         struct spdk_nvme_ns *ns;
 269
 270         ns = spdk_nvme_ctrlr_get_ns(ctrlr, ns_id);
 271         cdata.crkey = CR_KEY;
 272         cdata.prkey = 0;
 273
 274         outstanding_commands = 0;
 275         reserve_command_result = -1;
 276
 277         ret = spdk_nvme_ns_cmd_reservation_acquire(ns, qpair, &cdata,
 278                         false,
 279                         SPDK_NVME_RESERVE_ACQUIRE,
 280                         SPDK_NVME_RESERVE_WRITE_EXCLUSIVE,
 281                         reservation_ns_completion, NULL);
 282         if (ret) {
 283                 fprintf(stderr, "Reservation Acquire Failed\n");
 284                 return -1;
 285         }
 286
 287         outstanding_commands++;
 288         while (outstanding_commands) {
 289                 spdk_nvme_qpair_process_completions(qpair, 100);
 290         }
 291
 292         if (reserve_command_result)
 293                 fprintf(stderr, "Reservation Acquire Failed\n");
 294
 295         return 0;
 296 }
 297
 298 static int
 299 reservation_ns_release(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair, uint16_t ns_id)
 300 {
 301         int ret;
 302         struct spdk_nvme_reservation_key_data cdata;
 303         struct spdk_nvme_ns *ns;
 304
 305         ns = spdk_nvme_ctrlr_get_ns(ctrlr, ns_id);
 306         cdata.crkey = CR_KEY;
 307
 308         outstanding_commands = 0;
 309         reserve_command_result = -1;
 310
 311         ret = spdk_nvme_ns_cmd_reservation_release(ns, qpair, &cdata,
 312                         false,
 313                         SPDK_NVME_RESERVE_RELEASE,
 314                         SPDK_NVME_RESERVE_WRITE_EXCLUSIVE,
 315                         reservation_ns_completion, NULL);
 316         if (ret) {
 317                 fprintf(stderr, "Reservation Release Failed\n");
 318                 return -1;
 319         }
 320
 321         outstanding_commands++;
 322         while (outstanding_commands) {
 323                 spdk_nvme_qpair_process_completions(qpair, 100);
 324         }
 325
 326         if (reserve_command_result)
 327                 fprintf(stderr, "Reservation Release Failed\n");
 328
 329         return 0;
 330 }
 331
 332 static void
 333 reserve_controller(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_qpair *qpair,
 334                    const struct spdk_pci_addr *pci_addr)
 335 {
 336         const struct spdk_nvme_ctrlr_data       *cdata;
 337
 338         cdata = spdk_nvme_ctrlr_get_data(ctrlr);
 339
 340         printf("=====================================================\n");
 341         printf("NVMe Controller at PCI bus %d, device %d, function %d\n",
 342                pci_addr->bus, pci_addr->dev, pci_addr->func);
 343         printf("=====================================================\n");
 344
 345         printf("Reservations:                %s\n",
 346                cdata->oncs.reservations ? "Supported" : "Not Supported");
 347
 348         if (!cdata->oncs.reservations)
 349                 return;
 350
 351         set_host_identifier(ctrlr);
 352         get_host_identifier(ctrlr);
 353
 354         /* tested 1 namespace */
 355         reservation_ns_register(ctrlr, qpair, 1);
 356         reservation_ns_acquire(ctrlr, qpair, 1);
 357         reservation_ns_report(ctrlr, qpair, 1);
 358         reservation_ns_release(ctrlr, qpair, 1);
 359 }
 360
 361 static bool
 362 probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
 363          struct spdk_nvme_ctrlr_opts *opts)
 364 {
 365         return true;
 366 }
 367
 368 static void
 369 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
 370           struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
 371 {
 372         struct dev *dev;
 373
 374         /* add to dev list */
 375         dev = &devs[num_devs++];
 376         spdk_pci_addr_parse(&dev->pci_addr, trid->traddr);
 377         dev->ctrlr = ctrlr;
 378 }
 379
 380 int main(int argc, char **argv)
 381 {
 382         struct dev              *iter;
 383         int                     rc, i;
 384         struct spdk_env_opts    opts;
 385
 386         spdk_env_opts_init(&opts);
 387         opts.name = "reserve";
 388         opts.core_mask = "0x1";
 389         spdk_env_init(&opts);
 390
 391         if (spdk_nvme_probe(NULL, NULL, probe_cb, attach_cb, NULL) != 0) {
 392                 fprintf(stderr, "spdk_nvme_probe() failed\n");
 393                 return 1;
 394         }
 395
 396         rc = 0;
 397
 398         foreach_dev(iter) {
 399                 struct spdk_nvme_qpair *qpair;
 400
 401                 qpair = spdk_nvme_ctrlr_alloc_io_qpair(iter->ctrlr, 0);
 402                 if (!qpair) {
 403                         fprintf(stderr, "spdk_nvme_ctrlr_alloc_io_qpair() failed\n");
 404                         rc = 1;
 405                 } else {
 406                         reserve_controller(iter->ctrlr, qpair, &iter->pci_addr);
 407                 }
 408         }
 409
 410         printf("Cleaning up...\n");
 411
 412         for (i = 0; i < num_devs; i++) {
 413                 struct dev *dev = &devs[i];
 414                 spdk_nvme_detach(dev->ctrlr);
 415         }
 416
 417         return rc;
 418 }