ceph/src/spdk/dpdk/drivers/net/netvsc/hn_nvs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2018 Microsoft Corp.
   3  * Copyright (c) 2010-2012 Citrix Inc.
   4  * Copyright (c) 2012 NetApp Inc.
   5  * All rights reserved.
   6  */
   7
   8 /*
   9  * Network Virtualization Service.
  10  */
  11
  12
  13 #include <stdint.h>
  14 #include <string.h>
  15 #include <stdio.h>
  16 #include <errno.h>
  17 #include <unistd.h>
  18
  19 #include <rte_ethdev.h>
  20 #include <rte_string_fns.h>
  21 #include <rte_memzone.h>
  22 #include <rte_malloc.h>
  23 #include <rte_atomic.h>
  24 #include <rte_branch_prediction.h>
  25 #include <rte_ether.h>
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_cycles.h>
  29 #include <rte_memory.h>
  30 #include <rte_eal.h>
  31 #include <rte_dev.h>
  32 #include <rte_bus_vmbus.h>
  33
  34 #include "hn_logs.h"
  35 #include "hn_var.h"
  36 #include "hn_nvs.h"
  37
  38 static const uint32_t hn_nvs_version[] = {
  39         NVS_VERSION_61,
  40         NVS_VERSION_6,
  41         NVS_VERSION_5,
  42         NVS_VERSION_4,
  43         NVS_VERSION_2,
  44         NVS_VERSION_1
  45 };
  46
  47 static int hn_nvs_req_send(struct hn_data *hv,
  48                            void *req, uint32_t reqlen)
  49 {
  50         return rte_vmbus_chan_send(hn_primary_chan(hv),
  51                                    VMBUS_CHANPKT_TYPE_INBAND,
  52                                    req, reqlen, 0,
  53                                    VMBUS_CHANPKT_FLAG_NONE, NULL);
  54 }
  55
  56 static int
  57 hn_nvs_execute(struct hn_data *hv,
  58                void *req, uint32_t reqlen,
  59                void *resp, uint32_t resplen,
  60                uint32_t type)
  61 {
  62         struct vmbus_channel *chan = hn_primary_chan(hv);
  63         char buffer[NVS_RESPSIZE_MAX];
  64         const struct hn_nvs_hdr *hdr;
  65         uint32_t len;
  66         int ret;
  67
  68         /* Send request to ring buffer */
  69         ret = rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_INBAND,
  70                                   req, reqlen, 0,
  71                                   VMBUS_CHANPKT_FLAG_RC, NULL);
  72
  73         if (ret) {
  74                 PMD_DRV_LOG(ERR, "send request failed: %d", ret);
  75                 return ret;
  76         }
  77
  78  retry:
  79         len = sizeof(buffer);
  80         ret = rte_vmbus_chan_recv(chan, buffer, &len, NULL);
  81         if (ret == -EAGAIN) {
  82                 rte_delay_us(HN_CHAN_INTERVAL_US);
  83                 goto retry;
  84         }
  85
  86         if (ret < 0) {
  87                 PMD_DRV_LOG(ERR, "recv response failed: %d", ret);
  88                 return ret;
  89         }
  90
  91         hdr = (struct hn_nvs_hdr *)buffer;
  92         if (hdr->type != type) {
  93                 PMD_DRV_LOG(ERR, "unexpected NVS resp %#x, expect %#x",
  94                             hdr->type, type);
  95                 return -EINVAL;
  96         }
  97
  98         if (len < resplen) {
  99                 PMD_DRV_LOG(ERR,
 100                             "invalid NVS resp len %u (expect %u)",
 101                             len, resplen);
 102                 return -EINVAL;
 103         }
 104
 105         memcpy(resp, buffer, resplen);
 106
 107         /* All pass! */
 108         return 0;
 109 }
 110
 111 static int
 112 hn_nvs_doinit(struct hn_data *hv, uint32_t nvs_ver)
 113 {
 114         struct hn_nvs_init init;
 115         struct hn_nvs_init_resp resp;
 116         uint32_t status;
 117         int error;
 118
 119         memset(&init, 0, sizeof(init));
 120         init.type = NVS_TYPE_INIT;
 121         init.ver_min = nvs_ver;
 122         init.ver_max = nvs_ver;
 123
 124         error = hn_nvs_execute(hv, &init, sizeof(init),
 125                                &resp, sizeof(resp),
 126                                NVS_TYPE_INIT_RESP);
 127         if (error)
 128                 return error;
 129
 130         status = resp.status;
 131         if (status != NVS_STATUS_OK) {
 132                 /* Not fatal, try other versions */
 133                 PMD_INIT_LOG(DEBUG, "nvs init failed for ver 0x%x",
 134                              nvs_ver);
 135                 return -EINVAL;
 136         }
 137
 138         return 0;
 139 }
 140
 141 static int
 142 hn_nvs_conn_rxbuf(struct hn_data *hv)
 143 {
 144         struct hn_nvs_rxbuf_conn conn;
 145         struct hn_nvs_rxbuf_connresp resp;
 146         uint32_t status;
 147         int error;
 148
 149         /* Kernel has already setup RXBUF on primary channel. */
 150
 151         /*
 152          * Connect RXBUF to NVS.
 153          */
 154         conn.type = NVS_TYPE_RXBUF_CONN;
 155         conn.gpadl = hv->rxbuf_res->phys_addr;
 156         conn.sig = NVS_RXBUF_SIG;
 157         PMD_DRV_LOG(DEBUG, "connect rxbuff va=%p gpad=%#" PRIx64,
 158                     hv->rxbuf_res->addr,
 159                     hv->rxbuf_res->phys_addr);
 160
 161         error = hn_nvs_execute(hv, &conn, sizeof(conn),
 162                                &resp, sizeof(resp),
 163                                NVS_TYPE_RXBUF_CONNRESP);
 164         if (error) {
 165                 PMD_DRV_LOG(ERR,
 166                             "exec nvs rxbuf conn failed: %d",
 167                             error);
 168                 return error;
 169         }
 170
 171         status = resp.status;
 172         if (status != NVS_STATUS_OK) {
 173                 PMD_DRV_LOG(ERR,
 174                             "nvs rxbuf conn failed: %x", status);
 175                 return -EIO;
 176         }
 177         if (resp.nsect != 1) {
 178                 PMD_DRV_LOG(ERR,
 179                             "nvs rxbuf response num sections %u != 1",
 180                             resp.nsect);
 181                 return -EIO;
 182         }
 183
 184         PMD_DRV_LOG(INFO,
 185                     "receive buffer size %u count %u",
 186                     resp.nvs_sect[0].slotsz,
 187                     resp.nvs_sect[0].slotcnt);
 188         hv->rxbuf_section_cnt = resp.nvs_sect[0].slotcnt;
 189
 190         hv->rxbuf_info = rte_calloc("HN_RXBUF_INFO", hv->rxbuf_section_cnt,
 191                                     sizeof(*hv->rxbuf_info), RTE_CACHE_LINE_SIZE);
 192         if (!hv->rxbuf_info) {
 193                 PMD_DRV_LOG(ERR,
 194                             "could not allocate rxbuf info");
 195                 return -ENOMEM;
 196         }
 197
 198         return 0;
 199 }
 200
 201 static void
 202 hn_nvs_disconn_rxbuf(struct hn_data *hv)
 203 {
 204         struct hn_nvs_rxbuf_disconn disconn;
 205         int error;
 206
 207         /*
 208          * Disconnect RXBUF from NVS.
 209          */
 210         memset(&disconn, 0, sizeof(disconn));
 211         disconn.type = NVS_TYPE_RXBUF_DISCONN;
 212         disconn.sig = NVS_RXBUF_SIG;
 213
 214         /* NOTE: No response. */
 215         error = hn_nvs_req_send(hv, &disconn, sizeof(disconn));
 216         if (error) {
 217                 PMD_DRV_LOG(ERR,
 218                             "send nvs rxbuf disconn failed: %d",
 219                             error);
 220         }
 221
 222         rte_free(hv->rxbuf_info);
 223         /*
 224          * Linger long enough for NVS to disconnect RXBUF.
 225          */
 226         rte_delay_ms(200);
 227 }
 228
 229 static void
 230 hn_nvs_disconn_chim(struct hn_data *hv)
 231 {
 232         int error;
 233
 234         if (hv->chim_cnt != 0) {
 235                 struct hn_nvs_chim_disconn disconn;
 236
 237                 /* Disconnect chimney sending buffer from NVS. */
 238                 memset(&disconn, 0, sizeof(disconn));
 239                 disconn.type = NVS_TYPE_CHIM_DISCONN;
 240                 disconn.sig = NVS_CHIM_SIG;
 241
 242                 /* NOTE: No response. */
 243                 error = hn_nvs_req_send(hv, &disconn, sizeof(disconn));
 244
 245                 if (error) {
 246                         PMD_DRV_LOG(ERR,
 247                                     "send nvs chim disconn failed: %d", error);
 248                 }
 249
 250                 hv->chim_cnt = 0;
 251                 /*
 252                  * Linger long enough for NVS to disconnect chimney
 253                  * sending buffer.
 254                  */
 255                 rte_delay_ms(200);
 256         }
 257 }
 258
 259 static int
 260 hn_nvs_conn_chim(struct hn_data *hv)
 261 {
 262         struct hn_nvs_chim_conn chim;
 263         struct hn_nvs_chim_connresp resp;
 264         uint32_t sectsz;
 265         unsigned long len = hv->chim_res->len;
 266         int error;
 267
 268         /* Connect chimney sending buffer to NVS */
 269         memset(&chim, 0, sizeof(chim));
 270         chim.type = NVS_TYPE_CHIM_CONN;
 271         chim.gpadl = hv->chim_res->phys_addr;
 272         chim.sig = NVS_CHIM_SIG;
 273         PMD_DRV_LOG(DEBUG, "connect send buf va=%p gpad=%#" PRIx64,
 274                     hv->chim_res->addr,
 275                     hv->chim_res->phys_addr);
 276
 277         error = hn_nvs_execute(hv, &chim, sizeof(chim),
 278                                &resp, sizeof(resp),
 279                                NVS_TYPE_CHIM_CONNRESP);
 280         if (error) {
 281                 PMD_DRV_LOG(ERR, "exec nvs chim conn failed");
 282                 goto cleanup;
 283         }
 284
 285         if (resp.status != NVS_STATUS_OK) {
 286                 PMD_DRV_LOG(ERR, "nvs chim conn failed: %x",
 287                             resp.status);
 288                 error = -EIO;
 289                 goto cleanup;
 290         }
 291
 292         sectsz = resp.sectsz;
 293         if (sectsz == 0 || sectsz & (sizeof(uint32_t) - 1)) {
 294                 /* Can't use chimney sending buffer; done! */
 295                 PMD_DRV_LOG(NOTICE,
 296                             "invalid chimney sending buffer section size: %u",
 297                             sectsz);
 298                 return 0;
 299         }
 300
 301         hv->chim_szmax = sectsz;
 302         hv->chim_cnt = len / sectsz;
 303
 304         PMD_DRV_LOG(INFO, "send buffer %lu section size:%u, count:%u",
 305                     len, hv->chim_szmax, hv->chim_cnt);
 306
 307         if (len % hv->chim_szmax != 0) {
 308                 PMD_DRV_LOG(NOTICE,
 309                             "chimney sending sections are not properly aligned");
 310         }
 311
 312         /* Done! */
 313         return 0;
 314
 315 cleanup:
 316         hn_nvs_disconn_chim(hv);
 317         return error;
 318 }
 319
 320 /*
 321  * Configure MTU and enable VLAN.
 322  */
 323 static int
 324 hn_nvs_conf_ndis(struct hn_data *hv, unsigned int mtu)
 325 {
 326         struct hn_nvs_ndis_conf conf;
 327         int error;
 328
 329         memset(&conf, 0, sizeof(conf));
 330         conf.type = NVS_TYPE_NDIS_CONF;
 331         conf.mtu = mtu + ETHER_HDR_LEN;
 332         conf.caps = NVS_NDIS_CONF_VLAN;
 333
 334         /* TODO enable SRIOV */
 335         //if (hv->nvs_ver >= NVS_VERSION_5)
 336         //      conf.caps |= NVS_NDIS_CONF_SRIOV;
 337
 338         /* NOTE: No response. */
 339         error = hn_nvs_req_send(hv, &conf, sizeof(conf));
 340         if (error) {
 341                 PMD_DRV_LOG(ERR,
 342                             "send nvs ndis conf failed: %d", error);
 343                 return error;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static int
 350 hn_nvs_init_ndis(struct hn_data *hv)
 351 {
 352         struct hn_nvs_ndis_init ndis;
 353         int error;
 354
 355         memset(&ndis, 0, sizeof(ndis));
 356         ndis.type = NVS_TYPE_NDIS_INIT;
 357         ndis.ndis_major = NDIS_VERSION_MAJOR(hv->ndis_ver);
 358         ndis.ndis_minor = NDIS_VERSION_MINOR(hv->ndis_ver);
 359
 360         /* NOTE: No response. */
 361         error = hn_nvs_req_send(hv, &ndis, sizeof(ndis));
 362         if (error)
 363                 PMD_DRV_LOG(ERR,
 364                             "send nvs ndis init failed: %d", error);
 365
 366         return error;
 367 }
 368
 369 static int
 370 hn_nvs_init(struct hn_data *hv)
 371 {
 372         unsigned int i;
 373         int error;
 374
 375         /*
 376          * Find the supported NVS version and set NDIS version accordingly.
 377          */
 378         for (i = 0; i < RTE_DIM(hn_nvs_version); ++i) {
 379                 error = hn_nvs_doinit(hv, hn_nvs_version[i]);
 380                 if (error) {
 381                         PMD_INIT_LOG(DEBUG, "version %#x error %d",
 382                                      hn_nvs_version[i], error);
 383                         continue;
 384                 }
 385
 386                 hv->nvs_ver = hn_nvs_version[i];
 387
 388                 /* Set NDIS version according to NVS version. */
 389                 hv->ndis_ver = NDIS_VERSION_6_30;
 390                 if (hv->nvs_ver <= NVS_VERSION_4)
 391                         hv->ndis_ver = NDIS_VERSION_6_1;
 392
 393                 PMD_INIT_LOG(DEBUG,
 394                              "NVS version %#x, NDIS version %u.%u",
 395                              hv->nvs_ver, NDIS_VERSION_MAJOR(hv->ndis_ver),
 396                              NDIS_VERSION_MINOR(hv->ndis_ver));
 397                 return 0;
 398         }
 399
 400         PMD_DRV_LOG(ERR,
 401                     "no NVS compatible version available");
 402         return -ENXIO;
 403 }
 404
 405 int
 406 hn_nvs_attach(struct hn_data *hv, unsigned int mtu)
 407 {
 408         int error;
 409
 410         /*
 411          * Initialize NVS.
 412          */
 413         error = hn_nvs_init(hv);
 414         if (error)
 415                 return error;
 416
 417         /** Configure NDIS before initializing it. */
 418         if (hv->nvs_ver >= NVS_VERSION_2) {
 419                 error = hn_nvs_conf_ndis(hv, mtu);
 420                 if (error)
 421                         return error;
 422         }
 423
 424         /*
 425          * Initialize NDIS.
 426          */
 427         error = hn_nvs_init_ndis(hv);
 428         if (error)
 429                 return error;
 430
 431         /*
 432          * Connect RXBUF.
 433          */
 434         error = hn_nvs_conn_rxbuf(hv);
 435         if (error)
 436                 return error;
 437
 438         /*
 439          * Connect chimney sending buffer.
 440          */
 441         error = hn_nvs_conn_chim(hv);
 442         if (error) {
 443                 hn_nvs_disconn_rxbuf(hv);
 444                 return error;
 445         }
 446
 447         return 0;
 448 }
 449
 450 void
 451 hn_nvs_detach(struct hn_data *hv __rte_unused)
 452 {
 453         PMD_INIT_FUNC_TRACE();
 454
 455         /* NOTE: there are no requests to stop the NVS. */
 456         hn_nvs_disconn_rxbuf(hv);
 457         hn_nvs_disconn_chim(hv);
 458 }
 459
 460 /*
 461  * Ack the consumed RXBUF associated w/ this channel packet,
 462  * so that this RXBUF can be recycled by the hypervisor.
 463  */
 464 void
 465 hn_nvs_ack_rxbuf(struct vmbus_channel *chan, uint64_t tid)
 466 {
 467         unsigned int retries = 0;
 468         struct hn_nvs_rndis_ack ack = {
 469                 .type = NVS_TYPE_RNDIS_ACK,
 470                 .status = NVS_STATUS_OK,
 471         };
 472         int error;
 473
 474         PMD_RX_LOG(DEBUG, "ack RX id %" PRIu64, tid);
 475
 476  again:
 477         error = rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_COMP,
 478                                     &ack, sizeof(ack), tid,
 479                                     VMBUS_CHANPKT_FLAG_NONE, NULL);
 480
 481         if (error == 0)
 482                 return;
 483
 484         if (error == -EAGAIN) {
 485                 /*
 486                  * NOTE:
 487                  * This should _not_ happen in real world, since the
 488                  * consumption of the TX bufring from the TX path is
 489                  * controlled.
 490                  */
 491                 PMD_RX_LOG(NOTICE, "RXBUF ack retry");
 492                 if (++retries < 10) {
 493                         rte_delay_ms(1);
 494                         goto again;
 495                 }
 496         }
 497         /* RXBUF leaks! */
 498         PMD_DRV_LOG(ERR, "RXBUF ack failed");
 499 }
 500
 501 int
 502 hn_nvs_alloc_subchans(struct hn_data *hv, uint32_t *nsubch)
 503 {
 504         struct hn_nvs_subch_req req;
 505         struct hn_nvs_subch_resp resp;
 506         int error;
 507
 508         memset(&req, 0, sizeof(req));
 509         req.type = NVS_TYPE_SUBCH_REQ;
 510         req.op = NVS_SUBCH_OP_ALLOC;
 511         req.nsubch = *nsubch;
 512
 513         error = hn_nvs_execute(hv, &req, sizeof(req),
 514                                &resp, sizeof(resp),
 515                                NVS_TYPE_SUBCH_RESP);
 516         if (error)
 517                 return error;
 518
 519         if (resp.status != NVS_STATUS_OK) {
 520                 PMD_INIT_LOG(ERR,
 521                              "nvs subch alloc failed: %#x",
 522                              resp.status);
 523                 return -EIO;
 524         }
 525
 526         if (resp.nsubch > *nsubch) {
 527                 PMD_INIT_LOG(NOTICE,
 528                              "%u subchans are allocated, requested %u",
 529                              resp.nsubch, *nsubch);
 530         }
 531         *nsubch = resp.nsubch;
 532
 533         return 0;
 534 }
 535
 536 void
 537 hn_nvs_set_datapath(struct hn_data *hv, uint32_t path)
 538 {
 539         struct hn_nvs_datapath dp;
 540
 541         memset(&dp, 0, sizeof(dp));
 542         dp.type = NVS_TYPE_SET_DATAPATH;
 543         dp.active_path = path;
 544
 545         hn_nvs_req_send(hv, &dp, sizeof(dp));
 546 }