ceph/src/seastar/dpdk/drivers/net/netvsc/hn_nvs.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2018 Microsoft Corp.
   3  * Copyright (c) 2010-2012 Citrix Inc.
   4  * Copyright (c) 2012 NetApp Inc.
   5  * All rights reserved.
   6  */
   7
   8 /*
   9  * Network Virtualization Service.
  10  */
  11
  12
  13 #include <stdint.h>
  14 #include <string.h>
  15 #include <stdio.h>
  16 #include <errno.h>
  17 #include <unistd.h>
  18
  19 #include <rte_ethdev.h>
  20 #include <rte_string_fns.h>
  21 #include <rte_memzone.h>
  22 #include <rte_malloc.h>
  23 #include <rte_atomic.h>
  24 #include <rte_branch_prediction.h>
  25 #include <rte_ether.h>
  26 #include <rte_common.h>
  27 #include <rte_errno.h>
  28 #include <rte_cycles.h>
  29 #include <rte_memory.h>
  30 #include <rte_eal.h>
  31 #include <rte_dev.h>
  32 #include <rte_bus_vmbus.h>
  33
  34 #include "hn_logs.h"
  35 #include "hn_var.h"
  36 #include "hn_nvs.h"
  37
  38 static const uint32_t hn_nvs_version[] = {
  39         NVS_VERSION_61,
  40         NVS_VERSION_6,
  41         NVS_VERSION_5,
  42         NVS_VERSION_4,
  43         NVS_VERSION_2,
  44         NVS_VERSION_1
  45 };
  46
  47 static int hn_nvs_req_send(struct hn_data *hv,
  48                            void *req, uint32_t reqlen)
  49 {
  50         return rte_vmbus_chan_send(hn_primary_chan(hv),
  51                                    VMBUS_CHANPKT_TYPE_INBAND,
  52                                    req, reqlen, 0,
  53                                    VMBUS_CHANPKT_FLAG_NONE, NULL);
  54 }
  55
  56 static int
  57 hn_nvs_execute(struct hn_data *hv,
  58                void *req, uint32_t reqlen,
  59                void *resp, uint32_t resplen,
  60                uint32_t type)
  61 {
  62         struct vmbus_channel *chan = hn_primary_chan(hv);
  63         char buffer[NVS_RESPSIZE_MAX];
  64         const struct hn_nvs_hdr *hdr;
  65         uint32_t len;
  66         int ret;
  67
  68         /* Send request to ring buffer */
  69         ret = rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_INBAND,
  70                                   req, reqlen, 0,
  71                                   VMBUS_CHANPKT_FLAG_RC, NULL);
  72
  73         if (ret) {
  74                 PMD_DRV_LOG(ERR, "send request failed: %d", ret);
  75                 return ret;
  76         }
  77
  78  retry:
  79         len = sizeof(buffer);
  80         ret = rte_vmbus_chan_recv(chan, buffer, &len, NULL);
  81         if (ret == -EAGAIN) {
  82                 rte_delay_us(HN_CHAN_INTERVAL_US);
  83                 goto retry;
  84         }
  85
  86         if (ret < 0) {
  87                 PMD_DRV_LOG(ERR, "recv response failed: %d", ret);
  88                 return ret;
  89         }
  90
  91         hdr = (struct hn_nvs_hdr *)buffer;
  92         if (hdr->type != type) {
  93                 PMD_DRV_LOG(ERR, "unexpected NVS resp %#x, expect %#x",
  94                             hdr->type, type);
  95                 return -EINVAL;
  96         }
  97
  98         if (len < resplen) {
  99                 PMD_DRV_LOG(ERR,
 100                             "invalid NVS resp len %u (expect %u)",
 101                             len, resplen);
 102                 return -EINVAL;
 103         }
 104
 105         memcpy(resp, buffer, resplen);
 106
 107         /* All pass! */
 108         return 0;
 109 }
 110
 111 static int
 112 hn_nvs_doinit(struct hn_data *hv, uint32_t nvs_ver)
 113 {
 114         struct hn_nvs_init init;
 115         struct hn_nvs_init_resp resp;
 116         uint32_t status;
 117         int error;
 118
 119         memset(&init, 0, sizeof(init));
 120         init.type = NVS_TYPE_INIT;
 121         init.ver_min = nvs_ver;
 122         init.ver_max = nvs_ver;
 123
 124         error = hn_nvs_execute(hv, &init, sizeof(init),
 125                                &resp, sizeof(resp),
 126                                NVS_TYPE_INIT_RESP);
 127         if (error)
 128                 return error;
 129
 130         status = resp.status;
 131         if (status != NVS_STATUS_OK) {
 132                 /* Not fatal, try other versions */
 133                 PMD_INIT_LOG(DEBUG, "nvs init failed for ver 0x%x",
 134                              nvs_ver);
 135                 return -EINVAL;
 136         }
 137
 138         return 0;
 139 }
 140
 141 static int
 142 hn_nvs_conn_rxbuf(struct hn_data *hv)
 143 {
 144         struct hn_nvs_rxbuf_conn conn;
 145         struct hn_nvs_rxbuf_connresp resp;
 146         uint32_t status;
 147         int error;
 148
 149         /* Kernel has already setup RXBUF on primary channel. */
 150
 151         /*
 152          * Connect RXBUF to NVS.
 153          */
 154         conn.type = NVS_TYPE_RXBUF_CONN;
 155         conn.gpadl = hv->rxbuf_res->phys_addr;
 156         conn.sig = NVS_RXBUF_SIG;
 157         PMD_DRV_LOG(DEBUG, "connect rxbuff va=%p gpad=%#" PRIx64,
 158                     hv->rxbuf_res->addr,
 159                     hv->rxbuf_res->phys_addr);
 160
 161         error = hn_nvs_execute(hv, &conn, sizeof(conn),
 162                                &resp, sizeof(resp),
 163                                NVS_TYPE_RXBUF_CONNRESP);
 164         if (error) {
 165                 PMD_DRV_LOG(ERR,
 166                             "exec nvs rxbuf conn failed: %d",
 167                             error);
 168                 return error;
 169         }
 170
 171         status = resp.status;
 172         if (status != NVS_STATUS_OK) {
 173                 PMD_DRV_LOG(ERR,
 174                             "nvs rxbuf conn failed: %x", status);
 175                 return -EIO;
 176         }
 177         if (resp.nsect != 1) {
 178                 PMD_DRV_LOG(ERR,
 179                             "nvs rxbuf response num sections %u != 1",
 180                             resp.nsect);
 181                 return -EIO;
 182         }
 183
 184         PMD_DRV_LOG(INFO,
 185                     "receive buffer size %u count %u",
 186                     resp.nvs_sect[0].slotsz,
 187                     resp.nvs_sect[0].slotcnt);
 188         hv->rxbuf_section_cnt = resp.nvs_sect[0].slotcnt;
 189
 190         hv->rxbuf_info = rte_calloc("HN_RXBUF_INFO", hv->rxbuf_section_cnt,
 191                                     sizeof(*hv->rxbuf_info), RTE_CACHE_LINE_SIZE);
 192         if (!hv->rxbuf_info) {
 193                 PMD_DRV_LOG(ERR,
 194                             "could not allocate rxbuf info");
 195                 return -ENOMEM;
 196         }
 197
 198         return 0;
 199 }
 200
 201 static void
 202 hn_nvs_disconn_rxbuf(struct hn_data *hv)
 203 {
 204         struct hn_nvs_rxbuf_disconn disconn;
 205         int error;
 206
 207         /*
 208          * Disconnect RXBUF from NVS.
 209          */
 210         memset(&disconn, 0, sizeof(disconn));
 211         disconn.type = NVS_TYPE_RXBUF_DISCONN;
 212         disconn.sig = NVS_RXBUF_SIG;
 213
 214         /* NOTE: No response. */
 215         error = hn_nvs_req_send(hv, &disconn, sizeof(disconn));
 216         if (error) {
 217                 PMD_DRV_LOG(ERR,
 218                             "send nvs rxbuf disconn failed: %d",
 219                             error);
 220         }
 221
 222         rte_free(hv->rxbuf_info);
 223         /*
 224          * Linger long enough for NVS to disconnect RXBUF.
 225          */
 226         rte_delay_ms(200);
 227 }
 228
 229 static void
 230 hn_nvs_disconn_chim(struct hn_data *hv)
 231 {
 232         int error;
 233
 234         if (hv->chim_cnt != 0) {
 235                 struct hn_nvs_chim_disconn disconn;
 236
 237                 /* Disconnect chimney sending buffer from NVS. */
 238                 memset(&disconn, 0, sizeof(disconn));
 239                 disconn.type = NVS_TYPE_CHIM_DISCONN;
 240                 disconn.sig = NVS_CHIM_SIG;
 241
 242                 /* NOTE: No response. */
 243                 error = hn_nvs_req_send(hv, &disconn, sizeof(disconn));
 244
 245                 if (error) {
 246                         PMD_DRV_LOG(ERR,
 247                                     "send nvs chim disconn failed: %d", error);
 248                 }
 249
 250                 hv->chim_cnt = 0;
 251                 /*
 252                  * Linger long enough for NVS to disconnect chimney
 253                  * sending buffer.
 254                  */
 255                 rte_delay_ms(200);
 256         }
 257 }
 258
 259 static int
 260 hn_nvs_conn_chim(struct hn_data *hv)
 261 {
 262         struct hn_nvs_chim_conn chim;
 263         struct hn_nvs_chim_connresp resp;
 264         uint32_t sectsz;
 265         unsigned long len = hv->chim_res->len;
 266         int error;
 267
 268         /* Connect chimney sending buffer to NVS */
 269         memset(&chim, 0, sizeof(chim));
 270         chim.type = NVS_TYPE_CHIM_CONN;
 271         chim.gpadl = hv->chim_res->phys_addr;
 272         chim.sig = NVS_CHIM_SIG;
 273         PMD_DRV_LOG(DEBUG, "connect send buf va=%p gpad=%#" PRIx64,
 274                     hv->chim_res->addr,
 275                     hv->chim_res->phys_addr);
 276
 277         error = hn_nvs_execute(hv, &chim, sizeof(chim),
 278                                &resp, sizeof(resp),
 279                                NVS_TYPE_CHIM_CONNRESP);
 280         if (error) {
 281                 PMD_DRV_LOG(ERR, "exec nvs chim conn failed");
 282                 return error;
 283         }
 284
 285         if (resp.status != NVS_STATUS_OK) {
 286                 PMD_DRV_LOG(ERR, "nvs chim conn failed: %x",
 287                             resp.status);
 288                 return -EIO;
 289         }
 290
 291         sectsz = resp.sectsz;
 292         if (sectsz == 0 || sectsz & (sizeof(uint32_t) - 1)) {
 293                 /* Can't use chimney sending buffer; done! */
 294                 PMD_DRV_LOG(NOTICE,
 295                             "invalid chimney sending buffer section size: %u",
 296                             sectsz);
 297                 error = -EINVAL;
 298                 goto cleanup;
 299         }
 300
 301         hv->chim_szmax = sectsz;
 302         hv->chim_cnt = len / sectsz;
 303
 304         PMD_DRV_LOG(INFO, "send buffer %lu section size:%u, count:%u",
 305                     len, hv->chim_szmax, hv->chim_cnt);
 306
 307         /* Done! */
 308         return 0;
 309
 310 cleanup:
 311         hn_nvs_disconn_chim(hv);
 312         return error;
 313 }
 314
 315 /*
 316  * Configure MTU and enable VLAN.
 317  */
 318 static int
 319 hn_nvs_conf_ndis(struct hn_data *hv, unsigned int mtu)
 320 {
 321         struct hn_nvs_ndis_conf conf;
 322         int error;
 323
 324         memset(&conf, 0, sizeof(conf));
 325         conf.type = NVS_TYPE_NDIS_CONF;
 326         conf.mtu = mtu + ETHER_HDR_LEN;
 327         conf.caps = NVS_NDIS_CONF_VLAN;
 328
 329         /* enable SRIOV */
 330         if (hv->nvs_ver >= NVS_VERSION_5)
 331                 conf.caps |= NVS_NDIS_CONF_SRIOV;
 332
 333         /* NOTE: No response. */
 334         error = hn_nvs_req_send(hv, &conf, sizeof(conf));
 335         if (error) {
 336                 PMD_DRV_LOG(ERR,
 337                             "send nvs ndis conf failed: %d", error);
 338                 return error;
 339         }
 340
 341         return 0;
 342 }
 343
 344 static int
 345 hn_nvs_init_ndis(struct hn_data *hv)
 346 {
 347         struct hn_nvs_ndis_init ndis;
 348         int error;
 349
 350         memset(&ndis, 0, sizeof(ndis));
 351         ndis.type = NVS_TYPE_NDIS_INIT;
 352         ndis.ndis_major = NDIS_VERSION_MAJOR(hv->ndis_ver);
 353         ndis.ndis_minor = NDIS_VERSION_MINOR(hv->ndis_ver);
 354
 355         /* NOTE: No response. */
 356         error = hn_nvs_req_send(hv, &ndis, sizeof(ndis));
 357         if (error)
 358                 PMD_DRV_LOG(ERR,
 359                             "send nvs ndis init failed: %d", error);
 360
 361         return error;
 362 }
 363
 364 static int
 365 hn_nvs_init(struct hn_data *hv)
 366 {
 367         unsigned int i;
 368         int error;
 369
 370         /*
 371          * Find the supported NVS version and set NDIS version accordingly.
 372          */
 373         for (i = 0; i < RTE_DIM(hn_nvs_version); ++i) {
 374                 error = hn_nvs_doinit(hv, hn_nvs_version[i]);
 375                 if (error) {
 376                         PMD_INIT_LOG(DEBUG, "version %#x error %d",
 377                                      hn_nvs_version[i], error);
 378                         continue;
 379                 }
 380
 381                 hv->nvs_ver = hn_nvs_version[i];
 382
 383                 /* Set NDIS version according to NVS version. */
 384                 hv->ndis_ver = NDIS_VERSION_6_30;
 385                 if (hv->nvs_ver <= NVS_VERSION_4)
 386                         hv->ndis_ver = NDIS_VERSION_6_1;
 387
 388                 PMD_INIT_LOG(DEBUG,
 389                              "NVS version %#x, NDIS version %u.%u",
 390                              hv->nvs_ver, NDIS_VERSION_MAJOR(hv->ndis_ver),
 391                              NDIS_VERSION_MINOR(hv->ndis_ver));
 392                 return 0;
 393         }
 394
 395         PMD_DRV_LOG(ERR,
 396                     "no NVS compatible version available");
 397         return -ENXIO;
 398 }
 399
 400 int
 401 hn_nvs_attach(struct hn_data *hv, unsigned int mtu)
 402 {
 403         int error;
 404
 405         /*
 406          * Initialize NVS.
 407          */
 408         error = hn_nvs_init(hv);
 409         if (error)
 410                 return error;
 411
 412         /** Configure NDIS before initializing it. */
 413         if (hv->nvs_ver >= NVS_VERSION_2) {
 414                 error = hn_nvs_conf_ndis(hv, mtu);
 415                 if (error)
 416                         return error;
 417         }
 418
 419         /*
 420          * Initialize NDIS.
 421          */
 422         error = hn_nvs_init_ndis(hv);
 423         if (error)
 424                 return error;
 425
 426         /*
 427          * Connect RXBUF.
 428          */
 429         error = hn_nvs_conn_rxbuf(hv);
 430         if (error)
 431                 return error;
 432
 433         /*
 434          * Connect chimney sending buffer.
 435          */
 436         error = hn_nvs_conn_chim(hv);
 437         if (error) {
 438                 hn_nvs_disconn_rxbuf(hv);
 439                 return error;
 440         }
 441
 442         return 0;
 443 }
 444
 445 void
 446 hn_nvs_detach(struct hn_data *hv __rte_unused)
 447 {
 448         PMD_INIT_FUNC_TRACE();
 449
 450         /* NOTE: there are no requests to stop the NVS. */
 451         hn_nvs_disconn_rxbuf(hv);
 452         hn_nvs_disconn_chim(hv);
 453 }
 454
 455 /*
 456  * Ack the consumed RXBUF associated w/ this channel packet,
 457  * so that this RXBUF can be recycled by the hypervisor.
 458  */
 459 void
 460 hn_nvs_ack_rxbuf(struct vmbus_channel *chan, uint64_t tid)
 461 {
 462         unsigned int retries = 0;
 463         struct hn_nvs_rndis_ack ack = {
 464                 .type = NVS_TYPE_RNDIS_ACK,
 465                 .status = NVS_STATUS_OK,
 466         };
 467         int error;
 468
 469         PMD_RX_LOG(DEBUG, "ack RX id %" PRIu64, tid);
 470
 471  again:
 472         error = rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_COMP,
 473                                     &ack, sizeof(ack), tid,
 474                                     VMBUS_CHANPKT_FLAG_NONE, NULL);
 475
 476         if (error == 0)
 477                 return;
 478
 479         if (error == -EAGAIN) {
 480                 /*
 481                  * NOTE:
 482                  * This should _not_ happen in real world, since the
 483                  * consumption of the TX bufring from the TX path is
 484                  * controlled.
 485                  */
 486                 PMD_RX_LOG(NOTICE, "RXBUF ack retry");
 487                 if (++retries < 10) {
 488                         rte_delay_ms(1);
 489                         goto again;
 490                 }
 491         }
 492         /* RXBUF leaks! */
 493         PMD_DRV_LOG(ERR, "RXBUF ack failed");
 494 }
 495
 496 int
 497 hn_nvs_alloc_subchans(struct hn_data *hv, uint32_t *nsubch)
 498 {
 499         struct hn_nvs_subch_req req;
 500         struct hn_nvs_subch_resp resp;
 501         int error;
 502
 503         memset(&req, 0, sizeof(req));
 504         req.type = NVS_TYPE_SUBCH_REQ;
 505         req.op = NVS_SUBCH_OP_ALLOC;
 506         req.nsubch = *nsubch;
 507
 508         error = hn_nvs_execute(hv, &req, sizeof(req),
 509                                &resp, sizeof(resp),
 510                                NVS_TYPE_SUBCH_RESP);
 511         if (error)
 512                 return error;
 513
 514         if (resp.status != NVS_STATUS_OK) {
 515                 PMD_INIT_LOG(ERR,
 516                              "nvs subch alloc failed: %#x",
 517                              resp.status);
 518                 return -EIO;
 519         }
 520
 521         if (resp.nsubch > *nsubch) {
 522                 PMD_INIT_LOG(NOTICE,
 523                              "%u subchans are allocated, requested %u",
 524                              resp.nsubch, *nsubch);
 525         }
 526         *nsubch = resp.nsubch;
 527
 528         return 0;
 529 }
 530
 531 void
 532 hn_nvs_set_datapath(struct hn_data *hv, uint32_t path)
 533 {
 534         struct hn_nvs_datapath dp;
 535         int error;
 536
 537         PMD_DRV_LOG(DEBUG, "set datapath %s",
 538                     path ? "VF" : "Synthetic");
 539
 540         memset(&dp, 0, sizeof(dp));
 541         dp.type = NVS_TYPE_SET_DATAPATH;
 542         dp.active_path = path;
 543
 544         error = hn_nvs_req_send(hv, &dp, sizeof(dp));
 545         if (error) {
 546                 PMD_DRV_LOG(ERR,
 547                             "send set datapath failed: %d",
 548                             error);
 549         }
 550 }