drivers/net/ethernet/mellanox/mlx5/core/ipoib.c

   1 /*
   2  * Copyright (c) 2017, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <rdma/ib_verbs.h>
  34 #include <linux/mlx5/fs.h>
  35 #include "en.h"
  36 #include "ipoib.h"
  37
  38 #define IB_DEFAULT_Q_KEY   0xb1b
  39
  40 static int mlx5i_open(struct net_device *netdev);
  41 static int mlx5i_close(struct net_device *netdev);
  42 static int  mlx5i_dev_init(struct net_device *dev);
  43 static void mlx5i_dev_cleanup(struct net_device *dev);
  44
  45 static const struct net_device_ops mlx5i_netdev_ops = {
  46         .ndo_open                = mlx5i_open,
  47         .ndo_stop                = mlx5i_close,
  48         .ndo_init                = mlx5i_dev_init,
  49         .ndo_uninit              = mlx5i_dev_cleanup,
  50 };
  51
  52 /* IPoIB mlx5 netdev profile */
  53
  54 /* Called directly after IPoIB netdevice was created to initialize SW structs */
  55 static void mlx5i_init(struct mlx5_core_dev *mdev,
  56                        struct net_device *netdev,
  57                        const struct mlx5e_profile *profile,
  58                        void *ppriv)
  59 {
  60         struct mlx5e_priv *priv  = mlx5i_epriv(netdev);
  61
  62         priv->mdev        = mdev;
  63         priv->netdev      = netdev;
  64         priv->profile     = profile;
  65         priv->ppriv       = ppriv;
  66
  67         mlx5e_build_nic_params(mdev, &priv->channels.params, profile->max_nch(mdev));
  68
  69         /* Override RQ params as IPoIB supports only LINKED LIST RQ for now */
  70         mlx5e_set_rq_type_params(mdev, &priv->channels.params, MLX5_WQ_TYPE_LINKED_LIST);
  71         priv->channels.params.lro_en = false;
  72
  73         mutex_init(&priv->state_lock);
  74
  75         netdev->hw_features    |= NETIF_F_SG;
  76         netdev->hw_features    |= NETIF_F_IP_CSUM;
  77         netdev->hw_features    |= NETIF_F_IPV6_CSUM;
  78         netdev->hw_features    |= NETIF_F_GRO;
  79         netdev->hw_features    |= NETIF_F_TSO;
  80         netdev->hw_features    |= NETIF_F_TSO6;
  81         netdev->hw_features    |= NETIF_F_RXCSUM;
  82         netdev->hw_features    |= NETIF_F_RXHASH;
  83
  84         netdev->netdev_ops = &mlx5i_netdev_ops;
  85 }
  86
  87 /* Called directly before IPoIB netdevice is destroyed to cleanup SW structs */
  88 static void mlx5i_cleanup(struct mlx5e_priv *priv)
  89 {
  90         /* Do nothing .. */
  91 }
  92
  93 #define MLX5_QP_ENHANCED_ULP_STATELESS_MODE 2
  94
  95 static int mlx5i_create_underlay_qp(struct mlx5_core_dev *mdev, struct mlx5_core_qp *qp)
  96 {
  97         struct mlx5_qp_context *context = NULL;
  98         u32 *in = NULL;
  99         void *addr_path;
 100         int ret = 0;
 101         int inlen;
 102         void *qpc;
 103
 104         inlen = MLX5_ST_SZ_BYTES(create_qp_in);
 105         in = mlx5_vzalloc(inlen);
 106         if (!in)
 107                 return -ENOMEM;
 108
 109         qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
 110         MLX5_SET(qpc, qpc, st, MLX5_QP_ST_UD);
 111         MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
 112         MLX5_SET(qpc, qpc, ulp_stateless_offload_mode,
 113                  MLX5_QP_ENHANCED_ULP_STATELESS_MODE);
 114
 115         addr_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
 116         MLX5_SET(ads, addr_path, port, 1);
 117         MLX5_SET(ads, addr_path, grh, 1);
 118
 119         ret = mlx5_core_create_qp(mdev, qp, in, inlen);
 120         if (ret) {
 121                 mlx5_core_err(mdev, "Failed creating IPoIB QP err : %d\n", ret);
 122                 goto out;
 123         }
 124
 125         /* QP states */
 126         context = kzalloc(sizeof(*context), GFP_KERNEL);
 127         if (!context) {
 128                 ret = -ENOMEM;
 129                 goto out;
 130         }
 131
 132         context->flags = cpu_to_be32(MLX5_QP_PM_MIGRATED << 11);
 133         context->pri_path.port = 1;
 134         context->qkey = cpu_to_be32(IB_DEFAULT_Q_KEY);
 135
 136         ret = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_RST2INIT_QP, 0, context, qp);
 137         if (ret) {
 138                 mlx5_core_err(mdev, "Failed to modify qp RST2INIT, err: %d\n", ret);
 139                 goto out;
 140         }
 141         memset(context, 0, sizeof(*context));
 142
 143         ret = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_INIT2RTR_QP, 0, context, qp);
 144         if (ret) {
 145                 mlx5_core_err(mdev, "Failed to modify qp INIT2RTR, err: %d\n", ret);
 146                 goto out;
 147         }
 148
 149         ret = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_RTR2RTS_QP, 0, context, qp);
 150         if (ret) {
 151                 mlx5_core_err(mdev, "Failed to modify qp RTR2RTS, err: %d\n", ret);
 152                 goto out;
 153         }
 154
 155 out:
 156         kfree(context);
 157         kvfree(in);
 158         return ret;
 159 }
 160
 161 static void mlx5i_destroy_underlay_qp(struct mlx5_core_dev *mdev, struct mlx5_core_qp *qp)
 162 {
 163         mlx5_fs_remove_rx_underlay_qpn(mdev, qp->qpn);
 164
 165         mlx5_core_destroy_qp(mdev, qp);
 166 }
 167
 168 static int mlx5i_init_tx(struct mlx5e_priv *priv)
 169 {
 170         struct mlx5i_priv *ipriv = priv->ppriv;
 171         int err;
 172
 173         err = mlx5i_create_underlay_qp(priv->mdev, &ipriv->qp);
 174         if (err) {
 175                 mlx5_core_warn(priv->mdev, "create underlay QP failed, %d\n", err);
 176                 return err;
 177         }
 178
 179         mlx5_fs_add_rx_underlay_qpn(priv->mdev, ipriv->qp.qpn);
 180
 181         err = mlx5e_create_tis(priv->mdev, 0 /* tc */, ipriv->qp.qpn, &priv->tisn[0]);
 182         if (err) {
 183                 mlx5_core_warn(priv->mdev, "create tis failed, %d\n", err);
 184                 return err;
 185         }
 186
 187         return 0;
 188 }
 189
 190 static void mlx5i_cleanup_tx(struct mlx5e_priv *priv)
 191 {
 192         struct mlx5i_priv *ipriv = priv->ppriv;
 193
 194         mlx5e_destroy_tis(priv->mdev, priv->tisn[0]);
 195         mlx5i_destroy_underlay_qp(priv->mdev, &ipriv->qp);
 196 }
 197
 198 static int mlx5i_create_flow_steering(struct mlx5e_priv *priv)
 199 {
 200         int err;
 201
 202         priv->fs.ns = mlx5_get_flow_namespace(priv->mdev,
 203                                                MLX5_FLOW_NAMESPACE_KERNEL);
 204
 205         if (!priv->fs.ns)
 206                 return -EINVAL;
 207
 208         err = mlx5e_arfs_create_tables(priv);
 209         if (err) {
 210                 netdev_err(priv->netdev, "Failed to create arfs tables, err=%d\n",
 211                            err);
 212                 priv->netdev->hw_features &= ~NETIF_F_NTUPLE;
 213         }
 214
 215         err = mlx5e_create_ttc_table(priv);
 216         if (err) {
 217                 netdev_err(priv->netdev, "Failed to create ttc table, err=%d\n",
 218                            err);
 219                 goto err_destroy_arfs_tables;
 220         }
 221
 222         return 0;
 223
 224 err_destroy_arfs_tables:
 225         mlx5e_arfs_destroy_tables(priv);
 226
 227         return err;
 228 }
 229
 230 static void mlx5i_destroy_flow_steering(struct mlx5e_priv *priv)
 231 {
 232         mlx5e_destroy_ttc_table(priv);
 233         mlx5e_arfs_destroy_tables(priv);
 234 }
 235
 236 static int mlx5i_init_rx(struct mlx5e_priv *priv)
 237 {
 238         int err;
 239
 240         err = mlx5e_create_indirect_rqt(priv);
 241         if (err)
 242                 return err;
 243
 244         err = mlx5e_create_direct_rqts(priv);
 245         if (err)
 246                 goto err_destroy_indirect_rqts;
 247
 248         err = mlx5e_create_indirect_tirs(priv);
 249         if (err)
 250                 goto err_destroy_direct_rqts;
 251
 252         err = mlx5e_create_direct_tirs(priv);
 253         if (err)
 254                 goto err_destroy_indirect_tirs;
 255
 256         err = mlx5i_create_flow_steering(priv);
 257         if (err)
 258                 goto err_destroy_direct_tirs;
 259
 260         return 0;
 261
 262 err_destroy_direct_tirs:
 263         mlx5e_destroy_direct_tirs(priv);
 264 err_destroy_indirect_tirs:
 265         mlx5e_destroy_indirect_tirs(priv);
 266 err_destroy_direct_rqts:
 267         mlx5e_destroy_direct_rqts(priv);
 268 err_destroy_indirect_rqts:
 269         mlx5e_destroy_rqt(priv, &priv->indir_rqt);
 270         return err;
 271 }
 272
 273 static void mlx5i_cleanup_rx(struct mlx5e_priv *priv)
 274 {
 275         mlx5i_destroy_flow_steering(priv);
 276         mlx5e_destroy_direct_tirs(priv);
 277         mlx5e_destroy_indirect_tirs(priv);
 278         mlx5e_destroy_direct_rqts(priv);
 279         mlx5e_destroy_rqt(priv, &priv->indir_rqt);
 280 }
 281
 282 static const struct mlx5e_profile mlx5i_nic_profile = {
 283         .init              = mlx5i_init,
 284         .cleanup           = mlx5i_cleanup,
 285         .init_tx           = mlx5i_init_tx,
 286         .cleanup_tx        = mlx5i_cleanup_tx,
 287         .init_rx           = mlx5i_init_rx,
 288         .cleanup_rx        = mlx5i_cleanup_rx,
 289         .enable            = NULL, /* mlx5i_enable */
 290         .disable           = NULL, /* mlx5i_disable */
 291         .update_stats      = NULL, /* mlx5i_update_stats */
 292         .max_nch           = mlx5e_get_max_num_channels,
 293         .rx_handlers.handle_rx_cqe       = mlx5i_handle_rx_cqe,
 294         .rx_handlers.handle_rx_cqe_mpwqe = NULL, /* Not supported */
 295         .max_tc            = MLX5I_MAX_NUM_TC,
 296 };
 297
 298 /* mlx5i netdev NDos */
 299
 300 static int mlx5i_dev_init(struct net_device *dev)
 301 {
 302         struct mlx5e_priv    *priv   = mlx5i_epriv(dev);
 303         struct mlx5i_priv    *ipriv  = priv->ppriv;
 304
 305         /* Set dev address using underlay QP */
 306         dev->dev_addr[1] = (ipriv->qp.qpn >> 16) & 0xff;
 307         dev->dev_addr[2] = (ipriv->qp.qpn >>  8) & 0xff;
 308         dev->dev_addr[3] = (ipriv->qp.qpn) & 0xff;
 309
 310         return 0;
 311 }
 312
 313 static void mlx5i_dev_cleanup(struct net_device *dev)
 314 {
 315         struct mlx5e_priv    *priv   = mlx5i_epriv(dev);
 316         struct mlx5_core_dev *mdev   = priv->mdev;
 317         struct mlx5i_priv    *ipriv  = priv->ppriv;
 318         struct mlx5_qp_context context;
 319
 320         /* detach qp from flow-steering by reset it */
 321         mlx5_core_qp_modify(mdev, MLX5_CMD_OP_2RST_QP, 0, &context, &ipriv->qp);
 322 }
 323
 324 static int mlx5i_open(struct net_device *netdev)
 325 {
 326         struct mlx5e_priv *priv = mlx5i_epriv(netdev);
 327         int err;
 328
 329         mutex_lock(&priv->state_lock);
 330
 331         set_bit(MLX5E_STATE_OPENED, &priv->state);
 332
 333         err = mlx5e_open_channels(priv, &priv->channels);
 334         if (err)
 335                 goto err_clear_state_opened_flag;
 336
 337         mlx5e_refresh_tirs(priv, false);
 338         mlx5e_activate_priv_channels(priv);
 339         mutex_unlock(&priv->state_lock);
 340         return 0;
 341
 342 err_clear_state_opened_flag:
 343         clear_bit(MLX5E_STATE_OPENED, &priv->state);
 344         mutex_unlock(&priv->state_lock);
 345         return err;
 346 }
 347
 348 static int mlx5i_close(struct net_device *netdev)
 349 {
 350         struct mlx5e_priv *priv = mlx5i_epriv(netdev);
 351
 352         /* May already be CLOSED in case a previous configuration operation
 353          * (e.g RX/TX queue size change) that involves close&open failed.
 354          */
 355         mutex_lock(&priv->state_lock);
 356
 357         if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
 358                 goto unlock;
 359
 360         clear_bit(MLX5E_STATE_OPENED, &priv->state);
 361
 362         netif_carrier_off(priv->netdev);
 363         mlx5e_deactivate_priv_channels(priv);
 364         mlx5e_close_channels(&priv->channels);
 365 unlock:
 366         mutex_unlock(&priv->state_lock);
 367         return 0;
 368 }
 369
 370 /* IPoIB RDMA netdev callbacks */
 371 static int mlx5i_attach_mcast(struct net_device *netdev, struct ib_device *hca,
 372                               union ib_gid *gid, u16 lid, int set_qkey,
 373                               u32 qkey)
 374 {
 375         struct mlx5e_priv    *epriv = mlx5i_epriv(netdev);
 376         struct mlx5_core_dev *mdev  = epriv->mdev;
 377         struct mlx5i_priv    *ipriv = epriv->ppriv;
 378         int err;
 379
 380         mlx5_core_dbg(mdev, "attaching QPN 0x%x, MGID %pI6\n", ipriv->qp.qpn, gid->raw);
 381         err = mlx5_core_attach_mcg(mdev, gid, ipriv->qp.qpn);
 382         if (err)
 383                 mlx5_core_warn(mdev, "failed attaching QPN 0x%x, MGID %pI6\n",
 384                                ipriv->qp.qpn, gid->raw);
 385
 386         if (set_qkey) {
 387                 mlx5_core_dbg(mdev, "%s setting qkey 0x%x\n",
 388                               netdev->name, qkey);
 389                 ipriv->qkey = qkey;
 390         }
 391
 392         return err;
 393 }
 394
 395 static int mlx5i_detach_mcast(struct net_device *netdev, struct ib_device *hca,
 396                               union ib_gid *gid, u16 lid)
 397 {
 398         struct mlx5e_priv    *epriv = mlx5i_epriv(netdev);
 399         struct mlx5_core_dev *mdev  = epriv->mdev;
 400         struct mlx5i_priv    *ipriv = epriv->ppriv;
 401         int err;
 402
 403         mlx5_core_dbg(mdev, "detaching QPN 0x%x, MGID %pI6\n", ipriv->qp.qpn, gid->raw);
 404
 405         err = mlx5_core_detach_mcg(mdev, gid, ipriv->qp.qpn);
 406         if (err)
 407                 mlx5_core_dbg(mdev, "failed dettaching QPN 0x%x, MGID %pI6\n",
 408                               ipriv->qp.qpn, gid->raw);
 409
 410         return err;
 411 }
 412
 413 static int mlx5i_xmit(struct net_device *dev, struct sk_buff *skb,
 414                       struct ib_ah *address, u32 dqpn)
 415 {
 416         struct mlx5e_priv *epriv = mlx5i_epriv(dev);
 417         struct mlx5e_txqsq *sq   = epriv->txq2sq[skb_get_queue_mapping(skb)];
 418         struct mlx5_ib_ah *mah   = to_mah(address);
 419         struct mlx5i_priv *ipriv = epriv->ppriv;
 420
 421         return mlx5i_sq_xmit(sq, skb, &mah->av, dqpn, ipriv->qkey);
 422 }
 423
 424 static int mlx5i_check_required_hca_cap(struct mlx5_core_dev *mdev)
 425 {
 426         if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_IB)
 427                 return -EOPNOTSUPP;
 428
 429         if (!MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads)) {
 430                 mlx5_core_warn(mdev, "IPoIB enhanced offloads are not supported\n");
 431                 return -EOPNOTSUPP;
 432         }
 433
 434         return 0;
 435 }
 436
 437 struct net_device *mlx5_rdma_netdev_alloc(struct mlx5_core_dev *mdev,
 438                                           struct ib_device *ibdev,
 439                                           const char *name,
 440                                           void (*setup)(struct net_device *))
 441 {
 442         const struct mlx5e_profile *profile = &mlx5i_nic_profile;
 443         int nch = profile->max_nch(mdev);
 444         struct net_device *netdev;
 445         struct mlx5i_priv *ipriv;
 446         struct mlx5e_priv *epriv;
 447         struct rdma_netdev *rn;
 448         int err;
 449
 450         if (mlx5i_check_required_hca_cap(mdev)) {
 451                 mlx5_core_warn(mdev, "Accelerated mode is not supported\n");
 452                 return ERR_PTR(-EOPNOTSUPP);
 453         }
 454
 455         /* This function should only be called once per mdev */
 456         err = mlx5e_create_mdev_resources(mdev);
 457         if (err)
 458                 return NULL;
 459
 460         netdev = alloc_netdev_mqs(sizeof(struct mlx5i_priv) + sizeof(struct mlx5e_priv),
 461                                   name, NET_NAME_UNKNOWN,
 462                                   setup,
 463                                   nch * MLX5E_MAX_NUM_TC,
 464                                   nch);
 465         if (!netdev) {
 466                 mlx5_core_warn(mdev, "alloc_netdev_mqs failed\n");
 467                 goto free_mdev_resources;
 468         }
 469
 470         ipriv = netdev_priv(netdev);
 471         epriv = mlx5i_epriv(netdev);
 472
 473         epriv->wq = create_singlethread_workqueue("mlx5i");
 474         if (!epriv->wq)
 475                 goto err_free_netdev;
 476
 477         profile->init(mdev, netdev, profile, ipriv);
 478
 479         mlx5e_attach_netdev(epriv);
 480         netif_carrier_off(netdev);
 481
 482         /* set rdma_netdev func pointers */
 483         rn = &ipriv->rn;
 484         rn->hca  = ibdev;
 485         rn->send = mlx5i_xmit;
 486         rn->attach_mcast = mlx5i_attach_mcast;
 487         rn->detach_mcast = mlx5i_detach_mcast;
 488
 489         return netdev;
 490
 491 err_free_netdev:
 492         free_netdev(netdev);
 493 free_mdev_resources:
 494         mlx5e_destroy_mdev_resources(mdev);
 495
 496         return NULL;
 497 }
 498 EXPORT_SYMBOL(mlx5_rdma_netdev_alloc);
 499
 500 void mlx5_rdma_netdev_free(struct net_device *netdev)
 501 {
 502         struct mlx5e_priv          *priv    = mlx5i_epriv(netdev);
 503         const struct mlx5e_profile *profile = priv->profile;
 504
 505         mlx5e_detach_netdev(priv);
 506         profile->cleanup(priv);
 507         destroy_workqueue(priv->wq);
 508         free_netdev(netdev);
 509
 510         mlx5e_destroy_mdev_resources(priv->mdev);
 511 }
 512 EXPORT_SYMBOL(mlx5_rdma_netdev_free);
 513