drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2020 Mellanox Technologies. */
   3
   4 #include <net/dst_metadata.h>
   5 #include <linux/netdevice.h>
   6 #include <linux/list.h>
   7 #include <linux/rculist.h>
   8 #include <linux/rtnetlink.h>
   9 #include <linux/workqueue.h>
  10 #include <linux/spinlock.h>
  11 #include "tc.h"
  12 #include "neigh.h"
  13 #include "en_rep.h"
  14 #include "eswitch.h"
  15 #include "lib/fs_chains.h"
  16 #include "en/tc_ct.h"
  17 #include "en/mapping.h"
  18 #include "en/tc_tun.h"
  19 #include "lib/port_tun.h"
  20 #include "en/tc/sample.h"
  21 #include "en_accel/ipsec_rxtx.h"
  22 #include "en/tc/int_port.h"
  23
  24 struct mlx5e_rep_indr_block_priv {
  25         struct net_device *netdev;
  26         struct mlx5e_rep_priv *rpriv;
  27
  28         struct list_head list;
  29 };
  30
  31 int mlx5e_rep_encap_entry_attach(struct mlx5e_priv *priv,
  32                                  struct mlx5e_encap_entry *e,
  33                                  struct mlx5e_neigh *m_neigh,
  34                                  struct net_device *neigh_dev)
  35 {
  36         struct mlx5e_rep_priv *rpriv = priv->ppriv;
  37         struct mlx5_rep_uplink_priv *uplink_priv = &rpriv->uplink_priv;
  38         struct mlx5_tun_entropy *tun_entropy = &uplink_priv->tun_entropy;
  39         struct mlx5e_neigh_hash_entry *nhe;
  40         int err;
  41
  42         err = mlx5_tun_entropy_refcount_inc(tun_entropy, e->reformat_type);
  43         if (err)
  44                 return err;
  45
  46         mutex_lock(&rpriv->neigh_update.encap_lock);
  47         nhe = mlx5e_rep_neigh_entry_lookup(priv, m_neigh);
  48         if (!nhe) {
  49                 err = mlx5e_rep_neigh_entry_create(priv, m_neigh, neigh_dev, &nhe);
  50                 if (err) {
  51                         mutex_unlock(&rpriv->neigh_update.encap_lock);
  52                         mlx5_tun_entropy_refcount_dec(tun_entropy,
  53                                                       e->reformat_type);
  54                         return err;
  55                 }
  56         }
  57
  58         e->nhe = nhe;
  59         spin_lock(&nhe->encap_list_lock);
  60         list_add_rcu(&e->encap_list, &nhe->encap_list);
  61         spin_unlock(&nhe->encap_list_lock);
  62
  63         mutex_unlock(&rpriv->neigh_update.encap_lock);
  64
  65         return 0;
  66 }
  67
  68 void mlx5e_rep_encap_entry_detach(struct mlx5e_priv *priv,
  69                                   struct mlx5e_encap_entry *e)
  70 {
  71         struct mlx5e_rep_priv *rpriv = priv->ppriv;
  72         struct mlx5_rep_uplink_priv *uplink_priv = &rpriv->uplink_priv;
  73         struct mlx5_tun_entropy *tun_entropy = &uplink_priv->tun_entropy;
  74
  75         if (!e->nhe)
  76                 return;
  77
  78         spin_lock(&e->nhe->encap_list_lock);
  79         list_del_rcu(&e->encap_list);
  80         spin_unlock(&e->nhe->encap_list_lock);
  81
  82         mlx5e_rep_neigh_entry_release(e->nhe);
  83         e->nhe = NULL;
  84         mlx5_tun_entropy_refcount_dec(tun_entropy, e->reformat_type);
  85 }
  86
  87 void mlx5e_rep_update_flows(struct mlx5e_priv *priv,
  88                             struct mlx5e_encap_entry *e,
  89                             bool neigh_connected,
  90                             unsigned char ha[ETH_ALEN])
  91 {
  92         struct ethhdr *eth = (struct ethhdr *)e->encap_header;
  93         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
  94         bool encap_connected;
  95         LIST_HEAD(flow_list);
  96
  97         ASSERT_RTNL();
  98
  99         mutex_lock(&esw->offloads.encap_tbl_lock);
 100         encap_connected = !!(e->flags & MLX5_ENCAP_ENTRY_VALID);
 101         if (encap_connected == neigh_connected && ether_addr_equal(e->h_dest, ha))
 102                 goto unlock;
 103
 104         mlx5e_take_all_encap_flows(e, &flow_list);
 105
 106         if ((e->flags & MLX5_ENCAP_ENTRY_VALID) &&
 107             (!neigh_connected || !ether_addr_equal(e->h_dest, ha)))
 108                 mlx5e_tc_encap_flows_del(priv, e, &flow_list);
 109
 110         if (neigh_connected && !(e->flags & MLX5_ENCAP_ENTRY_VALID)) {
 111                 struct net_device *route_dev;
 112
 113                 ether_addr_copy(e->h_dest, ha);
 114                 ether_addr_copy(eth->h_dest, ha);
 115                 /* Update the encap source mac, in case that we delete
 116                  * the flows when encap source mac changed.
 117                  */
 118                 route_dev = __dev_get_by_index(dev_net(priv->netdev), e->route_dev_ifindex);
 119                 if (route_dev)
 120                         ether_addr_copy(eth->h_source, route_dev->dev_addr);
 121
 122                 mlx5e_tc_encap_flows_add(priv, e, &flow_list);
 123         }
 124 unlock:
 125         mutex_unlock(&esw->offloads.encap_tbl_lock);
 126         mlx5e_put_flow_list(priv, &flow_list);
 127 }
 128
 129 static int
 130 mlx5e_rep_setup_tc_cls_flower(struct mlx5e_priv *priv,
 131                               struct flow_cls_offload *cls_flower, int flags)
 132 {
 133         switch (cls_flower->command) {
 134         case FLOW_CLS_REPLACE:
 135                 return mlx5e_configure_flower(priv->netdev, priv, cls_flower,
 136                                               flags);
 137         case FLOW_CLS_DESTROY:
 138                 return mlx5e_delete_flower(priv->netdev, priv, cls_flower,
 139                                            flags);
 140         case FLOW_CLS_STATS:
 141                 return mlx5e_stats_flower(priv->netdev, priv, cls_flower,
 142                                           flags);
 143         default:
 144                 return -EOPNOTSUPP;
 145         }
 146 }
 147
 148 static
 149 int mlx5e_rep_setup_tc_cls_matchall(struct mlx5e_priv *priv,
 150                                     struct tc_cls_matchall_offload *ma)
 151 {
 152         switch (ma->command) {
 153         case TC_CLSMATCHALL_REPLACE:
 154                 return mlx5e_tc_configure_matchall(priv, ma);
 155         case TC_CLSMATCHALL_DESTROY:
 156                 return mlx5e_tc_delete_matchall(priv, ma);
 157         case TC_CLSMATCHALL_STATS:
 158                 mlx5e_tc_stats_matchall(priv, ma);
 159                 return 0;
 160         default:
 161                 return -EOPNOTSUPP;
 162         }
 163 }
 164
 165 static int mlx5e_rep_setup_tc_cb(enum tc_setup_type type, void *type_data,
 166                                  void *cb_priv)
 167 {
 168         unsigned long flags = MLX5_TC_FLAG(INGRESS) | MLX5_TC_FLAG(ESW_OFFLOAD);
 169         struct mlx5e_priv *priv = cb_priv;
 170
 171         if (!priv->netdev || !netif_device_present(priv->netdev))
 172                 return -EOPNOTSUPP;
 173
 174         switch (type) {
 175         case TC_SETUP_CLSFLOWER:
 176                 return mlx5e_rep_setup_tc_cls_flower(priv, type_data, flags);
 177         case TC_SETUP_CLSMATCHALL:
 178                 return mlx5e_rep_setup_tc_cls_matchall(priv, type_data);
 179         default:
 180                 return -EOPNOTSUPP;
 181         }
 182 }
 183
 184 static int mlx5e_rep_setup_ft_cb(enum tc_setup_type type, void *type_data,
 185                                  void *cb_priv)
 186 {
 187         struct flow_cls_offload tmp, *f = type_data;
 188         struct mlx5e_priv *priv = cb_priv;
 189         struct mlx5_eswitch *esw;
 190         unsigned long flags;
 191         int err;
 192
 193         flags = MLX5_TC_FLAG(INGRESS) |
 194                 MLX5_TC_FLAG(ESW_OFFLOAD) |
 195                 MLX5_TC_FLAG(FT_OFFLOAD);
 196         esw = priv->mdev->priv.eswitch;
 197
 198         switch (type) {
 199         case TC_SETUP_CLSFLOWER:
 200                 memcpy(&tmp, f, sizeof(*f));
 201
 202                 if (!mlx5_chains_prios_supported(esw_chains(esw)))
 203                         return -EOPNOTSUPP;
 204
 205                 /* Re-use tc offload path by moving the ft flow to the
 206                  * reserved ft chain.
 207                  *
 208                  * FT offload can use prio range [0, INT_MAX], so we normalize
 209                  * it to range [1, mlx5_esw_chains_get_prio_range(esw)]
 210                  * as with tc, where prio 0 isn't supported.
 211                  *
 212                  * We only support chain 0 of FT offload.
 213                  */
 214                 if (tmp.common.prio >= mlx5_chains_get_prio_range(esw_chains(esw)))
 215                         return -EOPNOTSUPP;
 216                 if (tmp.common.chain_index != 0)
 217                         return -EOPNOTSUPP;
 218
 219                 tmp.common.chain_index = mlx5_chains_get_nf_ft_chain(esw_chains(esw));
 220                 tmp.common.prio++;
 221                 err = mlx5e_rep_setup_tc_cls_flower(priv, &tmp, flags);
 222                 memcpy(&f->stats, &tmp.stats, sizeof(f->stats));
 223                 return err;
 224         default:
 225                 return -EOPNOTSUPP;
 226         }
 227 }
 228
 229 static LIST_HEAD(mlx5e_rep_block_tc_cb_list);
 230 static LIST_HEAD(mlx5e_rep_block_ft_cb_list);
 231 int mlx5e_rep_setup_tc(struct net_device *dev, enum tc_setup_type type,
 232                        void *type_data)
 233 {
 234         struct mlx5e_priv *priv = netdev_priv(dev);
 235         struct flow_block_offload *f = type_data;
 236
 237         f->unlocked_driver_cb = true;
 238
 239         switch (type) {
 240         case TC_SETUP_BLOCK:
 241                 return flow_block_cb_setup_simple(type_data,
 242                                                   &mlx5e_rep_block_tc_cb_list,
 243                                                   mlx5e_rep_setup_tc_cb,
 244                                                   priv, priv, true);
 245         case TC_SETUP_FT:
 246                 return flow_block_cb_setup_simple(type_data,
 247                                                   &mlx5e_rep_block_ft_cb_list,
 248                                                   mlx5e_rep_setup_ft_cb,
 249                                                   priv, priv, true);
 250         default:
 251                 return -EOPNOTSUPP;
 252         }
 253 }
 254
 255 int mlx5e_rep_tc_init(struct mlx5e_rep_priv *rpriv)
 256 {
 257         struct mlx5_rep_uplink_priv *uplink_priv = &rpriv->uplink_priv;
 258         int err;
 259
 260         mutex_init(&uplink_priv->unready_flows_lock);
 261         INIT_LIST_HEAD(&uplink_priv->unready_flows);
 262
 263         /* init shared tc flow table */
 264         err = mlx5e_tc_esw_init(&uplink_priv->tc_ht);
 265         return err;
 266 }
 267
 268 void mlx5e_rep_tc_cleanup(struct mlx5e_rep_priv *rpriv)
 269 {
 270         /* delete shared tc flow table */
 271         mlx5e_tc_esw_cleanup(&rpriv->uplink_priv.tc_ht);
 272         mutex_destroy(&rpriv->uplink_priv.unready_flows_lock);
 273 }
 274
 275 void mlx5e_rep_tc_enable(struct mlx5e_priv *priv)
 276 {
 277         struct mlx5e_rep_priv *rpriv = priv->ppriv;
 278
 279         INIT_WORK(&rpriv->uplink_priv.reoffload_flows_work,
 280                   mlx5e_tc_reoffload_flows_work);
 281 }
 282
 283 void mlx5e_rep_tc_disable(struct mlx5e_priv *priv)
 284 {
 285         struct mlx5e_rep_priv *rpriv = priv->ppriv;
 286
 287         cancel_work_sync(&rpriv->uplink_priv.reoffload_flows_work);
 288 }
 289
 290 int mlx5e_rep_tc_event_port_affinity(struct mlx5e_priv *priv)
 291 {
 292         struct mlx5e_rep_priv *rpriv = priv->ppriv;
 293
 294         queue_work(priv->wq, &rpriv->uplink_priv.reoffload_flows_work);
 295
 296         return NOTIFY_OK;
 297 }
 298
 299 static struct mlx5e_rep_indr_block_priv *
 300 mlx5e_rep_indr_block_priv_lookup(struct mlx5e_rep_priv *rpriv,
 301                                  struct net_device *netdev)
 302 {
 303         struct mlx5e_rep_indr_block_priv *cb_priv;
 304
 305         list_for_each_entry(cb_priv,
 306                             &rpriv->uplink_priv.tc_indr_block_priv_list,
 307                             list)
 308                 if (cb_priv->netdev == netdev)
 309                         return cb_priv;
 310
 311         return NULL;
 312 }
 313
 314 static int
 315 mlx5e_rep_indr_offload(struct net_device *netdev,
 316                        struct flow_cls_offload *flower,
 317                        struct mlx5e_rep_indr_block_priv *indr_priv,
 318                        unsigned long flags)
 319 {
 320         struct mlx5e_priv *priv = netdev_priv(indr_priv->rpriv->netdev);
 321         int err = 0;
 322
 323         if (!netif_device_present(indr_priv->rpriv->netdev))
 324                 return -EOPNOTSUPP;
 325
 326         switch (flower->command) {
 327         case FLOW_CLS_REPLACE:
 328                 err = mlx5e_configure_flower(netdev, priv, flower, flags);
 329                 break;
 330         case FLOW_CLS_DESTROY:
 331                 err = mlx5e_delete_flower(netdev, priv, flower, flags);
 332                 break;
 333         case FLOW_CLS_STATS:
 334                 err = mlx5e_stats_flower(netdev, priv, flower, flags);
 335                 break;
 336         default:
 337                 err = -EOPNOTSUPP;
 338         }
 339
 340         return err;
 341 }
 342
 343 static int mlx5e_rep_indr_setup_tc_cb(enum tc_setup_type type,
 344                                       void *type_data, void *indr_priv)
 345 {
 346         unsigned long flags = MLX5_TC_FLAG(EGRESS) | MLX5_TC_FLAG(ESW_OFFLOAD);
 347         struct mlx5e_rep_indr_block_priv *priv = indr_priv;
 348
 349         switch (type) {
 350         case TC_SETUP_CLSFLOWER:
 351                 return mlx5e_rep_indr_offload(priv->netdev, type_data, priv,
 352                                               flags);
 353         default:
 354                 return -EOPNOTSUPP;
 355         }
 356 }
 357
 358 static int mlx5e_rep_indr_setup_ft_cb(enum tc_setup_type type,
 359                                       void *type_data, void *indr_priv)
 360 {
 361         struct mlx5e_rep_indr_block_priv *priv = indr_priv;
 362         struct flow_cls_offload *f = type_data;
 363         struct flow_cls_offload tmp;
 364         struct mlx5e_priv *mpriv;
 365         struct mlx5_eswitch *esw;
 366         unsigned long flags;
 367         int err;
 368
 369         mpriv = netdev_priv(priv->rpriv->netdev);
 370         esw = mpriv->mdev->priv.eswitch;
 371
 372         flags = MLX5_TC_FLAG(EGRESS) |
 373                 MLX5_TC_FLAG(ESW_OFFLOAD) |
 374                 MLX5_TC_FLAG(FT_OFFLOAD);
 375
 376         switch (type) {
 377         case TC_SETUP_CLSFLOWER:
 378                 memcpy(&tmp, f, sizeof(*f));
 379
 380                 /* Re-use tc offload path by moving the ft flow to the
 381                  * reserved ft chain.
 382                  *
 383                  * FT offload can use prio range [0, INT_MAX], so we normalize
 384                  * it to range [1, mlx5_esw_chains_get_prio_range(esw)]
 385                  * as with tc, where prio 0 isn't supported.
 386                  *
 387                  * We only support chain 0 of FT offload.
 388                  */
 389                 if (!mlx5_chains_prios_supported(esw_chains(esw)) ||
 390                     tmp.common.prio >= mlx5_chains_get_prio_range(esw_chains(esw)) ||
 391                     tmp.common.chain_index)
 392                         return -EOPNOTSUPP;
 393
 394                 tmp.common.chain_index = mlx5_chains_get_nf_ft_chain(esw_chains(esw));
 395                 tmp.common.prio++;
 396                 err = mlx5e_rep_indr_offload(priv->netdev, &tmp, priv, flags);
 397                 memcpy(&f->stats, &tmp.stats, sizeof(f->stats));
 398                 return err;
 399         default:
 400                 return -EOPNOTSUPP;
 401         }
 402 }
 403
 404 static void mlx5e_rep_indr_block_unbind(void *cb_priv)
 405 {
 406         struct mlx5e_rep_indr_block_priv *indr_priv = cb_priv;
 407
 408         list_del(&indr_priv->list);
 409         kfree(indr_priv);
 410 }
 411
 412 static LIST_HEAD(mlx5e_block_cb_list);
 413
 414 static int
 415 mlx5e_rep_indr_setup_block(struct net_device *netdev, struct Qdisc *sch,
 416                            struct mlx5e_rep_priv *rpriv,
 417                            struct flow_block_offload *f,
 418                            flow_setup_cb_t *setup_cb,
 419                            void *data,
 420                            void (*cleanup)(struct flow_block_cb *block_cb))
 421 {
 422         struct mlx5e_priv *priv = netdev_priv(rpriv->netdev);
 423         struct mlx5e_rep_indr_block_priv *indr_priv;
 424         struct flow_block_cb *block_cb;
 425
 426         if (!mlx5e_tc_tun_device_to_offload(priv, netdev) &&
 427             !(is_vlan_dev(netdev) && vlan_dev_real_dev(netdev) == rpriv->netdev))
 428                 return -EOPNOTSUPP;
 429
 430         if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
 431                 return -EOPNOTSUPP;
 432
 433         f->unlocked_driver_cb = true;
 434         f->driver_block_list = &mlx5e_block_cb_list;
 435
 436         switch (f->command) {
 437         case FLOW_BLOCK_BIND:
 438                 indr_priv = mlx5e_rep_indr_block_priv_lookup(rpriv, netdev);
 439                 if (indr_priv)
 440                         return -EEXIST;
 441
 442                 indr_priv = kmalloc(sizeof(*indr_priv), GFP_KERNEL);
 443                 if (!indr_priv)
 444                         return -ENOMEM;
 445
 446                 indr_priv->netdev = netdev;
 447                 indr_priv->rpriv = rpriv;
 448                 list_add(&indr_priv->list,
 449                          &rpriv->uplink_priv.tc_indr_block_priv_list);
 450
 451                 block_cb = flow_indr_block_cb_alloc(setup_cb, indr_priv, indr_priv,
 452                                                     mlx5e_rep_indr_block_unbind,
 453                                                     f, netdev, sch, data, rpriv,
 454                                                     cleanup);
 455                 if (IS_ERR(block_cb)) {
 456                         list_del(&indr_priv->list);
 457                         kfree(indr_priv);
 458                         return PTR_ERR(block_cb);
 459                 }
 460                 flow_block_cb_add(block_cb, f);
 461                 list_add_tail(&block_cb->driver_list, &mlx5e_block_cb_list);
 462
 463                 return 0;
 464         case FLOW_BLOCK_UNBIND:
 465                 indr_priv = mlx5e_rep_indr_block_priv_lookup(rpriv, netdev);
 466                 if (!indr_priv)
 467                         return -ENOENT;
 468
 469                 block_cb = flow_block_cb_lookup(f->block, setup_cb, indr_priv);
 470                 if (!block_cb)
 471                         return -ENOENT;
 472
 473                 flow_indr_block_cb_remove(block_cb, f);
 474                 list_del(&block_cb->driver_list);
 475                 return 0;
 476         default:
 477                 return -EOPNOTSUPP;
 478         }
 479         return 0;
 480 }
 481
 482 static
 483 int mlx5e_rep_indr_setup_cb(struct net_device *netdev, struct Qdisc *sch, void *cb_priv,
 484                             enum tc_setup_type type, void *type_data,
 485                             void *data,
 486                             void (*cleanup)(struct flow_block_cb *block_cb))
 487 {
 488         switch (type) {
 489         case TC_SETUP_BLOCK:
 490                 return mlx5e_rep_indr_setup_block(netdev, sch, cb_priv, type_data,
 491                                                   mlx5e_rep_indr_setup_tc_cb,
 492                                                   data, cleanup);
 493         case TC_SETUP_FT:
 494                 return mlx5e_rep_indr_setup_block(netdev, sch, cb_priv, type_data,
 495                                                   mlx5e_rep_indr_setup_ft_cb,
 496                                                   data, cleanup);
 497         default:
 498                 return -EOPNOTSUPP;
 499         }
 500 }
 501
 502 int mlx5e_rep_tc_netdevice_event_register(struct mlx5e_rep_priv *rpriv)
 503 {
 504         struct mlx5_rep_uplink_priv *uplink_priv = &rpriv->uplink_priv;
 505
 506         /* init indirect block notifications */
 507         INIT_LIST_HEAD(&uplink_priv->tc_indr_block_priv_list);
 508
 509         return flow_indr_dev_register(mlx5e_rep_indr_setup_cb, rpriv);
 510 }
 511
 512 void mlx5e_rep_tc_netdevice_event_unregister(struct mlx5e_rep_priv *rpriv)
 513 {
 514         flow_indr_dev_unregister(mlx5e_rep_indr_setup_cb, rpriv,
 515                                  mlx5e_rep_indr_block_unbind);
 516 }
 517
 518 static bool mlx5e_restore_tunnel(struct mlx5e_priv *priv, struct sk_buff *skb,
 519                                  struct mlx5e_tc_update_priv *tc_priv,
 520                                  u32 tunnel_id)
 521 {
 522         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 523         struct tunnel_match_enc_opts enc_opts = {};
 524         struct mlx5_rep_uplink_priv *uplink_priv;
 525         struct mlx5e_rep_priv *uplink_rpriv;
 526         struct metadata_dst *tun_dst;
 527         struct tunnel_match_key key;
 528         u32 tun_id, enc_opts_id;
 529         struct net_device *dev;
 530         int err;
 531
 532         enc_opts_id = tunnel_id & ENC_OPTS_BITS_MASK;
 533         tun_id = tunnel_id >> ENC_OPTS_BITS;
 534
 535         if (!tun_id)
 536                 return true;
 537
 538         uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
 539         uplink_priv = &uplink_rpriv->uplink_priv;
 540
 541         err = mapping_find(uplink_priv->tunnel_mapping, tun_id, &key);
 542         if (err) {
 543                 WARN_ON_ONCE(true);
 544                 netdev_dbg(priv->netdev,
 545                            "Couldn't find tunnel for tun_id: %d, err: %d\n",
 546                            tun_id, err);
 547                 return false;
 548         }
 549
 550         if (enc_opts_id) {
 551                 err = mapping_find(uplink_priv->tunnel_enc_opts_mapping,
 552                                    enc_opts_id, &enc_opts);
 553                 if (err) {
 554                         netdev_dbg(priv->netdev,
 555                                    "Couldn't find tunnel (opts) for tun_id: %d, err: %d\n",
 556                                    enc_opts_id, err);
 557                         return false;
 558                 }
 559         }
 560
 561         if (key.enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
 562                 tun_dst = __ip_tun_set_dst(key.enc_ipv4.src, key.enc_ipv4.dst,
 563                                            key.enc_ip.tos, key.enc_ip.ttl,
 564                                            key.enc_tp.dst, TUNNEL_KEY,
 565                                            key32_to_tunnel_id(key.enc_key_id.keyid),
 566                                            enc_opts.key.len);
 567         } else if (key.enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
 568                 tun_dst = __ipv6_tun_set_dst(&key.enc_ipv6.src, &key.enc_ipv6.dst,
 569                                              key.enc_ip.tos, key.enc_ip.ttl,
 570                                              key.enc_tp.dst, 0, TUNNEL_KEY,
 571                                              key32_to_tunnel_id(key.enc_key_id.keyid),
 572                                              enc_opts.key.len);
 573         } else {
 574                 netdev_dbg(priv->netdev,
 575                            "Couldn't restore tunnel, unsupported addr_type: %d\n",
 576                            key.enc_control.addr_type);
 577                 return false;
 578         }
 579
 580         if (!tun_dst) {
 581                 netdev_dbg(priv->netdev, "Couldn't restore tunnel, no tun_dst\n");
 582                 return false;
 583         }
 584
 585         tun_dst->u.tun_info.key.tp_src = key.enc_tp.src;
 586
 587         if (enc_opts.key.len)
 588                 ip_tunnel_info_opts_set(&tun_dst->u.tun_info,
 589                                         enc_opts.key.data,
 590                                         enc_opts.key.len,
 591                                         enc_opts.key.dst_opt_type);
 592
 593         skb_dst_set(skb, (struct dst_entry *)tun_dst);
 594         dev = dev_get_by_index(&init_net, key.filter_ifindex);
 595         if (!dev) {
 596                 netdev_dbg(priv->netdev,
 597                            "Couldn't find tunnel device with ifindex: %d\n",
 598                            key.filter_ifindex);
 599                 return false;
 600         }
 601
 602         /* Set fwd_dev so we do dev_put() after datapath */
 603         tc_priv->fwd_dev = dev;
 604
 605         skb->dev = dev;
 606
 607         return true;
 608 }
 609
 610 static bool mlx5e_restore_skb_chain(struct sk_buff *skb, u32 chain, u32 reg_c1,
 611                                     struct mlx5e_tc_update_priv *tc_priv)
 612 {
 613         struct mlx5e_priv *priv = netdev_priv(skb->dev);
 614         u32 tunnel_id = (reg_c1 >> ESW_TUN_OFFSET) & TUNNEL_ID_MASK;
 615
 616 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
 617         if (chain) {
 618                 struct mlx5_rep_uplink_priv *uplink_priv;
 619                 struct mlx5e_rep_priv *uplink_rpriv;
 620                 struct tc_skb_ext *tc_skb_ext;
 621                 struct mlx5_eswitch *esw;
 622                 u32 zone_restore_id;
 623
 624                 tc_skb_ext = tc_skb_ext_alloc(skb);
 625                 if (!tc_skb_ext) {
 626                         WARN_ON(1);
 627                         return false;
 628                 }
 629                 tc_skb_ext->chain = chain;
 630                 zone_restore_id = reg_c1 & ESW_ZONE_ID_MASK;
 631                 esw = priv->mdev->priv.eswitch;
 632                 uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
 633                 uplink_priv = &uplink_rpriv->uplink_priv;
 634                 if (!mlx5e_tc_ct_restore_flow(uplink_priv->ct_priv, skb,
 635                                               zone_restore_id))
 636                         return false;
 637         }
 638 #endif /* CONFIG_NET_TC_SKB_EXT */
 639
 640         return mlx5e_restore_tunnel(priv, skb, tc_priv, tunnel_id);
 641 }
 642
 643 static void mlx5_rep_tc_post_napi_receive(struct mlx5e_tc_update_priv *tc_priv)
 644 {
 645         if (tc_priv->fwd_dev)
 646                 dev_put(tc_priv->fwd_dev);
 647 }
 648
 649 static void mlx5e_restore_skb_sample(struct mlx5e_priv *priv, struct sk_buff *skb,
 650                                      struct mlx5_mapped_obj *mapped_obj,
 651                                      struct mlx5e_tc_update_priv *tc_priv)
 652 {
 653         if (!mlx5e_restore_tunnel(priv, skb, tc_priv, mapped_obj->sample.tunnel_id)) {
 654                 netdev_dbg(priv->netdev,
 655                            "Failed to restore tunnel info for sampled packet\n");
 656                 return;
 657         }
 658         mlx5e_tc_sample_skb(skb, mapped_obj);
 659         mlx5_rep_tc_post_napi_receive(tc_priv);
 660 }
 661
 662 static bool mlx5e_restore_skb_int_port(struct mlx5e_priv *priv, struct sk_buff *skb,
 663                                        struct mlx5_mapped_obj *mapped_obj,
 664                                        struct mlx5e_tc_update_priv *tc_priv,
 665                                        bool *forward_tx,
 666                                        u32 reg_c1)
 667 {
 668         u32 tunnel_id = (reg_c1 >> ESW_TUN_OFFSET) & TUNNEL_ID_MASK;
 669         struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
 670         struct mlx5_rep_uplink_priv *uplink_priv;
 671         struct mlx5e_rep_priv *uplink_rpriv;
 672
 673         /* Tunnel restore takes precedence over int port restore */
 674         if (tunnel_id)
 675                 return mlx5e_restore_tunnel(priv, skb, tc_priv, tunnel_id);
 676
 677         uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH);
 678         uplink_priv = &uplink_rpriv->uplink_priv;
 679
 680         if (mlx5e_tc_int_port_dev_fwd(uplink_priv->int_port_priv, skb,
 681                                       mapped_obj->int_port_metadata, forward_tx)) {
 682                 /* Set fwd_dev for future dev_put */
 683                 tc_priv->fwd_dev = skb->dev;
 684
 685                 return true;
 686         }
 687
 688         return false;
 689 }
 690
 691 void mlx5e_rep_tc_receive(struct mlx5_cqe64 *cqe, struct mlx5e_rq *rq,
 692                           struct sk_buff *skb)
 693 {
 694         u32 reg_c1 = be32_to_cpu(cqe->ft_metadata);
 695         struct mlx5e_tc_update_priv tc_priv = {};
 696         struct mlx5_mapped_obj mapped_obj;
 697         struct mlx5_eswitch *esw;
 698         bool forward_tx = false;
 699         struct mlx5e_priv *priv;
 700         u32 reg_c0;
 701         int err;
 702
 703         reg_c0 = (be32_to_cpu(cqe->sop_drop_qpn) & MLX5E_TC_FLOW_ID_MASK);
 704         if (!reg_c0 || reg_c0 == MLX5_FS_DEFAULT_FLOW_TAG)
 705                 goto forward;
 706
 707         /* If reg_c0 is not equal to the default flow tag then skb->mark
 708          * is not supported and must be reset back to 0.
 709          */
 710         skb->mark = 0;
 711
 712         priv = netdev_priv(skb->dev);
 713         esw = priv->mdev->priv.eswitch;
 714         err = mapping_find(esw->offloads.reg_c0_obj_pool, reg_c0, &mapped_obj);
 715         if (err) {
 716                 netdev_dbg(priv->netdev,
 717                            "Couldn't find mapped object for reg_c0: %d, err: %d\n",
 718                            reg_c0, err);
 719                 goto free_skb;
 720         }
 721
 722         if (mapped_obj.type == MLX5_MAPPED_OBJ_CHAIN) {
 723                 if (!mlx5e_restore_skb_chain(skb, mapped_obj.chain, reg_c1, &tc_priv) &&
 724                     !mlx5_ipsec_is_rx_flow(cqe))
 725                         goto free_skb;
 726         } else if (mapped_obj.type == MLX5_MAPPED_OBJ_SAMPLE) {
 727                 mlx5e_restore_skb_sample(priv, skb, &mapped_obj, &tc_priv);
 728                 goto free_skb;
 729         } else if (mapped_obj.type == MLX5_MAPPED_OBJ_INT_PORT_METADATA) {
 730                 if (!mlx5e_restore_skb_int_port(priv, skb, &mapped_obj, &tc_priv,
 731                                                 &forward_tx, reg_c1))
 732                         goto free_skb;
 733         } else {
 734                 netdev_dbg(priv->netdev, "Invalid mapped object type: %d\n", mapped_obj.type);
 735                 goto free_skb;
 736         }
 737
 738 forward:
 739         if (forward_tx)
 740                 dev_queue_xmit(skb);
 741         else
 742                 napi_gro_receive(rq->cq.napi, skb);
 743
 744         mlx5_rep_tc_post_napi_receive(&tc_priv);
 745
 746         return;
 747
 748 free_skb:
 749         dev_kfree_skb_any(skb);
 750 }