lib/netdev-tc-offloads.c

   1 /*
   2  * Copyright (c) 2016 Mellanox Technologies, Ltd.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16
  17 #include <config.h>
  18 #include "netdev-tc-offloads.h"
  19
  20 #include <errno.h>
  21 #include <linux/if_ether.h>
  22
  23 #include "dpif.h"
  24 #include "hash.h"
  25 #include "openvswitch/hmap.h"
  26 #include "openvswitch/match.h"
  27 #include "openvswitch/ofpbuf.h"
  28 #include "openvswitch/thread.h"
  29 #include "openvswitch/types.h"
  30 #include "openvswitch/vlog.h"
  31 #include "netdev-linux.h"
  32 #include "netlink.h"
  33 #include "netlink-socket.h"
  34 #include "odp-netlink.h"
  35 #include "tc.h"
  36 #include "unaligned.h"
  37 #include "util.h"
  38
  39 VLOG_DEFINE_THIS_MODULE(netdev_tc_offloads);
  40
  41 static struct vlog_rate_limit error_rl = VLOG_RATE_LIMIT_INIT(60, 5);
  42
  43 static struct hmap ufid_tc = HMAP_INITIALIZER(&ufid_tc);
  44 static struct ovs_mutex ufid_lock = OVS_MUTEX_INITIALIZER;
  45
  46 /**
  47  * struct ufid_tc_data - data entry for ufid_tc hmap.
  48  * @ufid_node: Element in @ufid_tc hash table by ufid key.
  49  * @tc_node: Element in @ufid_tc hash table by prio/handle/ifindex key.
  50  * @ufid: ufid assigned to the flow
  51  * @prio: tc priority
  52  * @handle: tc handle
  53  * @ifindex: netdev ifindex.
  54  * @netdev: netdev associated with the tc rule
  55  */
  56 struct ufid_tc_data {
  57     struct hmap_node ufid_node;
  58     struct hmap_node tc_node;
  59     ovs_u128 ufid;
  60     uint16_t prio;
  61     uint32_t handle;
  62     int ifindex;
  63     struct netdev *netdev;
  64 };
  65
  66 /* Remove matching ufid entry from ufid_tc hashmap. */
  67 static void
  68 del_ufid_tc_mapping(const ovs_u128 *ufid)
  69 {
  70     size_t ufid_hash = hash_bytes(ufid, sizeof *ufid, 0);
  71     struct ufid_tc_data *data;
  72
  73     ovs_mutex_lock(&ufid_lock);
  74     HMAP_FOR_EACH_WITH_HASH(data, ufid_node, ufid_hash, &ufid_tc) {
  75         if (ovs_u128_equals(*ufid, data->ufid)) {
  76             break;
  77         }
  78     }
  79
  80     if (!data) {
  81         ovs_mutex_unlock(&ufid_lock);
  82         return;
  83     }
  84
  85     hmap_remove(&ufid_tc, &data->ufid_node);
  86     hmap_remove(&ufid_tc, &data->tc_node);
  87     netdev_close(data->netdev);
  88     free(data);
  89     ovs_mutex_unlock(&ufid_lock);
  90 }
  91
  92 /* Add ufid entry to ufid_tc hashmap.
  93  * If entry exists already it will be replaced. */
  94 static void
  95 add_ufid_tc_mapping(const ovs_u128 *ufid, int prio, int handle,
  96                     struct netdev *netdev, int ifindex)
  97 {
  98     size_t ufid_hash = hash_bytes(ufid, sizeof *ufid, 0);
  99     size_t tc_hash = hash_int(hash_int(prio, handle), ifindex);
 100     struct ufid_tc_data *new_data = xzalloc(sizeof *new_data);
 101
 102     del_ufid_tc_mapping(ufid);
 103
 104     new_data->ufid = *ufid;
 105     new_data->prio = prio;
 106     new_data->handle = handle;
 107     new_data->netdev = netdev_ref(netdev);
 108     new_data->ifindex = ifindex;
 109
 110     ovs_mutex_lock(&ufid_lock);
 111     hmap_insert(&ufid_tc, &new_data->ufid_node, ufid_hash);
 112     hmap_insert(&ufid_tc, &new_data->tc_node, tc_hash);
 113     ovs_mutex_unlock(&ufid_lock);
 114 }
 115
 116 /* Get ufid from ufid_tc hashmap.
 117  *
 118  * If netdev output param is not NULL then the function will return
 119  * associated netdev on success and a refcount is taken on that netdev.
 120  * The caller is then responsible to close the netdev.
 121  *
 122  * Returns handle if successful and fill prio and netdev for that ufid.
 123  * Otherwise returns 0.
 124  */
 125 static int
 126 get_ufid_tc_mapping(const ovs_u128 *ufid, int *prio, struct netdev **netdev)
 127 {
 128     size_t ufid_hash = hash_bytes(ufid, sizeof *ufid, 0);
 129     struct ufid_tc_data *data;
 130     int handle = 0;
 131
 132     ovs_mutex_lock(&ufid_lock);
 133     HMAP_FOR_EACH_WITH_HASH(data, ufid_node, ufid_hash, &ufid_tc) {
 134         if (ovs_u128_equals(*ufid, data->ufid)) {
 135             if (prio) {
 136                 *prio = data->prio;
 137             }
 138             if (netdev) {
 139                 *netdev = netdev_ref(data->netdev);
 140             }
 141             handle = data->handle;
 142             break;
 143         }
 144     }
 145     ovs_mutex_unlock(&ufid_lock);
 146
 147     return handle;
 148 }
 149
 150 /* Find ufid entry in ufid_tc hashmap using prio, handle and netdev.
 151  * The result is saved in ufid.
 152  *
 153  * Returns true on success.
 154  */
 155 static bool
 156 find_ufid(int prio, int handle, struct netdev *netdev, ovs_u128 *ufid)
 157 {
 158     int ifindex = netdev_get_ifindex(netdev);
 159     struct ufid_tc_data *data;
 160     size_t tc_hash = hash_int(hash_int(prio, handle), ifindex);
 161
 162     ovs_mutex_lock(&ufid_lock);
 163     HMAP_FOR_EACH_WITH_HASH(data, tc_node, tc_hash,  &ufid_tc) {
 164         if (data->prio == prio && data->handle == handle
 165             && data->ifindex == ifindex) {
 166             *ufid = data->ufid;
 167             break;
 168         }
 169     }
 170     ovs_mutex_unlock(&ufid_lock);
 171
 172     return (data != NULL);
 173 }
 174
 175 struct prio_map_data {
 176     struct hmap_node node;
 177     struct tc_flower_key mask;
 178     ovs_be16 protocol;
 179     uint16_t prio;
 180 };
 181
 182 /* Get free prio for tc flower
 183  * If prio is already allocated for mask/eth_type combination then return it.
 184  * If not assign new prio.
 185  *
 186  * Return prio on success or 0 if we are out of prios.
 187  */
 188 static uint16_t
 189 get_prio_for_tc_flower(struct tc_flower *flower)
 190 {
 191     static struct hmap prios = HMAP_INITIALIZER(&prios);
 192     static struct ovs_mutex prios_lock = OVS_MUTEX_INITIALIZER;
 193     static uint16_t last_prio = 0;
 194     size_t key_len = sizeof(struct tc_flower_key);
 195     size_t hash = hash_bytes(&flower->mask, key_len,
 196                              (OVS_FORCE uint32_t) flower->key.eth_type);
 197     struct prio_map_data *data;
 198     struct prio_map_data *new_data;
 199
 200     /* We can use the same prio for same mask/eth combination but must have
 201      * different prio if not. Flower classifier will reject same prio for
 202      * different mask/eth combination. */
 203     ovs_mutex_lock(&prios_lock);
 204     HMAP_FOR_EACH_WITH_HASH(data, node, hash, &prios) {
 205         if (!memcmp(&flower->mask, &data->mask, key_len)
 206             && data->protocol == flower->key.eth_type) {
 207             ovs_mutex_unlock(&prios_lock);
 208             return data->prio;
 209         }
 210     }
 211
 212     if (last_prio == UINT16_MAX) {
 213         /* last_prio can overflow if there will be many different kinds of
 214          * flows which shouldn't happen organically. */
 215         ovs_mutex_unlock(&prios_lock);
 216         return 0;
 217     }
 218
 219     new_data = xzalloc(sizeof *new_data);
 220     memcpy(&new_data->mask, &flower->mask, key_len);
 221     new_data->prio = ++last_prio;
 222     new_data->protocol = flower->key.eth_type;
 223     hmap_insert(&prios, &new_data->node, hash);
 224     ovs_mutex_unlock(&prios_lock);
 225
 226     return new_data->prio;
 227 }
 228
 229 int
 230 netdev_tc_flow_flush(struct netdev *netdev)
 231 {
 232     int ifindex = netdev_get_ifindex(netdev);
 233
 234     if (ifindex < 0) {
 235         VLOG_ERR_RL(&error_rl, "failed to get ifindex for %s: %s",
 236                     netdev_get_name(netdev), ovs_strerror(-ifindex));
 237         return -ifindex;
 238     }
 239
 240     return tc_flush(ifindex);
 241 }
 242
 243 int
 244 netdev_tc_flow_dump_create(struct netdev *netdev,
 245                            struct netdev_flow_dump **dump_out)
 246 {
 247     struct netdev_flow_dump *dump;
 248     int ifindex;
 249
 250     ifindex = netdev_get_ifindex(netdev);
 251     if (ifindex < 0) {
 252         VLOG_ERR_RL(&error_rl, "failed to get ifindex for %s: %s",
 253                     netdev_get_name(netdev), ovs_strerror(-ifindex));
 254         return -ifindex;
 255     }
 256
 257     dump = xzalloc(sizeof *dump);
 258     dump->nl_dump = xzalloc(sizeof *dump->nl_dump);
 259     dump->netdev = netdev_ref(netdev);
 260     tc_dump_flower_start(ifindex, dump->nl_dump);
 261
 262     *dump_out = dump;
 263
 264     return 0;
 265 }
 266
 267 int
 268 netdev_tc_flow_dump_destroy(struct netdev_flow_dump *dump)
 269 {
 270     nl_dump_done(dump->nl_dump);
 271     netdev_close(dump->netdev);
 272     free(dump->nl_dump);
 273     free(dump);
 274     return 0;
 275 }
 276
 277 static int
 278 parse_tc_flower_to_match(struct tc_flower *flower,
 279                          struct match *match,
 280                          struct nlattr **actions,
 281                          struct dpif_flow_stats *stats,
 282                          struct ofpbuf *buf) {
 283     size_t act_off;
 284     struct tc_flower_key *key = &flower->key;
 285     struct tc_flower_key *mask = &flower->mask;
 286     odp_port_t outport = 0;
 287
 288     if (flower->ifindex_out) {
 289         outport = netdev_ifindex_to_odp_port(flower->ifindex_out);
 290         if (!outport) {
 291             return ENOENT;
 292         }
 293     }
 294
 295     ofpbuf_clear(buf);
 296
 297     match_init_catchall(match);
 298     match_set_dl_src_masked(match, key->src_mac, mask->src_mac);
 299     match_set_dl_dst_masked(match, key->dst_mac, mask->dst_mac);
 300
 301     if (key->eth_type == htons(ETH_TYPE_VLAN)) {
 302         match_set_dl_vlan(match, htons(key->vlan_id));
 303         match_set_dl_vlan_pcp(match, key->vlan_prio);
 304         match_set_dl_type(match, key->encap_eth_type);
 305         flow_fix_vlan_tpid(&match->flow);
 306     } else {
 307         match_set_dl_type(match, key->eth_type);
 308     }
 309
 310     if (is_ip_any(&match->flow)) {
 311         if (key->ip_proto) {
 312             match_set_nw_proto(match, key->ip_proto);
 313         }
 314
 315         match_set_nw_src_masked(match, key->ipv4.ipv4_src, mask->ipv4.ipv4_src);
 316         match_set_nw_dst_masked(match, key->ipv4.ipv4_dst, mask->ipv4.ipv4_dst);
 317
 318         match_set_ipv6_src_masked(match,
 319                                   &key->ipv6.ipv6_src, &mask->ipv6.ipv6_src);
 320         match_set_ipv6_dst_masked(match,
 321                                   &key->ipv6.ipv6_dst, &mask->ipv6.ipv6_dst);
 322
 323         if (key->ip_proto == IPPROTO_TCP) {
 324             match_set_tp_dst_masked(match, key->tcp_dst, mask->tcp_dst);
 325             match_set_tp_src_masked(match, key->tcp_src, mask->tcp_src);
 326         } else if (key->ip_proto == IPPROTO_UDP) {
 327             match_set_tp_dst_masked(match, key->udp_dst, mask->udp_dst);
 328             match_set_tp_src_masked(match, key->udp_src, mask->udp_src);
 329         }
 330     }
 331
 332     if (flower->tunnel.tunnel) {
 333         match_set_tun_id(match, flower->tunnel.id);
 334         if (flower->tunnel.ipv4.ipv4_dst) {
 335             match_set_tun_src(match, flower->tunnel.ipv4.ipv4_src);
 336             match_set_tun_dst(match, flower->tunnel.ipv4.ipv4_dst);
 337         } else if (!is_all_zeros(&flower->tunnel.ipv6.ipv6_dst,
 338                    sizeof flower->tunnel.ipv6.ipv6_dst)) {
 339             match_set_tun_ipv6_src(match, &flower->tunnel.ipv6.ipv6_src);
 340             match_set_tun_ipv6_dst(match, &flower->tunnel.ipv6.ipv6_dst);
 341         }
 342         if (flower->tunnel.tp_dst) {
 343             match_set_tun_tp_dst(match, flower->tunnel.tp_dst);
 344         }
 345     }
 346
 347     act_off = nl_msg_start_nested(buf, OVS_FLOW_ATTR_ACTIONS);
 348     {
 349         if (flower->vlan_pop) {
 350             nl_msg_put_flag(buf, OVS_ACTION_ATTR_POP_VLAN);
 351         }
 352
 353         if (flower->vlan_push_id || flower->vlan_push_prio) {
 354             struct ovs_action_push_vlan *push;
 355             push = nl_msg_put_unspec_zero(buf, OVS_ACTION_ATTR_PUSH_VLAN,
 356                                           sizeof *push);
 357
 358             push->vlan_tpid = htons(ETH_TYPE_VLAN);
 359             push->vlan_tci = htons(flower->vlan_push_id
 360                                    | (flower->vlan_push_prio << 13)
 361                                    | VLAN_CFI);
 362         }
 363
 364         if (flower->set.set) {
 365             size_t set_offset = nl_msg_start_nested(buf, OVS_ACTION_ATTR_SET);
 366             size_t tunnel_offset =
 367                 nl_msg_start_nested(buf, OVS_KEY_ATTR_TUNNEL);
 368
 369             nl_msg_put_be64(buf, OVS_TUNNEL_KEY_ATTR_ID, flower->set.id);
 370             if (flower->set.ipv4.ipv4_src) {
 371                 nl_msg_put_be32(buf, OVS_TUNNEL_KEY_ATTR_IPV4_SRC,
 372                                 flower->set.ipv4.ipv4_src);
 373             }
 374             if (flower->set.ipv4.ipv4_dst) {
 375                 nl_msg_put_be32(buf, OVS_TUNNEL_KEY_ATTR_IPV4_DST,
 376                                 flower->set.ipv4.ipv4_dst);
 377             }
 378             if (!is_all_zeros(&flower->set.ipv6.ipv6_src,
 379                               sizeof flower->set.ipv6.ipv6_src)) {
 380                 nl_msg_put_in6_addr(buf, OVS_TUNNEL_KEY_ATTR_IPV6_SRC,
 381                                     &flower->set.ipv6.ipv6_src);
 382             }
 383             if (!is_all_zeros(&flower->set.ipv6.ipv6_dst,
 384                               sizeof flower->set.ipv6.ipv6_dst)) {
 385                 nl_msg_put_in6_addr(buf, OVS_TUNNEL_KEY_ATTR_IPV6_DST,
 386                                     &flower->set.ipv6.ipv6_dst);
 387             }
 388             nl_msg_put_be16(buf, OVS_TUNNEL_KEY_ATTR_TP_DST,
 389                             flower->set.tp_dst);
 390
 391             nl_msg_end_nested(buf, tunnel_offset);
 392             nl_msg_end_nested(buf, set_offset);
 393         }
 394
 395         if (flower->ifindex_out > 0) {
 396             nl_msg_put_u32(buf, OVS_ACTION_ATTR_OUTPUT, odp_to_u32(outport));
 397         }
 398
 399     }
 400     nl_msg_end_nested(buf, act_off);
 401
 402     *actions = ofpbuf_at_assert(buf, act_off, sizeof(struct nlattr));
 403
 404     if (stats) {
 405         memset(stats, 0, sizeof *stats);
 406         stats->n_packets = get_32aligned_u64(&flower->stats.n_packets);
 407         stats->n_bytes = get_32aligned_u64(&flower->stats.n_bytes);
 408         stats->used = flower->lastused;
 409     }
 410
 411     return 0;
 412 }
 413
 414 bool
 415 netdev_tc_flow_dump_next(struct netdev_flow_dump *dump,
 416                          struct match *match,
 417                          struct nlattr **actions,
 418                          struct dpif_flow_stats *stats,
 419                          ovs_u128 *ufid,
 420                          struct ofpbuf *rbuffer,
 421                          struct ofpbuf *wbuffer)
 422 {
 423     struct ofpbuf nl_flow;
 424
 425     while (nl_dump_next(dump->nl_dump, &nl_flow, rbuffer)) {
 426         struct tc_flower flower;
 427         struct netdev *netdev = dump->netdev;
 428
 429         if (parse_netlink_to_tc_flower(&nl_flow, &flower)) {
 430             continue;
 431         }
 432
 433         if (parse_tc_flower_to_match(&flower, match, actions, stats,
 434                                      wbuffer)) {
 435             continue;
 436         }
 437
 438         if (flower.act_cookie.len) {
 439             *ufid = *((ovs_u128 *) flower.act_cookie.data);
 440         } else if (!find_ufid(flower.prio, flower.handle, netdev, ufid)) {
 441             continue;
 442         }
 443
 444         match->wc.masks.in_port.odp_port = u32_to_odp(UINT32_MAX);
 445         match->flow.in_port.odp_port = dump->port;
 446
 447         return true;
 448     }
 449
 450     return false;
 451 }
 452
 453 static int
 454 parse_put_flow_set_action(struct tc_flower *flower, const struct nlattr *set,
 455                           size_t set_len)
 456 {
 457     static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
 458     const struct nlattr *set_attr;
 459     size_t set_left;
 460
 461     NL_ATTR_FOR_EACH_UNSAFE(set_attr, set_left, set, set_len) {
 462         if (nl_attr_type(set_attr) == OVS_KEY_ATTR_TUNNEL) {
 463             const struct nlattr *tunnel = nl_attr_get(set_attr);
 464             const size_t tunnel_len = nl_attr_get_size(set_attr);
 465             const struct nlattr *tun_attr;
 466             size_t tun_left;
 467
 468             flower->set.set = true;
 469             NL_ATTR_FOR_EACH_UNSAFE(tun_attr, tun_left, tunnel, tunnel_len) {
 470                 switch (nl_attr_type(tun_attr)) {
 471                 case OVS_TUNNEL_KEY_ATTR_ID: {
 472                     flower->set.id = nl_attr_get_be64(tun_attr);
 473                 }
 474                 break;
 475                 case OVS_TUNNEL_KEY_ATTR_IPV4_SRC: {
 476                     flower->set.ipv4.ipv4_src = nl_attr_get_be32(tun_attr);
 477                 }
 478                 break;
 479                 case OVS_TUNNEL_KEY_ATTR_IPV4_DST: {
 480                     flower->set.ipv4.ipv4_dst = nl_attr_get_be32(tun_attr);
 481                 }
 482                 break;
 483                 case OVS_TUNNEL_KEY_ATTR_IPV6_SRC: {
 484                     flower->set.ipv6.ipv6_src =
 485                         nl_attr_get_in6_addr(tun_attr);
 486                 }
 487                 break;
 488                 case OVS_TUNNEL_KEY_ATTR_IPV6_DST: {
 489                     flower->set.ipv6.ipv6_dst =
 490                         nl_attr_get_in6_addr(tun_attr);
 491                 }
 492                 break;
 493                 case OVS_TUNNEL_KEY_ATTR_TP_SRC: {
 494                     flower->set.tp_src = nl_attr_get_be16(tun_attr);
 495                 }
 496                 break;
 497                 case OVS_TUNNEL_KEY_ATTR_TP_DST: {
 498                     flower->set.tp_dst = nl_attr_get_be16(tun_attr);
 499                 }
 500                 break;
 501                 }
 502             }
 503         } else {
 504             VLOG_DBG_RL(&rl, "unsupported set action type: %d",
 505                         nl_attr_type(set_attr));
 506             return EOPNOTSUPP;
 507         }
 508     }
 509     return 0;
 510 }
 511
 512 static int
 513 test_key_and_mask(struct match *match)
 514 {
 515     static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
 516     const struct flow *key = &match->flow;
 517     struct flow *mask = &match->wc.masks;
 518
 519     if (mask->pkt_mark) {
 520         VLOG_DBG_RL(&rl, "offloading attribute pkt_mark isn't supported");
 521         return EOPNOTSUPP;
 522     }
 523
 524     if (mask->recirc_id && key->recirc_id) {
 525         VLOG_DBG_RL(&rl, "offloading attribute recirc_id isn't supported");
 526         return EOPNOTSUPP;
 527     }
 528     mask->recirc_id = 0;
 529
 530     if (mask->dp_hash) {
 531         VLOG_DBG_RL(&rl, "offloading attribute dp_hash isn't supported");
 532         return EOPNOTSUPP;
 533     }
 534
 535     if (mask->conj_id) {
 536         VLOG_DBG_RL(&rl, "offloading attribute conj_id isn't supported");
 537         return EOPNOTSUPP;
 538     }
 539
 540     if (mask->skb_priority) {
 541         VLOG_DBG_RL(&rl, "offloading attribute skb_priority isn't supported");
 542         return EOPNOTSUPP;
 543     }
 544
 545     if (mask->actset_output) {
 546         VLOG_DBG_RL(&rl,
 547                     "offloading attribute actset_output isn't supported");
 548         return EOPNOTSUPP;
 549     }
 550
 551     if (mask->ct_state) {
 552         VLOG_DBG_RL(&rl, "offloading attribute ct_state isn't supported");
 553         return EOPNOTSUPP;
 554     }
 555
 556     if (mask->ct_zone) {
 557         VLOG_DBG_RL(&rl, "offloading attribute ct_zone isn't supported");
 558         return EOPNOTSUPP;
 559     }
 560
 561     if (mask->ct_mark) {
 562         VLOG_DBG_RL(&rl, "offloading attribute ct_mark isn't supported");
 563         return EOPNOTSUPP;
 564     }
 565
 566     if (mask->packet_type && key->packet_type) {
 567         VLOG_DBG_RL(&rl, "offloading attribute packet_type isn't supported");
 568         return EOPNOTSUPP;
 569     }
 570     mask->packet_type = 0;
 571
 572     if (!ovs_u128_is_zero(mask->ct_label)) {
 573         VLOG_DBG_RL(&rl, "offloading attribute ct_label isn't supported");
 574         return EOPNOTSUPP;
 575     }
 576
 577     for (int i = 0; i < FLOW_N_REGS; i++) {
 578         if (mask->regs[i]) {
 579             VLOG_DBG_RL(&rl,
 580                         "offloading attribute regs[%d] isn't supported", i);
 581             return EOPNOTSUPP;
 582         }
 583     }
 584
 585     if (mask->metadata) {
 586         VLOG_DBG_RL(&rl, "offloading attribute metadata isn't supported");
 587         return EOPNOTSUPP;
 588     }
 589
 590     if (mask->nw_tos) {
 591         VLOG_DBG_RL(&rl, "offloading attribute nw_tos isn't supported");
 592         return EOPNOTSUPP;
 593     }
 594
 595     if (mask->nw_ttl) {
 596         VLOG_DBG_RL(&rl, "offloading attribute nw_ttl isn't supported");
 597         return EOPNOTSUPP;
 598     }
 599
 600     if (mask->nw_frag) {
 601         VLOG_DBG_RL(&rl, "offloading attribute nw_frag isn't supported");
 602         return EOPNOTSUPP;
 603     }
 604
 605     for (int i = 0; i < FLOW_MAX_MPLS_LABELS; i++) {
 606         if (mask->mpls_lse[i]) {
 607             VLOG_DBG_RL(&rl, "offloading attribute mpls_lse isn't supported");
 608             return EOPNOTSUPP;
 609         }
 610     }
 611
 612     if (key->dl_type == htons(ETH_TYPE_IP) &&
 613         key->nw_proto == IPPROTO_ICMP) {
 614         if (mask->tp_src) {
 615             VLOG_DBG_RL(&rl,
 616                         "offloading attribute icmp_type isn't supported");
 617             return EOPNOTSUPP;
 618         }
 619         if (mask->tp_dst) {
 620             VLOG_DBG_RL(&rl,
 621                         "offloading attribute icmp_code isn't supported");
 622             return EOPNOTSUPP;
 623         }
 624     } else if (key->dl_type == htons(ETH_TYPE_IP) &&
 625                key->nw_proto == IPPROTO_IGMP) {
 626         if (mask->tp_src) {
 627             VLOG_DBG_RL(&rl,
 628                         "offloading attribute igmp_type isn't supported");
 629             return EOPNOTSUPP;
 630         }
 631         if (mask->tp_dst) {
 632             VLOG_DBG_RL(&rl,
 633                         "offloading attribute igmp_code isn't supported");
 634             return EOPNOTSUPP;
 635         }
 636     } else if (key->dl_type == htons(ETH_TYPE_IPV6) &&
 637                key->nw_proto == IPPROTO_ICMPV6) {
 638         if (mask->tp_src) {
 639             VLOG_DBG_RL(&rl,
 640                         "offloading attribute icmp_type isn't supported");
 641             return EOPNOTSUPP;
 642         }
 643         if (mask->tp_dst) {
 644             VLOG_DBG_RL(&rl,
 645                         "offloading attribute icmp_code isn't supported");
 646             return EOPNOTSUPP;
 647         }
 648     }
 649     if (is_ip_any(key) && key->nw_proto == IPPROTO_TCP && mask->tcp_flags) {
 650         if (mask->tcp_flags) {
 651             VLOG_DBG_RL(&rl,
 652                         "offloading attribute tcp_flags isn't supported");
 653             return EOPNOTSUPP;
 654         }
 655     }
 656
 657     if (!is_all_zeros(mask, sizeof *mask)) {
 658         VLOG_DBG_RL(&rl, "offloading isn't supported, unknown attribute");
 659         return EOPNOTSUPP;
 660     }
 661
 662     return 0;
 663 }
 664
 665 int
 666 netdev_tc_flow_put(struct netdev *netdev, struct match *match,
 667                    struct nlattr *actions, size_t actions_len,
 668                    const ovs_u128 *ufid, struct offload_info *info,
 669                    struct dpif_flow_stats *stats OVS_UNUSED)
 670 {
 671     static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
 672     struct tc_flower flower;
 673     const struct flow *key = &match->flow;
 674     struct flow *mask = &match->wc.masks;
 675     const struct flow_tnl *tnl = &match->flow.tunnel;
 676     struct nlattr *nla;
 677     size_t left;
 678     int prio = 0;
 679     int handle;
 680     int ifindex;
 681     int err;
 682
 683     ifindex = netdev_get_ifindex(netdev);
 684     if (ifindex < 0) {
 685         VLOG_ERR_RL(&error_rl, "failed to get ifindex for %s: %s",
 686                     netdev_get_name(netdev), ovs_strerror(-ifindex));
 687         return -ifindex;
 688     }
 689
 690     memset(&flower, 0, sizeof flower);
 691
 692     if (tnl->tun_id) {
 693         VLOG_DBG_RL(&rl,
 694                     "tunnel: id %#" PRIx64 " src " IP_FMT
 695                     " dst " IP_FMT " tp_src %d tp_dst %d",
 696                     ntohll(tnl->tun_id),
 697                     IP_ARGS(tnl->ip_src), IP_ARGS(tnl->ip_dst),
 698                     ntohs(tnl->tp_src), ntohs(tnl->tp_dst));
 699         flower.tunnel.id = tnl->tun_id;
 700         flower.tunnel.ipv4.ipv4_src = tnl->ip_src;
 701         flower.tunnel.ipv4.ipv4_dst = tnl->ip_dst;
 702         flower.tunnel.ipv6.ipv6_src = tnl->ipv6_src;
 703         flower.tunnel.ipv6.ipv6_dst = tnl->ipv6_dst;
 704         flower.tunnel.tp_src = tnl->tp_src;
 705         flower.tunnel.tp_dst = tnl->tp_dst;
 706         flower.tunnel.tunnel = true;
 707
 708         memset(&mask->tunnel, 0, sizeof mask->tunnel);
 709     }
 710
 711     flower.key.eth_type = key->dl_type;
 712     flower.mask.eth_type = mask->dl_type;
 713
 714     if (mask->vlans[0].tci) {
 715         ovs_be16 vid_mask = mask->vlans[0].tci & htons(VLAN_VID_MASK);
 716         ovs_be16 pcp_mask = mask->vlans[0].tci & htons(VLAN_PCP_MASK);
 717         ovs_be16 cfi = mask->vlans[0].tci & htons(VLAN_CFI);
 718
 719         if (cfi && key->vlans[0].tci & htons(VLAN_CFI)
 720             && (!vid_mask || vid_mask == htons(VLAN_VID_MASK))
 721             && (!pcp_mask || pcp_mask == htons(VLAN_PCP_MASK))
 722             && (vid_mask || pcp_mask)) {
 723             if (vid_mask) {
 724                 flower.key.vlan_id = vlan_tci_to_vid(key->vlans[0].tci);
 725                 VLOG_DBG_RL(&rl, "vlan_id: %d\n", flower.key.vlan_id);
 726             }
 727             if (pcp_mask) {
 728                 flower.key.vlan_prio = vlan_tci_to_pcp(key->vlans[0].tci);
 729                 VLOG_DBG_RL(&rl, "vlan_prio: %d\n", flower.key.vlan_prio);
 730             }
 731             flower.key.encap_eth_type = flower.key.eth_type;
 732             flower.key.eth_type = htons(ETH_TYPE_VLAN);
 733         } else if (mask->vlans[0].tci == htons(0xffff) &&
 734                    ntohs(key->vlans[0].tci) == 0) {
 735             /* exact && no vlan */
 736         } else {
 737             /* partial mask */
 738             return EOPNOTSUPP;
 739         }
 740     } else if (mask->vlans[1].tci) {
 741         return EOPNOTSUPP;
 742     }
 743     memset(mask->vlans, 0, sizeof mask->vlans);
 744
 745     flower.key.dst_mac = key->dl_dst;
 746     flower.mask.dst_mac = mask->dl_dst;
 747     flower.key.src_mac = key->dl_src;
 748     flower.mask.src_mac = mask->dl_src;
 749     memset(&mask->dl_dst, 0, sizeof mask->dl_dst);
 750     memset(&mask->dl_src, 0, sizeof mask->dl_src);
 751     mask->dl_type = 0;
 752     mask->in_port.odp_port = 0;
 753
 754     if (is_ip_any(key)) {
 755         flower.key.ip_proto = key->nw_proto;
 756         flower.mask.ip_proto = mask->nw_proto;
 757
 758         if (key->nw_proto == IPPROTO_TCP) {
 759             flower.key.tcp_dst = key->tp_dst;
 760             flower.mask.tcp_dst = mask->tp_dst;
 761             flower.key.tcp_src = key->tp_src;
 762             flower.mask.tcp_src = mask->tp_src;
 763             mask->tp_src = 0;
 764             mask->tp_dst = 0;
 765         } else if (key->nw_proto == IPPROTO_UDP) {
 766             flower.key.udp_dst = key->tp_dst;
 767             flower.mask.udp_dst = mask->tp_dst;
 768             flower.key.udp_src = key->tp_src;
 769             flower.mask.udp_src = mask->tp_src;
 770             mask->tp_src = 0;
 771             mask->tp_dst = 0;
 772         } else if (key->nw_proto == IPPROTO_SCTP) {
 773             flower.key.sctp_dst = key->tp_dst;
 774             flower.mask.sctp_dst = mask->tp_dst;
 775             flower.key.sctp_src = key->tp_src;
 776             flower.mask.sctp_src = mask->tp_src;
 777             mask->tp_src = 0;
 778             mask->tp_dst = 0;
 779         }
 780
 781         mask->nw_frag = 0;
 782         mask->nw_tos = 0;
 783         mask->nw_proto = 0;
 784
 785         if (key->dl_type == htons(ETH_P_IP)) {
 786             flower.key.ipv4.ipv4_src = key->nw_src;
 787             flower.mask.ipv4.ipv4_src = mask->nw_src;
 788             flower.key.ipv4.ipv4_dst = key->nw_dst;
 789             flower.mask.ipv4.ipv4_dst = mask->nw_dst;
 790             mask->nw_src = 0;
 791             mask->nw_dst = 0;
 792         } else if (key->dl_type == htons(ETH_P_IPV6)) {
 793             flower.key.ipv6.ipv6_src = key->ipv6_src;
 794             flower.mask.ipv6.ipv6_src = mask->ipv6_src;
 795             flower.key.ipv6.ipv6_dst = key->ipv6_dst;
 796             flower.mask.ipv6.ipv6_dst = mask->ipv6_dst;
 797             memset(&mask->ipv6_src, 0, sizeof mask->ipv6_src);
 798             memset(&mask->ipv6_dst, 0, sizeof mask->ipv6_dst);
 799         }
 800     }
 801
 802     err = test_key_and_mask(match);
 803     if (err) {
 804         return err;
 805     }
 806
 807     NL_ATTR_FOR_EACH(nla, left, actions, actions_len) {
 808         if (nl_attr_type(nla) == OVS_ACTION_ATTR_OUTPUT) {
 809             odp_port_t port = nl_attr_get_odp_port(nla);
 810             struct netdev *outdev = netdev_ports_get(port, info->dpif_class);
 811
 812             flower.ifindex_out = netdev_get_ifindex(outdev);
 813             flower.set.tp_dst = info->tp_dst_port;
 814             netdev_close(outdev);
 815         } else if (nl_attr_type(nla) == OVS_ACTION_ATTR_PUSH_VLAN) {
 816             const struct ovs_action_push_vlan *vlan_push = nl_attr_get(nla);
 817
 818             flower.vlan_push_id = vlan_tci_to_vid(vlan_push->vlan_tci);
 819             flower.vlan_push_prio = vlan_tci_to_pcp(vlan_push->vlan_tci);
 820         } else if (nl_attr_type(nla) == OVS_ACTION_ATTR_POP_VLAN) {
 821             flower.vlan_pop = 1;
 822         } else if (nl_attr_type(nla) == OVS_ACTION_ATTR_SET) {
 823             const struct nlattr *set = nl_attr_get(nla);
 824             const size_t set_len = nl_attr_get_size(nla);
 825
 826             err = parse_put_flow_set_action(&flower, set, set_len);
 827             if (err) {
 828                 return err;
 829             }
 830         } else {
 831             VLOG_DBG_RL(&rl, "unsupported put action type: %d",
 832                         nl_attr_type(nla));
 833             return EOPNOTSUPP;
 834         }
 835     }
 836
 837     handle = get_ufid_tc_mapping(ufid, &prio, NULL);
 838     if (handle && prio) {
 839         VLOG_DBG_RL(&rl, "updating old handle: %d prio: %d", handle, prio);
 840         tc_del_filter(ifindex, prio, handle);
 841     }
 842
 843     if (!prio) {
 844         prio = get_prio_for_tc_flower(&flower);
 845         if (prio == 0) {
 846             VLOG_ERR_RL(&rl, "couldn't get tc prio: %s", ovs_strerror(ENOSPC));
 847             return ENOSPC;
 848         }
 849     }
 850
 851     flower.act_cookie.data = ufid;
 852     flower.act_cookie.len = sizeof *ufid;
 853
 854     err = tc_replace_flower(ifindex, prio, handle, &flower);
 855     if (!err) {
 856         add_ufid_tc_mapping(ufid, flower.prio, flower.handle, netdev, ifindex);
 857     }
 858
 859     return err;
 860 }
 861
 862 int
 863 netdev_tc_flow_get(struct netdev *netdev OVS_UNUSED,
 864                    struct match *match,
 865                    struct nlattr **actions,
 866                    const ovs_u128 *ufid,
 867                    struct dpif_flow_stats *stats,
 868                    struct ofpbuf *buf)
 869 {
 870     static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(5, 20);
 871     struct netdev *dev;
 872     struct tc_flower flower;
 873     odp_port_t in_port;
 874     int prio = 0;
 875     int ifindex;
 876     int handle;
 877     int err;
 878
 879     handle = get_ufid_tc_mapping(ufid, &prio, &dev);
 880     if (!handle) {
 881         return ENOENT;
 882     }
 883
 884     ifindex = netdev_get_ifindex(dev);
 885     if (ifindex < 0) {
 886         VLOG_ERR_RL(&error_rl, "failed to get ifindex for %s: %s",
 887                     netdev_get_name(dev), ovs_strerror(-ifindex));
 888         netdev_close(dev);
 889         return -ifindex;
 890     }
 891
 892     VLOG_DBG_RL(&rl, "flow get (dev %s prio %d handle %d)",
 893                 netdev_get_name(dev), prio, handle);
 894     err = tc_get_flower(ifindex, prio, handle, &flower);
 895     netdev_close(dev);
 896     if (err) {
 897         VLOG_ERR_RL(&error_rl, "flow get failed (dev %s prio %d handle %d): %s",
 898                     netdev_get_name(dev), prio, handle, ovs_strerror(err));
 899         return err;
 900     }
 901
 902     in_port = netdev_ifindex_to_odp_port(ifindex);
 903     parse_tc_flower_to_match(&flower, match, actions, stats, buf);
 904
 905     match->wc.masks.in_port.odp_port = u32_to_odp(UINT32_MAX);
 906     match->flow.in_port.odp_port = in_port;
 907
 908     return 0;
 909 }
 910
 911 int
 912 netdev_tc_flow_del(struct netdev *netdev OVS_UNUSED,
 913                    const ovs_u128 *ufid,
 914                    struct dpif_flow_stats *stats)
 915 {
 916     struct netdev *dev;
 917     int prio = 0;
 918     int ifindex;
 919     int handle;
 920     int error;
 921
 922     handle = get_ufid_tc_mapping(ufid, &prio, &dev);
 923     if (!handle) {
 924         return ENOENT;
 925     }
 926
 927     ifindex = netdev_get_ifindex(dev);
 928     if (ifindex < 0) {
 929         VLOG_ERR_RL(&error_rl, "failed to get ifindex for %s: %s",
 930                     netdev_get_name(dev), ovs_strerror(-ifindex));
 931         netdev_close(dev);
 932         return -ifindex;
 933     }
 934
 935     error = tc_del_filter(ifindex, prio, handle);
 936     del_ufid_tc_mapping(ufid);
 937
 938     netdev_close(dev);
 939
 940     if (stats) {
 941         memset(stats, 0, sizeof *stats);
 942     }
 943     return error;
 944 }
 945
 946 int
 947 netdev_tc_init_flow_api(struct netdev *netdev)
 948 {
 949     int ifindex;
 950     int error;
 951
 952     ifindex = netdev_get_ifindex(netdev);
 953     if (ifindex < 0) {
 954         VLOG_ERR_RL(&error_rl, "failed to get ifindex for %s: %s",
 955                     netdev_get_name(netdev), ovs_strerror(-ifindex));
 956         return -ifindex;
 957     }
 958
 959     error = tc_add_del_ingress_qdisc(ifindex, true);
 960
 961     if (error && error != EEXIST) {
 962         VLOG_ERR("failed adding ingress qdisc required for offloading: %s",
 963                  ovs_strerror(error));
 964         return error;
 965     }
 966
 967     VLOG_INFO("added ingress qdisc to %s", netdev_get_name(netdev));
 968
 969     return 0;
 970 }