lib/odp-execute.c

   1 /*
   2  * Copyright (c) 2009, 2010, 2011, 2012, 2013, 2014, 2015 Nicira, Inc.
   3  * Copyright (c) 2013 Simon Horman
   4  *
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <config.h>
  19 #include "odp-execute.h"
  20 #include <sys/types.h>
  21 #include <netinet/in.h>
  22 #include <arpa/inet.h>
  23 #include <netinet/icmp6.h>
  24 #include <netinet/ip6.h>
  25 #include <stdlib.h>
  26 #include <string.h>
  27
  28 #include "coverage.h"
  29 #include "dp-packet.h"
  30 #include "dpif.h"
  31 #include "netlink.h"
  32 #include "odp-netlink.h"
  33 #include "odp-util.h"
  34 #include "packets.h"
  35 #include "flow.h"
  36 #include "unaligned.h"
  37 #include "util.h"
  38 #include "csum.h"
  39 #include "conntrack.h"
  40 #include "openvswitch/vlog.h"
  41
  42 VLOG_DEFINE_THIS_MODULE(odp_execute);
  43 COVERAGE_DEFINE(datapath_drop_sample_error);
  44 COVERAGE_DEFINE(datapath_drop_nsh_decap_error);
  45 COVERAGE_DEFINE(drop_action_of_pipeline);
  46 COVERAGE_DEFINE(drop_action_bridge_not_found);
  47 COVERAGE_DEFINE(drop_action_recursion_too_deep);
  48 COVERAGE_DEFINE(drop_action_too_many_resubmit);
  49 COVERAGE_DEFINE(drop_action_stack_too_deep);
  50 COVERAGE_DEFINE(drop_action_no_recirculation_context);
  51 COVERAGE_DEFINE(drop_action_recirculation_conflict);
  52 COVERAGE_DEFINE(drop_action_too_many_mpls_labels);
  53 COVERAGE_DEFINE(drop_action_invalid_tunnel_metadata);
  54 COVERAGE_DEFINE(drop_action_unsupported_packet_type);
  55 COVERAGE_DEFINE(drop_action_congestion);
  56 COVERAGE_DEFINE(drop_action_forwarding_disabled);
  57
  58 static void
  59 dp_update_drop_action_counter(enum xlate_error drop_reason,
  60                               int delta)
  61 {
  62    static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5);
  63
  64    switch (drop_reason) {
  65    case XLATE_OK:
  66         COVERAGE_ADD(drop_action_of_pipeline, delta);
  67         break;
  68    case XLATE_BRIDGE_NOT_FOUND:
  69         COVERAGE_ADD(drop_action_bridge_not_found, delta);
  70         break;
  71    case XLATE_RECURSION_TOO_DEEP:
  72         COVERAGE_ADD(drop_action_recursion_too_deep, delta);
  73         break;
  74    case XLATE_TOO_MANY_RESUBMITS:
  75         COVERAGE_ADD(drop_action_too_many_resubmit, delta);
  76         break;
  77    case XLATE_STACK_TOO_DEEP:
  78         COVERAGE_ADD(drop_action_stack_too_deep, delta);
  79         break;
  80    case XLATE_NO_RECIRCULATION_CONTEXT:
  81         COVERAGE_ADD(drop_action_no_recirculation_context, delta);
  82         break;
  83    case XLATE_RECIRCULATION_CONFLICT:
  84         COVERAGE_ADD(drop_action_recirculation_conflict, delta);
  85         break;
  86    case XLATE_TOO_MANY_MPLS_LABELS:
  87         COVERAGE_ADD(drop_action_too_many_mpls_labels, delta);
  88         break;
  89    case XLATE_INVALID_TUNNEL_METADATA:
  90         COVERAGE_ADD(drop_action_invalid_tunnel_metadata, delta);
  91         break;
  92    case XLATE_UNSUPPORTED_PACKET_TYPE:
  93         COVERAGE_ADD(drop_action_unsupported_packet_type, delta);
  94         break;
  95    case XLATE_CONGESTION_DROP:
  96         COVERAGE_ADD(drop_action_congestion, delta);
  97         break;
  98    case XLATE_FORWARDING_DISABLED:
  99         COVERAGE_ADD(drop_action_forwarding_disabled, delta);
 100         break;
 101    case XLATE_MAX:
 102    default:
 103         VLOG_ERR_RL(&rl, "Invalid Drop reason type: %d", drop_reason);
 104    }
 105 }
 106
 107 /* Masked copy of an ethernet address. 'src' is already properly masked. */
 108 static void
 109 ether_addr_copy_masked(struct eth_addr *dst, const struct eth_addr src,
 110                        const struct eth_addr mask)
 111 {
 112     int i;
 113
 114     for (i = 0; i < ARRAY_SIZE(dst->be16); i++) {
 115         dst->be16[i] = src.be16[i] | (dst->be16[i] & ~mask.be16[i]);
 116     }
 117 }
 118
 119 static void
 120 odp_eth_set_addrs(struct dp_packet *packet, const struct ovs_key_ethernet *key,
 121                   const struct ovs_key_ethernet *mask)
 122 {
 123     struct eth_header *eh = dp_packet_eth(packet);
 124
 125     if (eh) {
 126         if (!mask) {
 127             eh->eth_src = key->eth_src;
 128             eh->eth_dst = key->eth_dst;
 129         } else {
 130             ether_addr_copy_masked(&eh->eth_src, key->eth_src, mask->eth_src);
 131             ether_addr_copy_masked(&eh->eth_dst, key->eth_dst, mask->eth_dst);
 132         }
 133     }
 134 }
 135
 136 static void
 137 odp_set_ipv4(struct dp_packet *packet, const struct ovs_key_ipv4 *key,
 138              const struct ovs_key_ipv4 *mask)
 139 {
 140     struct ip_header *nh = dp_packet_l3(packet);
 141     ovs_be32 ip_src_nh;
 142     ovs_be32 ip_dst_nh;
 143     ovs_be32 new_ip_src;
 144     ovs_be32 new_ip_dst;
 145     uint8_t new_tos;
 146     uint8_t new_ttl;
 147
 148     if (mask->ipv4_src) {
 149         ip_src_nh = get_16aligned_be32(&nh->ip_src);
 150         new_ip_src = key->ipv4_src | (ip_src_nh & ~mask->ipv4_src);
 151
 152         if (ip_src_nh != new_ip_src) {
 153             packet_set_ipv4_addr(packet, &nh->ip_src, new_ip_src);
 154         }
 155     }
 156
 157     if (mask->ipv4_dst) {
 158         ip_dst_nh = get_16aligned_be32(&nh->ip_dst);
 159         new_ip_dst = key->ipv4_dst | (ip_dst_nh & ~mask->ipv4_dst);
 160
 161         if (ip_dst_nh != new_ip_dst) {
 162             packet_set_ipv4_addr(packet, &nh->ip_dst, new_ip_dst);
 163         }
 164     }
 165
 166     if (mask->ipv4_tos) {
 167         new_tos = key->ipv4_tos | (nh->ip_tos & ~mask->ipv4_tos);
 168
 169         if (nh->ip_tos != new_tos) {
 170             nh->ip_csum = recalc_csum16(nh->ip_csum,
 171                                         htons((uint16_t) nh->ip_tos),
 172                                         htons((uint16_t) new_tos));
 173             nh->ip_tos = new_tos;
 174         }
 175     }
 176
 177     if (OVS_LIKELY(mask->ipv4_ttl)) {
 178         new_ttl = key->ipv4_ttl | (nh->ip_ttl & ~mask->ipv4_ttl);
 179
 180         if (OVS_LIKELY(nh->ip_ttl != new_ttl)) {
 181             nh->ip_csum = recalc_csum16(nh->ip_csum, htons(nh->ip_ttl << 8),
 182                                         htons(new_ttl << 8));
 183             nh->ip_ttl = new_ttl;
 184         }
 185     }
 186 }
 187
 188 static struct in6_addr *
 189 mask_ipv6_addr(const ovs_16aligned_be32 *old, const struct in6_addr *addr,
 190                const struct in6_addr *mask, struct in6_addr *masked)
 191 {
 192 #ifdef s6_addr32
 193     for (int i = 0; i < 4; i++) {
 194         masked->s6_addr32[i] = addr->s6_addr32[i]
 195             | (get_16aligned_be32(&old[i]) & ~mask->s6_addr32[i]);
 196     }
 197 #else
 198     const uint8_t *old8 = (const uint8_t *)old;
 199     for (int i = 0; i < 16; i++) {
 200         masked->s6_addr[i] = addr->s6_addr[i] | (old8[i] & ~mask->s6_addr[i]);
 201     }
 202 #endif
 203     return masked;
 204 }
 205
 206 static void
 207 odp_set_ipv6(struct dp_packet *packet, const struct ovs_key_ipv6 *key,
 208              const struct ovs_key_ipv6 *mask)
 209 {
 210     struct ovs_16aligned_ip6_hdr *nh = dp_packet_l3(packet);
 211     struct in6_addr sbuf, dbuf;
 212     uint8_t old_tc = ntohl(get_16aligned_be32(&nh->ip6_flow)) >> 20;
 213     ovs_be32 old_fl = get_16aligned_be32(&nh->ip6_flow) & htonl(0xfffff);
 214
 215     packet_set_ipv6(
 216         packet,
 217         mask_ipv6_addr(nh->ip6_src.be32, &key->ipv6_src, &mask->ipv6_src,
 218                        &sbuf),
 219         mask_ipv6_addr(nh->ip6_dst.be32, &key->ipv6_dst, &mask->ipv6_dst,
 220                        &dbuf),
 221         key->ipv6_tclass | (old_tc & ~mask->ipv6_tclass),
 222         key->ipv6_label | (old_fl & ~mask->ipv6_label),
 223         key->ipv6_hlimit | (nh->ip6_hlim & ~mask->ipv6_hlimit));
 224 }
 225
 226 static void
 227 odp_set_tcp(struct dp_packet *packet, const struct ovs_key_tcp *key,
 228              const struct ovs_key_tcp *mask)
 229 {
 230     struct tcp_header *th = dp_packet_l4(packet);
 231
 232     if (OVS_LIKELY(th && dp_packet_get_tcp_payload(packet))) {
 233         packet_set_tcp_port(packet,
 234                             key->tcp_src | (th->tcp_src & ~mask->tcp_src),
 235                             key->tcp_dst | (th->tcp_dst & ~mask->tcp_dst));
 236     }
 237 }
 238
 239 static void
 240 odp_set_udp(struct dp_packet *packet, const struct ovs_key_udp *key,
 241              const struct ovs_key_udp *mask)
 242 {
 243     struct udp_header *uh = dp_packet_l4(packet);
 244
 245     if (OVS_LIKELY(uh && dp_packet_get_udp_payload(packet))) {
 246         packet_set_udp_port(packet,
 247                             key->udp_src | (uh->udp_src & ~mask->udp_src),
 248                             key->udp_dst | (uh->udp_dst & ~mask->udp_dst));
 249     }
 250 }
 251
 252 static void
 253 odp_set_sctp(struct dp_packet *packet, const struct ovs_key_sctp *key,
 254              const struct ovs_key_sctp *mask)
 255 {
 256     struct sctp_header *sh = dp_packet_l4(packet);
 257
 258     if (OVS_LIKELY(sh && dp_packet_get_sctp_payload(packet))) {
 259         packet_set_sctp_port(packet,
 260                              key->sctp_src | (sh->sctp_src & ~mask->sctp_src),
 261                              key->sctp_dst | (sh->sctp_dst & ~mask->sctp_dst));
 262     }
 263 }
 264
 265 static void
 266 odp_set_tunnel_action(const struct nlattr *a, struct flow_tnl *tun_key)
 267 {
 268     ovs_assert(odp_tun_key_from_attr(a, tun_key, NULL) != ODP_FIT_ERROR);
 269 }
 270
 271 static void
 272 set_arp(struct dp_packet *packet, const struct ovs_key_arp *key,
 273         const struct ovs_key_arp *mask)
 274 {
 275     struct arp_eth_header *arp = dp_packet_l3(packet);
 276
 277     if (!mask) {
 278         arp->ar_op = key->arp_op;
 279         arp->ar_sha = key->arp_sha;
 280         put_16aligned_be32(&arp->ar_spa, key->arp_sip);
 281         arp->ar_tha = key->arp_tha;
 282         put_16aligned_be32(&arp->ar_tpa, key->arp_tip);
 283     } else {
 284         ovs_be32 ar_spa = get_16aligned_be32(&arp->ar_spa);
 285         ovs_be32 ar_tpa = get_16aligned_be32(&arp->ar_tpa);
 286
 287         arp->ar_op = key->arp_op | (arp->ar_op & ~mask->arp_op);
 288         ether_addr_copy_masked(&arp->ar_sha, key->arp_sha, mask->arp_sha);
 289         put_16aligned_be32(&arp->ar_spa,
 290                            key->arp_sip | (ar_spa & ~mask->arp_sip));
 291         ether_addr_copy_masked(&arp->ar_tha, key->arp_tha, mask->arp_tha);
 292         put_16aligned_be32(&arp->ar_tpa,
 293                            key->arp_tip | (ar_tpa & ~mask->arp_tip));
 294     }
 295 }
 296
 297 static void
 298 odp_set_nd_ext(struct dp_packet *packet, const struct ovs_key_nd_extensions
 299         *key, const struct ovs_key_nd_extensions *mask)
 300 {
 301     const struct ovs_nd_msg *ns = dp_packet_l4(packet);
 302     ovs_16aligned_be32 reserved = ns->rso_flags;
 303     uint8_t opt_type = ns->options[0].type;
 304
 305     if (mask->nd_reserved) {
 306         put_16aligned_be32(&reserved, key->nd_reserved);
 307     }
 308     if (mask->nd_options_type) {
 309         opt_type = key->nd_options_type;
 310     }
 311     packet_set_nd_ext(packet, reserved, opt_type);
 312 }
 313
 314 static void
 315 odp_set_nd(struct dp_packet *packet, const struct ovs_key_nd *key,
 316            const struct ovs_key_nd *mask)
 317 {
 318     const struct ovs_nd_msg *ns = dp_packet_l4(packet);
 319     const struct ovs_nd_lla_opt *lla_opt = dp_packet_get_nd_payload(packet);
 320
 321     if (OVS_LIKELY(ns && lla_opt)) {
 322         int bytes_remain = dp_packet_l4_size(packet) - sizeof(*ns);
 323         struct in6_addr tgt_buf;
 324         struct eth_addr sll_buf = eth_addr_zero;
 325         struct eth_addr tll_buf = eth_addr_zero;
 326
 327         while (bytes_remain >= ND_LLA_OPT_LEN && lla_opt->len != 0) {
 328             if (lla_opt->type == ND_OPT_SOURCE_LINKADDR
 329                 && lla_opt->len == 1) {
 330                 sll_buf = lla_opt->mac;
 331                 ether_addr_copy_masked(&sll_buf, key->nd_sll, mask->nd_sll);
 332
 333                 /* A packet can only contain one SLL or TLL option */
 334                 break;
 335             } else if (lla_opt->type == ND_OPT_TARGET_LINKADDR
 336                        && lla_opt->len == 1) {
 337                 tll_buf = lla_opt->mac;
 338                 ether_addr_copy_masked(&tll_buf, key->nd_tll, mask->nd_tll);
 339
 340                 /* A packet can only contain one SLL or TLL option */
 341                 break;
 342             }
 343
 344             lla_opt += lla_opt->len;
 345             bytes_remain -= lla_opt->len * ND_LLA_OPT_LEN;
 346         }
 347
 348         packet_set_nd(packet,
 349                       mask_ipv6_addr(ns->target.be32, &key->nd_target,
 350                                      &mask->nd_target, &tgt_buf),
 351                       sll_buf,
 352                       tll_buf);
 353     }
 354 }
 355
 356 /* Set the NSH header. Assumes the NSH header is present and matches the
 357  * MD format of the key. The slow path must take case of that. */
 358 static void
 359 odp_set_nsh(struct dp_packet *packet, const struct nlattr *a, bool has_mask)
 360 {
 361     struct ovs_key_nsh key, mask;
 362     struct nsh_hdr *nsh = dp_packet_l3(packet);
 363     uint8_t mdtype = nsh_md_type(nsh);
 364     ovs_be32 path_hdr;
 365
 366     if (has_mask) {
 367         odp_nsh_key_from_attr(a, &key, &mask, NULL);
 368     } else {
 369         odp_nsh_key_from_attr(a, &key, NULL, NULL);
 370     }
 371
 372     if (!has_mask) {
 373         nsh_set_flags_and_ttl(nsh, key.flags, key.ttl);
 374         put_16aligned_be32(&nsh->path_hdr, key.path_hdr);
 375         switch (mdtype) {
 376             case NSH_M_TYPE1:
 377                 for (int i = 0; i < 4; i++) {
 378                     put_16aligned_be32(&nsh->md1.context[i], key.context[i]);
 379                 }
 380                 break;
 381             case NSH_M_TYPE2:
 382             default:
 383                 /* No support for setting any other metadata format yet. */
 384                 break;
 385         }
 386     } else {
 387         uint8_t flags = nsh_get_flags(nsh);
 388         uint8_t ttl = nsh_get_ttl(nsh);
 389
 390         flags = key.flags | (flags & ~mask.flags);
 391         ttl = key.ttl | (ttl & ~mask.ttl);
 392         nsh_set_flags_and_ttl(nsh, flags, ttl);
 393
 394         uint32_t spi = ntohl(nsh_get_spi(nsh));
 395         uint8_t si = nsh_get_si(nsh);
 396         uint32_t spi_mask = nsh_path_hdr_to_spi_uint32(mask.path_hdr);
 397         uint8_t si_mask = nsh_path_hdr_to_si(mask.path_hdr);
 398         if (spi_mask == 0x00ffffff) {
 399             spi_mask = UINT32_MAX;
 400         }
 401         spi = nsh_path_hdr_to_spi_uint32(key.path_hdr) | (spi & ~spi_mask);
 402         si = nsh_path_hdr_to_si(key.path_hdr) | (si & ~si_mask);
 403         path_hdr = nsh_get_path_hdr(nsh);
 404         nsh_path_hdr_set_spi(&path_hdr, htonl(spi));
 405         nsh_path_hdr_set_si(&path_hdr, si);
 406         put_16aligned_be32(&nsh->path_hdr, path_hdr);
 407         switch (mdtype) {
 408             case NSH_M_TYPE1:
 409                 for (int i = 0; i < 4; i++) {
 410                     ovs_be32 p = get_16aligned_be32(&nsh->md1.context[i]);
 411                     ovs_be32 k = key.context[i];
 412                     ovs_be32 m = mask.context[i];
 413                     put_16aligned_be32(&nsh->md1.context[i], k | (p & ~m));
 414                 }
 415                 break;
 416             case NSH_M_TYPE2:
 417             default:
 418                 /* No support for setting any other metadata format yet. */
 419                 break;
 420         }
 421     }
 422 }
 423
 424 static void
 425 odp_execute_set_action(struct dp_packet *packet, const struct nlattr *a)
 426 {
 427     enum ovs_key_attr type = nl_attr_type(a);
 428     const struct ovs_key_ipv4 *ipv4_key;
 429     const struct ovs_key_ipv6 *ipv6_key;
 430     struct pkt_metadata *md = &packet->md;
 431
 432     switch (type) {
 433     case OVS_KEY_ATTR_PRIORITY:
 434         md->skb_priority = nl_attr_get_u32(a);
 435         break;
 436
 437     case OVS_KEY_ATTR_TUNNEL:
 438         odp_set_tunnel_action(a, &md->tunnel);
 439         break;
 440
 441     case OVS_KEY_ATTR_SKB_MARK:
 442         md->pkt_mark = nl_attr_get_u32(a);
 443         break;
 444
 445     case OVS_KEY_ATTR_ETHERNET:
 446         odp_eth_set_addrs(packet, nl_attr_get(a), NULL);
 447         break;
 448
 449     case OVS_KEY_ATTR_NSH: {
 450         odp_set_nsh(packet, a, false);
 451         break;
 452     }
 453
 454     case OVS_KEY_ATTR_IPV4:
 455         ipv4_key = nl_attr_get_unspec(a, sizeof(struct ovs_key_ipv4));
 456         packet_set_ipv4(packet, ipv4_key->ipv4_src,
 457                         ipv4_key->ipv4_dst, ipv4_key->ipv4_tos,
 458                         ipv4_key->ipv4_ttl);
 459         break;
 460
 461     case OVS_KEY_ATTR_IPV6:
 462         ipv6_key = nl_attr_get_unspec(a, sizeof(struct ovs_key_ipv6));
 463         packet_set_ipv6(packet, &ipv6_key->ipv6_src, &ipv6_key->ipv6_dst,
 464                         ipv6_key->ipv6_tclass, ipv6_key->ipv6_label,
 465                         ipv6_key->ipv6_hlimit);
 466         break;
 467
 468     case OVS_KEY_ATTR_TCP:
 469         if (OVS_LIKELY(dp_packet_get_tcp_payload(packet))) {
 470             const struct ovs_key_tcp *tcp_key
 471                 = nl_attr_get_unspec(a, sizeof(struct ovs_key_tcp));
 472
 473             packet_set_tcp_port(packet, tcp_key->tcp_src,
 474                                 tcp_key->tcp_dst);
 475         }
 476         break;
 477
 478     case OVS_KEY_ATTR_UDP:
 479         if (OVS_LIKELY(dp_packet_get_udp_payload(packet))) {
 480             const struct ovs_key_udp *udp_key
 481                 = nl_attr_get_unspec(a, sizeof(struct ovs_key_udp));
 482
 483             packet_set_udp_port(packet, udp_key->udp_src,
 484                                 udp_key->udp_dst);
 485         }
 486         break;
 487
 488     case OVS_KEY_ATTR_SCTP:
 489         if (OVS_LIKELY(dp_packet_get_sctp_payload(packet))) {
 490             const struct ovs_key_sctp *sctp_key
 491                 = nl_attr_get_unspec(a, sizeof(struct ovs_key_sctp));
 492
 493             packet_set_sctp_port(packet, sctp_key->sctp_src,
 494                                  sctp_key->sctp_dst);
 495         }
 496         break;
 497
 498     case OVS_KEY_ATTR_MPLS:
 499         set_mpls_lse(packet, nl_attr_get_be32(a));
 500         break;
 501
 502     case OVS_KEY_ATTR_ARP:
 503         set_arp(packet, nl_attr_get(a), NULL);
 504         break;
 505
 506     case OVS_KEY_ATTR_ICMP:
 507     case OVS_KEY_ATTR_ICMPV6:
 508         if (OVS_LIKELY(dp_packet_get_icmp_payload(packet))) {
 509             const struct ovs_key_icmp *icmp_key
 510                 = nl_attr_get_unspec(a, sizeof(struct ovs_key_icmp));
 511
 512             packet_set_icmp(packet, icmp_key->icmp_type, icmp_key->icmp_code);
 513         }
 514         break;
 515
 516     case OVS_KEY_ATTR_ND:
 517         if (OVS_LIKELY(dp_packet_get_nd_payload(packet))) {
 518             const struct ovs_key_nd *nd_key
 519                    = nl_attr_get_unspec(a, sizeof(struct ovs_key_nd));
 520             packet_set_nd(packet, &nd_key->nd_target, nd_key->nd_sll,
 521                           nd_key->nd_tll);
 522         }
 523         break;
 524
 525     case OVS_KEY_ATTR_ND_EXTENSIONS:
 526         if (OVS_LIKELY(dp_packet_get_nd_payload(packet))) {
 527             const struct ovs_key_nd_extensions *nd_ext_key
 528                  = nl_attr_get_unspec(a, sizeof(struct ovs_key_nd_extensions));
 529             ovs_16aligned_be32 rso_flags;
 530             put_16aligned_be32(&rso_flags, nd_ext_key->nd_reserved);
 531             packet_set_nd_ext(packet, rso_flags, nd_ext_key->nd_options_type);
 532         }
 533         break;
 534
 535     case OVS_KEY_ATTR_DP_HASH:
 536         md->dp_hash = nl_attr_get_u32(a);
 537         break;
 538
 539     case OVS_KEY_ATTR_RECIRC_ID:
 540         md->recirc_id = nl_attr_get_u32(a);
 541         break;
 542
 543     case OVS_KEY_ATTR_UNSPEC:
 544     case OVS_KEY_ATTR_PACKET_TYPE:
 545     case OVS_KEY_ATTR_ENCAP:
 546     case OVS_KEY_ATTR_ETHERTYPE:
 547     case OVS_KEY_ATTR_IN_PORT:
 548     case OVS_KEY_ATTR_VLAN:
 549     case OVS_KEY_ATTR_TCP_FLAGS:
 550     case OVS_KEY_ATTR_CT_STATE:
 551     case OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4:
 552     case OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6:
 553     case OVS_KEY_ATTR_CT_ZONE:
 554     case OVS_KEY_ATTR_CT_MARK:
 555     case OVS_KEY_ATTR_CT_LABELS:
 556     case __OVS_KEY_ATTR_MAX:
 557     default:
 558         OVS_NOT_REACHED();
 559     }
 560 }
 561
 562 #define get_mask(a, type) ((const type *)(const void *)(a + 1) + 1)
 563
 564 static void
 565 odp_execute_masked_set_action(struct dp_packet *packet,
 566                               const struct nlattr *a)
 567 {
 568     struct pkt_metadata *md = &packet->md;
 569     enum ovs_key_attr type = nl_attr_type(a);
 570     struct mpls_hdr *mh;
 571
 572     switch (type) {
 573     case OVS_KEY_ATTR_PRIORITY:
 574         md->skb_priority = nl_attr_get_u32(a)
 575             | (md->skb_priority & ~*get_mask(a, uint32_t));
 576         break;
 577
 578     case OVS_KEY_ATTR_SKB_MARK:
 579         md->pkt_mark = nl_attr_get_u32(a)
 580             | (md->pkt_mark & ~*get_mask(a, uint32_t));
 581         break;
 582
 583     case OVS_KEY_ATTR_ETHERNET:
 584         odp_eth_set_addrs(packet, nl_attr_get(a),
 585                           get_mask(a, struct ovs_key_ethernet));
 586         break;
 587
 588     case OVS_KEY_ATTR_NSH: {
 589         odp_set_nsh(packet, a, true);
 590         break;
 591     }
 592
 593     case OVS_KEY_ATTR_IPV4:
 594         odp_set_ipv4(packet, nl_attr_get(a),
 595                      get_mask(a, struct ovs_key_ipv4));
 596         break;
 597
 598     case OVS_KEY_ATTR_IPV6:
 599         odp_set_ipv6(packet, nl_attr_get(a),
 600                      get_mask(a, struct ovs_key_ipv6));
 601         break;
 602
 603     case OVS_KEY_ATTR_TCP:
 604         odp_set_tcp(packet, nl_attr_get(a),
 605                     get_mask(a, struct ovs_key_tcp));
 606         break;
 607
 608     case OVS_KEY_ATTR_UDP:
 609         odp_set_udp(packet, nl_attr_get(a),
 610                     get_mask(a, struct ovs_key_udp));
 611         break;
 612
 613     case OVS_KEY_ATTR_SCTP:
 614         odp_set_sctp(packet, nl_attr_get(a),
 615                      get_mask(a, struct ovs_key_sctp));
 616         break;
 617
 618     case OVS_KEY_ATTR_MPLS:
 619         mh = dp_packet_l2_5(packet);
 620         if (mh) {
 621             put_16aligned_be32(&mh->mpls_lse, nl_attr_get_be32(a)
 622                                | (get_16aligned_be32(&mh->mpls_lse)
 623                                   & ~*get_mask(a, ovs_be32)));
 624         }
 625         break;
 626
 627     case OVS_KEY_ATTR_ARP:
 628         set_arp(packet, nl_attr_get(a),
 629                 get_mask(a, struct ovs_key_arp));
 630         break;
 631
 632     case OVS_KEY_ATTR_ND:
 633         odp_set_nd(packet, nl_attr_get(a),
 634                    get_mask(a, struct ovs_key_nd));
 635         break;
 636
 637     case OVS_KEY_ATTR_ND_EXTENSIONS:
 638         odp_set_nd_ext(packet, nl_attr_get(a),
 639                        get_mask(a, struct ovs_key_nd_extensions));
 640         break;
 641
 642     case OVS_KEY_ATTR_DP_HASH:
 643         md->dp_hash = nl_attr_get_u32(a)
 644             | (md->dp_hash & ~*get_mask(a, uint32_t));
 645         break;
 646
 647     case OVS_KEY_ATTR_RECIRC_ID:
 648         md->recirc_id = nl_attr_get_u32(a)
 649             | (md->recirc_id & ~*get_mask(a, uint32_t));
 650         break;
 651
 652     case OVS_KEY_ATTR_TUNNEL:    /* Masked data not supported for tunnel. */
 653     case OVS_KEY_ATTR_PACKET_TYPE:
 654     case OVS_KEY_ATTR_UNSPEC:
 655     case OVS_KEY_ATTR_CT_STATE:
 656     case OVS_KEY_ATTR_CT_ZONE:
 657     case OVS_KEY_ATTR_CT_MARK:
 658     case OVS_KEY_ATTR_CT_LABELS:
 659     case OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4:
 660     case OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6:
 661     case OVS_KEY_ATTR_ENCAP:
 662     case OVS_KEY_ATTR_ETHERTYPE:
 663     case OVS_KEY_ATTR_IN_PORT:
 664     case OVS_KEY_ATTR_VLAN:
 665     case OVS_KEY_ATTR_ICMP:
 666     case OVS_KEY_ATTR_ICMPV6:
 667     case OVS_KEY_ATTR_TCP_FLAGS:
 668     case __OVS_KEY_ATTR_MAX:
 669     default:
 670         OVS_NOT_REACHED();
 671     }
 672 }
 673
 674 static void
 675 odp_execute_sample(void *dp, struct dp_packet *packet, bool steal,
 676                    const struct nlattr *action,
 677                    odp_execute_cb dp_execute_action)
 678 {
 679     const struct nlattr *subactions = NULL;
 680     const struct nlattr *a;
 681     struct dp_packet_batch pb;
 682     size_t left;
 683
 684     NL_NESTED_FOR_EACH_UNSAFE (a, left, action) {
 685         int type = nl_attr_type(a);
 686
 687         switch ((enum ovs_sample_attr) type) {
 688         case OVS_SAMPLE_ATTR_PROBABILITY:
 689             if (random_uint32() >= nl_attr_get_u32(a)) {
 690                 if (steal) {
 691                     COVERAGE_INC(datapath_drop_sample_error);
 692                     dp_packet_delete(packet);
 693                 }
 694                 return;
 695             }
 696             break;
 697
 698         case OVS_SAMPLE_ATTR_ACTIONS:
 699             subactions = a;
 700             break;
 701
 702         case OVS_SAMPLE_ATTR_UNSPEC:
 703         case __OVS_SAMPLE_ATTR_MAX:
 704         default:
 705             OVS_NOT_REACHED();
 706         }
 707     }
 708
 709     if (!steal) {
 710         /* The 'subactions' may modify the packet, but the modification
 711          * should not propagate beyond this sample action. Make a copy
 712          * the packet in case we don't own the packet, so that the
 713          * 'subactions' are only applid to the clone.  'odp_execute_actions'
 714          * will free the clone.  */
 715         packet = dp_packet_clone(packet);
 716     }
 717     dp_packet_batch_init_packet(&pb, packet);
 718     odp_execute_actions(dp, &pb, true, nl_attr_get(subactions),
 719                         nl_attr_get_size(subactions), dp_execute_action);
 720 }
 721
 722 static void
 723 odp_execute_clone(void *dp, struct dp_packet_batch *batch, bool steal,
 724                    const struct nlattr *actions,
 725                    odp_execute_cb dp_execute_action)
 726 {
 727     if (!steal) {
 728         /* The 'actions' may modify the packet, but the modification
 729          * should not propagate beyond this clone action. Make a copy
 730          * the packet in case we don't own the packet, so that the
 731          * 'actions' are only applied to the clone.  'odp_execute_actions'
 732          * will free the clone.  */
 733         struct dp_packet_batch clone_pkt_batch;
 734         dp_packet_batch_clone(&clone_pkt_batch, batch);
 735         dp_packet_batch_reset_cutlen(batch);
 736         odp_execute_actions(dp, &clone_pkt_batch, true, nl_attr_get(actions),
 737                         nl_attr_get_size(actions), dp_execute_action);
 738     }
 739     else {
 740         odp_execute_actions(dp, batch, true, nl_attr_get(actions),
 741                             nl_attr_get_size(actions), dp_execute_action);
 742     }
 743 }
 744
 745 static void
 746 odp_execute_check_pkt_len(void *dp, struct dp_packet *packet, bool steal,
 747                           const struct nlattr *action,
 748                           odp_execute_cb dp_execute_action)
 749 {
 750     static const struct nl_policy ovs_cpl_policy[] = {
 751         [OVS_CHECK_PKT_LEN_ATTR_PKT_LEN] = { .type = NL_A_U16 },
 752         [OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER] = { .type = NL_A_NESTED },
 753         [OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL]
 754             = { .type = NL_A_NESTED },
 755     };
 756     struct nlattr *attrs[ARRAY_SIZE(ovs_cpl_policy)];
 757
 758     if (!nl_parse_nested(action, ovs_cpl_policy, attrs, ARRAY_SIZE(attrs))) {
 759         OVS_NOT_REACHED();
 760     }
 761
 762     const struct nlattr *a;
 763     struct dp_packet_batch pb;
 764     uint32_t size = dp_packet_get_send_len(packet)
 765                     - dp_packet_l2_pad_size(packet);
 766
 767     a = attrs[OVS_CHECK_PKT_LEN_ATTR_PKT_LEN];
 768     if (size > nl_attr_get_u16(a)) {
 769         a = attrs[OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER];
 770     } else {
 771         a = attrs[OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL];
 772     }
 773
 774     if (!steal) {
 775         /* The 'subactions' may modify the packet, but the modification
 776          * should not propagate beyond this action. Make a copy
 777          * the packet in case we don't own the packet, so that the
 778          * 'subactions' are only applid to check_pkt_len. 'odp_execute_actions'
 779          * will free the clone.  */
 780         packet = dp_packet_clone(packet);
 781     }
 782     /* If nl_attr_get(a) is NULL, the packet will be freed by
 783      * odp_execute_actions. */
 784     dp_packet_batch_init_packet(&pb, packet);
 785     odp_execute_actions(dp, &pb, true, nl_attr_get(a), nl_attr_get_size(a),
 786                         dp_execute_action);
 787 }
 788
 789 static bool
 790 requires_datapath_assistance(const struct nlattr *a)
 791 {
 792     enum ovs_action_attr type = nl_attr_type(a);
 793
 794     switch (type) {
 795         /* These only make sense in the context of a datapath. */
 796     case OVS_ACTION_ATTR_OUTPUT:
 797     case OVS_ACTION_ATTR_LB_OUTPUT:
 798     case OVS_ACTION_ATTR_TUNNEL_PUSH:
 799     case OVS_ACTION_ATTR_TUNNEL_POP:
 800     case OVS_ACTION_ATTR_USERSPACE:
 801     case OVS_ACTION_ATTR_RECIRC:
 802     case OVS_ACTION_ATTR_CT:
 803     case OVS_ACTION_ATTR_METER:
 804         return true;
 805
 806     case OVS_ACTION_ATTR_SET:
 807     case OVS_ACTION_ATTR_SET_MASKED:
 808     case OVS_ACTION_ATTR_PUSH_VLAN:
 809     case OVS_ACTION_ATTR_POP_VLAN:
 810     case OVS_ACTION_ATTR_SAMPLE:
 811     case OVS_ACTION_ATTR_HASH:
 812     case OVS_ACTION_ATTR_PUSH_MPLS:
 813     case OVS_ACTION_ATTR_POP_MPLS:
 814     case OVS_ACTION_ATTR_TRUNC:
 815     case OVS_ACTION_ATTR_PUSH_ETH:
 816     case OVS_ACTION_ATTR_POP_ETH:
 817     case OVS_ACTION_ATTR_CLONE:
 818     case OVS_ACTION_ATTR_PUSH_NSH:
 819     case OVS_ACTION_ATTR_POP_NSH:
 820     case OVS_ACTION_ATTR_CT_CLEAR:
 821     case OVS_ACTION_ATTR_CHECK_PKT_LEN:
 822     case OVS_ACTION_ATTR_DROP:
 823         return false;
 824
 825     case OVS_ACTION_ATTR_UNSPEC:
 826     case __OVS_ACTION_ATTR_MAX:
 827         OVS_NOT_REACHED();
 828     }
 829
 830     return false;
 831 }
 832
 833 /* Executes all of the 'actions_len' bytes of datapath actions in 'actions' on
 834  * the packets in 'batch'.  If 'steal' is true, possibly modifies and
 835  * definitely free the packets in 'batch', otherwise leaves 'batch' unchanged.
 836  *
 837  * Some actions (e.g. output actions) can only be executed by a datapath.  This
 838  * function implements those actions by passing the action and the packets to
 839  * 'dp_execute_action' (along with 'dp').  If 'dp_execute_action' is passed a
 840  * true 'steal' parameter then it must definitely free the packets passed into
 841  * it.  The packet can be modified whether 'steal' is false or true.  If a
 842  * packet is removed from the batch, then the fate of the packet is determined
 843  * by the code that does this removal, irrespective of the value of 'steal'.
 844  * Otherwise, if the packet is not removed from the batch and 'steal' is false
 845  * then the packet could either be cloned or not. */
 846 void
 847 odp_execute_actions(void *dp, struct dp_packet_batch *batch, bool steal,
 848                     const struct nlattr *actions, size_t actions_len,
 849                     odp_execute_cb dp_execute_action)
 850 {
 851     struct dp_packet *packet;
 852     const struct nlattr *a;
 853     unsigned int left;
 854
 855     NL_ATTR_FOR_EACH_UNSAFE (a, left, actions, actions_len) {
 856         int type = nl_attr_type(a);
 857         bool last_action = (left <= NLA_ALIGN(a->nla_len));
 858
 859         if (requires_datapath_assistance(a)) {
 860             if (dp_execute_action) {
 861                 /* Allow 'dp_execute_action' to steal the packet data if we do
 862                  * not need it any more. */
 863                 bool should_steal = steal && last_action;
 864
 865                 dp_execute_action(dp, batch, a, should_steal);
 866
 867                 if (last_action || dp_packet_batch_is_empty(batch)) {
 868                     /* We do not need to free the packets.
 869                      * Either dp_execute_actions() has stolen them
 870                      * or the batch is freed due to errors. In either
 871                      * case we do not need to execute further actions.
 872                      */
 873                     return;
 874                 }
 875             }
 876             continue;
 877         }
 878
 879         switch ((enum ovs_action_attr) type) {
 880
 881         case OVS_ACTION_ATTR_HASH: {
 882             const struct ovs_action_hash *hash_act = nl_attr_get(a);
 883
 884             /* Calculate a hash value directly. This might not match the
 885              * value computed by the datapath, but it is much less expensive,
 886              * and the current use case (bonding) does not require a strict
 887              * match to work properly. */
 888             switch (hash_act->hash_alg) {
 889             case OVS_HASH_ALG_L4: {
 890                 struct flow flow;
 891                 uint32_t hash;
 892
 893                 DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
 894                     /* RSS hash can be used here instead of 5tuple for
 895                      * performance reasons. */
 896                     if (dp_packet_rss_valid(packet)) {
 897                         hash = dp_packet_get_rss_hash(packet);
 898                         hash = hash_int(hash, hash_act->hash_basis);
 899                     } else {
 900                         flow_extract(packet, &flow);
 901                         hash = flow_hash_5tuple(&flow, hash_act->hash_basis);
 902                     }
 903                     packet->md.dp_hash = hash;
 904                 }
 905                 break;
 906             }
 907             case OVS_HASH_ALG_SYM_L4: {
 908                 struct flow flow;
 909                 uint32_t hash;
 910
 911                 DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
 912                     flow_extract(packet, &flow);
 913                     hash = flow_hash_symmetric_l3l4(&flow,
 914                                                     hash_act->hash_basis,
 915                                                     false);
 916                     packet->md.dp_hash = hash;
 917                 }
 918                 break;
 919             }
 920             default:
 921                 /* Assert on unknown hash algorithm.  */
 922                 OVS_NOT_REACHED();
 923             }
 924             break;
 925         }
 926
 927         case OVS_ACTION_ATTR_PUSH_VLAN: {
 928             const struct ovs_action_push_vlan *vlan = nl_attr_get(a);
 929
 930             DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
 931                 eth_push_vlan(packet, vlan->vlan_tpid, vlan->vlan_tci);
 932             }
 933             break;
 934         }
 935
 936         case OVS_ACTION_ATTR_POP_VLAN:
 937             DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
 938                 eth_pop_vlan(packet);
 939             }
 940             break;
 941
 942         case OVS_ACTION_ATTR_PUSH_MPLS: {
 943             const struct ovs_action_push_mpls *mpls = nl_attr_get(a);
 944
 945             DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
 946                 push_mpls(packet, mpls->mpls_ethertype, mpls->mpls_lse);
 947             }
 948             break;
 949          }
 950
 951         case OVS_ACTION_ATTR_POP_MPLS:
 952             DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
 953                 pop_mpls(packet, nl_attr_get_be16(a));
 954             }
 955             break;
 956
 957         case OVS_ACTION_ATTR_SET:
 958             DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
 959                 odp_execute_set_action(packet, nl_attr_get(a));
 960             }
 961             break;
 962
 963         case OVS_ACTION_ATTR_SET_MASKED:
 964             DP_PACKET_BATCH_FOR_EACH(i, packet, batch) {
 965                 odp_execute_masked_set_action(packet, nl_attr_get(a));
 966             }
 967             break;
 968
 969         case OVS_ACTION_ATTR_SAMPLE:
 970             DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
 971                 odp_execute_sample(dp, packet, steal && last_action, a,
 972                                    dp_execute_action);
 973             }
 974
 975             if (last_action) {
 976                 /* We do not need to free the packets. odp_execute_sample() has
 977                  * stolen them*/
 978                 return;
 979             }
 980             break;
 981
 982         case OVS_ACTION_ATTR_TRUNC: {
 983             const struct ovs_action_trunc *trunc =
 984                         nl_attr_get_unspec(a, sizeof *trunc);
 985
 986             batch->trunc = true;
 987             DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
 988                 dp_packet_set_cutlen(packet, trunc->max_len);
 989             }
 990             break;
 991         }
 992
 993         case OVS_ACTION_ATTR_CLONE:
 994             odp_execute_clone(dp, batch, steal && last_action, a,
 995                                                 dp_execute_action);
 996             if (last_action) {
 997                 /* We do not need to free the packets. odp_execute_clone() has
 998                  * stolen them.  */
 999                 return;
1000             }
1001             break;
1002         case OVS_ACTION_ATTR_METER:
1003             /* Not implemented yet. */
1004             break;
1005         case OVS_ACTION_ATTR_PUSH_ETH: {
1006             const struct ovs_action_push_eth *eth = nl_attr_get(a);
1007
1008             DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
1009                 push_eth(packet, &eth->addresses.eth_dst,
1010                          &eth->addresses.eth_src);
1011             }
1012             break;
1013         }
1014
1015         case OVS_ACTION_ATTR_POP_ETH:
1016             DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
1017                 pop_eth(packet);
1018             }
1019             break;
1020
1021         case OVS_ACTION_ATTR_PUSH_NSH: {
1022             uint32_t buffer[NSH_HDR_MAX_LEN / 4];
1023             struct nsh_hdr *nsh_hdr = ALIGNED_CAST(struct nsh_hdr *, buffer);
1024             nsh_reset_ver_flags_ttl_len(nsh_hdr);
1025             odp_nsh_hdr_from_attr(nl_attr_get(a), nsh_hdr, NSH_HDR_MAX_LEN);
1026             DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
1027                 push_nsh(packet, nsh_hdr);
1028             }
1029             break;
1030         }
1031         case OVS_ACTION_ATTR_POP_NSH: {
1032             size_t i;
1033             const size_t num = dp_packet_batch_size(batch);
1034
1035             DP_PACKET_BATCH_REFILL_FOR_EACH (i, num, packet, batch) {
1036                 if (pop_nsh(packet)) {
1037                     dp_packet_batch_refill(batch, packet, i);
1038                 } else {
1039                     COVERAGE_INC(datapath_drop_nsh_decap_error);
1040                     dp_packet_delete(packet);
1041                 }
1042             }
1043             break;
1044         }
1045         case OVS_ACTION_ATTR_CT_CLEAR:
1046             DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
1047                 conntrack_clear(packet);
1048             }
1049             break;
1050
1051         case OVS_ACTION_ATTR_CHECK_PKT_LEN:
1052             DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
1053                 odp_execute_check_pkt_len(dp, packet, steal && last_action, a,
1054                                           dp_execute_action);
1055             }
1056
1057             if (last_action) {
1058                 /* We do not need to free the packets.
1059                  * odp_execute_check_pkt_len() has stolen them. */
1060                 return;
1061             }
1062             break;
1063
1064         case OVS_ACTION_ATTR_DROP:{
1065             const enum xlate_error *drop_reason = nl_attr_get(a);
1066
1067             dp_update_drop_action_counter(*drop_reason,
1068                                           dp_packet_batch_size(batch));
1069             dp_packet_delete_batch(batch, steal);
1070             return;
1071         }
1072         case OVS_ACTION_ATTR_OUTPUT:
1073         case OVS_ACTION_ATTR_LB_OUTPUT:
1074         case OVS_ACTION_ATTR_TUNNEL_PUSH:
1075         case OVS_ACTION_ATTR_TUNNEL_POP:
1076         case OVS_ACTION_ATTR_USERSPACE:
1077         case OVS_ACTION_ATTR_RECIRC:
1078         case OVS_ACTION_ATTR_CT:
1079         case OVS_ACTION_ATTR_UNSPEC:
1080         case __OVS_ACTION_ATTR_MAX:
1081             OVS_NOT_REACHED();
1082         }
1083     }
1084
1085     dp_packet_delete_batch(batch, steal);
1086 }