]> git.proxmox.com Git - ceph.git/blob - ceph/src/spdk/dpdk/drivers/net/tap/tap_flow.c
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
update source to Ceph Pacific 16.2.2
[ceph.git] / ceph / src / spdk / dpdk / drivers / net / tap / tap_flow.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017 6WIND S.A.
3 * Copyright 2017 Mellanox Technologies, Ltd
4 */
5
6 #include <errno.h>
7 #include <string.h>
8 #include <unistd.h>
9 #include <sys/queue.h>
10 #include <sys/resource.h>
11
12 #include <rte_byteorder.h>
13 #include <rte_jhash.h>
14 #include <rte_malloc.h>
15 #include <rte_eth_tap.h>
16 #include <tap_flow.h>
17 #include <tap_autoconf.h>
18 #include <tap_tcmsgs.h>
19 #include <tap_rss.h>
20
21 #ifndef HAVE_TC_FLOWER
22 /*
23 * For kernels < 4.2, this enum is not defined. Runtime checks will be made to
24 * avoid sending TC messages the kernel cannot understand.
25 */
26 enum {
27 TCA_FLOWER_UNSPEC,
28 TCA_FLOWER_CLASSID,
29 TCA_FLOWER_INDEV,
30 TCA_FLOWER_ACT,
31 TCA_FLOWER_KEY_ETH_DST, /* ETH_ALEN */
32 TCA_FLOWER_KEY_ETH_DST_MASK, /* ETH_ALEN */
33 TCA_FLOWER_KEY_ETH_SRC, /* ETH_ALEN */
34 TCA_FLOWER_KEY_ETH_SRC_MASK, /* ETH_ALEN */
35 TCA_FLOWER_KEY_ETH_TYPE, /* be16 */
36 TCA_FLOWER_KEY_IP_PROTO, /* u8 */
37 TCA_FLOWER_KEY_IPV4_SRC, /* be32 */
38 TCA_FLOWER_KEY_IPV4_SRC_MASK, /* be32 */
39 TCA_FLOWER_KEY_IPV4_DST, /* be32 */
40 TCA_FLOWER_KEY_IPV4_DST_MASK, /* be32 */
41 TCA_FLOWER_KEY_IPV6_SRC, /* struct in6_addr */
42 TCA_FLOWER_KEY_IPV6_SRC_MASK, /* struct in6_addr */
43 TCA_FLOWER_KEY_IPV6_DST, /* struct in6_addr */
44 TCA_FLOWER_KEY_IPV6_DST_MASK, /* struct in6_addr */
45 TCA_FLOWER_KEY_TCP_SRC, /* be16 */
46 TCA_FLOWER_KEY_TCP_DST, /* be16 */
47 TCA_FLOWER_KEY_UDP_SRC, /* be16 */
48 TCA_FLOWER_KEY_UDP_DST, /* be16 */
49 };
50 #endif
51 #ifndef HAVE_TC_VLAN_ID
52 enum {
53 /* TCA_FLOWER_FLAGS, */
54 TCA_FLOWER_KEY_VLAN_ID = TCA_FLOWER_KEY_UDP_DST + 2, /* be16 */
55 TCA_FLOWER_KEY_VLAN_PRIO, /* u8 */
56 TCA_FLOWER_KEY_VLAN_ETH_TYPE, /* be16 */
57 };
58 #endif
59 /*
60 * For kernels < 4.2 BPF related enums may not be defined.
61 * Runtime checks will be carried out to gracefully report on TC messages that
62 * are rejected by the kernel. Rejection reasons may be due to:
63 * 1. enum is not defined
64 * 2. enum is defined but kernel is not configured to support BPF system calls,
65 * BPF classifications or BPF actions.
66 */
67 #ifndef HAVE_TC_BPF
68 enum {
69 TCA_BPF_UNSPEC,
70 TCA_BPF_ACT,
71 TCA_BPF_POLICE,
72 TCA_BPF_CLASSID,
73 TCA_BPF_OPS_LEN,
74 TCA_BPF_OPS,
75 };
76 #endif
77 #ifndef HAVE_TC_BPF_FD
78 enum {
79 TCA_BPF_FD = TCA_BPF_OPS + 1,
80 TCA_BPF_NAME,
81 };
82 #endif
83 #ifndef HAVE_TC_ACT_BPF
84 #define tc_gen \
85 __u32 index; \
86 __u32 capab; \
87 int action; \
88 int refcnt; \
89 int bindcnt
90
91 struct tc_act_bpf {
92 tc_gen;
93 };
94
95 enum {
96 TCA_ACT_BPF_UNSPEC,
97 TCA_ACT_BPF_TM,
98 TCA_ACT_BPF_PARMS,
99 TCA_ACT_BPF_OPS_LEN,
100 TCA_ACT_BPF_OPS,
101 };
102
103 #endif
104 #ifndef HAVE_TC_ACT_BPF_FD
105 enum {
106 TCA_ACT_BPF_FD = TCA_ACT_BPF_OPS + 1,
107 TCA_ACT_BPF_NAME,
108 };
109 #endif
110
111 /* RSS key management */
112 enum bpf_rss_key_e {
113 KEY_CMD_GET = 1,
114 KEY_CMD_RELEASE,
115 KEY_CMD_INIT,
116 KEY_CMD_DEINIT,
117 };
118
119 enum key_status_e {
120 KEY_STAT_UNSPEC,
121 KEY_STAT_USED,
122 KEY_STAT_AVAILABLE,
123 };
124
125 #define ISOLATE_HANDLE 1
126 #define REMOTE_PROMISCUOUS_HANDLE 2
127
128 struct rte_flow {
129 LIST_ENTRY(rte_flow) next; /* Pointer to the next rte_flow structure */
130 struct rte_flow *remote_flow; /* associated remote flow */
131 int bpf_fd[SEC_MAX]; /* list of bfs fds per ELF section */
132 uint32_t key_idx; /* RSS rule key index into BPF map */
133 struct nlmsg msg;
134 };
135
136 struct convert_data {
137 uint16_t eth_type;
138 uint16_t ip_proto;
139 uint8_t vlan;
140 struct rte_flow *flow;
141 };
142
143 struct remote_rule {
144 struct rte_flow_attr attr;
145 struct rte_flow_item items[2];
146 struct rte_flow_action actions[2];
147 int mirred;
148 };
149
150 struct action_data {
151 char id[16];
152
153 union {
154 struct tc_gact gact;
155 struct tc_mirred mirred;
156 struct skbedit {
157 struct tc_skbedit skbedit;
158 uint16_t queue;
159 } skbedit;
160 struct bpf {
161 struct tc_act_bpf bpf;
162 int bpf_fd;
163 const char *annotation;
164 } bpf;
165 };
166 };
167
168 static int tap_flow_create_eth(const struct rte_flow_item *item, void *data);
169 static int tap_flow_create_vlan(const struct rte_flow_item *item, void *data);
170 static int tap_flow_create_ipv4(const struct rte_flow_item *item, void *data);
171 static int tap_flow_create_ipv6(const struct rte_flow_item *item, void *data);
172 static int tap_flow_create_udp(const struct rte_flow_item *item, void *data);
173 static int tap_flow_create_tcp(const struct rte_flow_item *item, void *data);
174 static int
175 tap_flow_validate(struct rte_eth_dev *dev,
176 const struct rte_flow_attr *attr,
177 const struct rte_flow_item items[],
178 const struct rte_flow_action actions[],
179 struct rte_flow_error *error);
180
181 static struct rte_flow *
182 tap_flow_create(struct rte_eth_dev *dev,
183 const struct rte_flow_attr *attr,
184 const struct rte_flow_item items[],
185 const struct rte_flow_action actions[],
186 struct rte_flow_error *error);
187
188 static void
189 tap_flow_free(struct pmd_internals *pmd,
190 struct rte_flow *flow);
191
192 static int
193 tap_flow_destroy(struct rte_eth_dev *dev,
194 struct rte_flow *flow,
195 struct rte_flow_error *error);
196
197 static int
198 tap_flow_isolate(struct rte_eth_dev *dev,
199 int set,
200 struct rte_flow_error *error);
201
202 static int bpf_rss_key(enum bpf_rss_key_e cmd, __u32 *key_idx);
203 static int rss_enable(struct pmd_internals *pmd,
204 const struct rte_flow_attr *attr,
205 struct rte_flow_error *error);
206 static int rss_add_actions(struct rte_flow *flow, struct pmd_internals *pmd,
207 const struct rte_flow_action_rss *rss,
208 struct rte_flow_error *error);
209
210 static const struct rte_flow_ops tap_flow_ops = {
211 .validate = tap_flow_validate,
212 .create = tap_flow_create,
213 .destroy = tap_flow_destroy,
214 .flush = tap_flow_flush,
215 .isolate = tap_flow_isolate,
216 };
217
218 /* Static initializer for items. */
219 #define ITEMS(...) \
220 (const enum rte_flow_item_type []){ \
221 __VA_ARGS__, RTE_FLOW_ITEM_TYPE_END, \
222 }
223
224 /* Structure to generate a simple graph of layers supported by the NIC. */
225 struct tap_flow_items {
226 /* Bit-mask corresponding to what is supported for this item. */
227 const void *mask;
228 const unsigned int mask_sz; /* Bit-mask size in bytes. */
229 /*
230 * Bit-mask corresponding to the default mask, if none is provided
231 * along with the item.
232 */
233 const void *default_mask;
234 /**
235 * Conversion function from rte_flow to netlink attributes.
236 *
237 * @param item
238 * rte_flow item to convert.
239 * @param data
240 * Internal structure to store the conversion.
241 *
242 * @return
243 * 0 on success, negative value otherwise.
244 */
245 int (*convert)(const struct rte_flow_item *item, void *data);
246 /** List of possible following items. */
247 const enum rte_flow_item_type *const items;
248 };
249
250 /* Graph of supported items and associated actions. */
251 static const struct tap_flow_items tap_flow_items[] = {
252 [RTE_FLOW_ITEM_TYPE_END] = {
253 .items = ITEMS(RTE_FLOW_ITEM_TYPE_ETH),
254 },
255 [RTE_FLOW_ITEM_TYPE_ETH] = {
256 .items = ITEMS(
257 RTE_FLOW_ITEM_TYPE_VLAN,
258 RTE_FLOW_ITEM_TYPE_IPV4,
259 RTE_FLOW_ITEM_TYPE_IPV6),
260 .mask = &(const struct rte_flow_item_eth){
261 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
262 .src.addr_bytes = "\xff\xff\xff\xff\xff\xff",
263 .type = -1,
264 },
265 .mask_sz = sizeof(struct rte_flow_item_eth),
266 .default_mask = &rte_flow_item_eth_mask,
267 .convert = tap_flow_create_eth,
268 },
269 [RTE_FLOW_ITEM_TYPE_VLAN] = {
270 .items = ITEMS(RTE_FLOW_ITEM_TYPE_IPV4,
271 RTE_FLOW_ITEM_TYPE_IPV6),
272 .mask = &(const struct rte_flow_item_vlan){
273 /* DEI matching is not supported */
274 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
275 .tci = 0xffef,
276 #else
277 .tci = 0xefff,
278 #endif
279 .inner_type = -1,
280 },
281 .mask_sz = sizeof(struct rte_flow_item_vlan),
282 .default_mask = &rte_flow_item_vlan_mask,
283 .convert = tap_flow_create_vlan,
284 },
285 [RTE_FLOW_ITEM_TYPE_IPV4] = {
286 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
287 RTE_FLOW_ITEM_TYPE_TCP),
288 .mask = &(const struct rte_flow_item_ipv4){
289 .hdr = {
290 .src_addr = -1,
291 .dst_addr = -1,
292 .next_proto_id = -1,
293 },
294 },
295 .mask_sz = sizeof(struct rte_flow_item_ipv4),
296 .default_mask = &rte_flow_item_ipv4_mask,
297 .convert = tap_flow_create_ipv4,
298 },
299 [RTE_FLOW_ITEM_TYPE_IPV6] = {
300 .items = ITEMS(RTE_FLOW_ITEM_TYPE_UDP,
301 RTE_FLOW_ITEM_TYPE_TCP),
302 .mask = &(const struct rte_flow_item_ipv6){
303 .hdr = {
304 .src_addr = {
305 "\xff\xff\xff\xff\xff\xff\xff\xff"
306 "\xff\xff\xff\xff\xff\xff\xff\xff",
307 },
308 .dst_addr = {
309 "\xff\xff\xff\xff\xff\xff\xff\xff"
310 "\xff\xff\xff\xff\xff\xff\xff\xff",
311 },
312 .proto = -1,
313 },
314 },
315 .mask_sz = sizeof(struct rte_flow_item_ipv6),
316 .default_mask = &rte_flow_item_ipv6_mask,
317 .convert = tap_flow_create_ipv6,
318 },
319 [RTE_FLOW_ITEM_TYPE_UDP] = {
320 .mask = &(const struct rte_flow_item_udp){
321 .hdr = {
322 .src_port = -1,
323 .dst_port = -1,
324 },
325 },
326 .mask_sz = sizeof(struct rte_flow_item_udp),
327 .default_mask = &rte_flow_item_udp_mask,
328 .convert = tap_flow_create_udp,
329 },
330 [RTE_FLOW_ITEM_TYPE_TCP] = {
331 .mask = &(const struct rte_flow_item_tcp){
332 .hdr = {
333 .src_port = -1,
334 .dst_port = -1,
335 },
336 },
337 .mask_sz = sizeof(struct rte_flow_item_tcp),
338 .default_mask = &rte_flow_item_tcp_mask,
339 .convert = tap_flow_create_tcp,
340 },
341 };
342
343 /*
344 * TC rules, by growing priority
345 *
346 * Remote netdevice Tap netdevice
347 * +-------------+-------------+ +-------------+-------------+
348 * | Ingress | Egress | | Ingress | Egress |
349 * |-------------|-------------| |-------------|-------------|
350 * | | \ / | | | REMOTE TX | prio 1
351 * | | \ / | | | \ / | prio 2
352 * | EXPLICIT | \ / | | EXPLICIT | \ / | .
353 * | | \ / | | | \ / | .
354 * | RULES | X | | RULES | X | .
355 * | . | / \ | | . | / \ | .
356 * | . | / \ | | . | / \ | .
357 * | . | / \ | | . | / \ | .
358 * | . | / \ | | . | / \ | .
359 *
360 * .... .... .... ....
361 *
362 * | . | \ / | | . | \ / | .
363 * | . | \ / | | . | \ / | .
364 * | | \ / | | | \ / |
365 * | LOCAL_MAC | \ / | | \ / | \ / | last prio - 5
366 * | PROMISC | X | | \ / | X | last prio - 4
367 * | ALLMULTI | / \ | | X | / \ | last prio - 3
368 * | BROADCAST | / \ | | / \ | / \ | last prio - 2
369 * | BROADCASTV6 | / \ | | / \ | / \ | last prio - 1
370 * | xx | / \ | | ISOLATE | / \ | last prio
371 * +-------------+-------------+ +-------------+-------------+
372 *
373 * The implicit flow rules are stored in a list in with mandatorily the last two
374 * being the ISOLATE and REMOTE_TX rules. e.g.:
375 *
376 * LOCAL_MAC -> BROADCAST -> BROADCASTV6 -> REMOTE_TX -> ISOLATE -> NULL
377 *
378 * That enables tap_flow_isolate() to remove implicit rules by popping the list
379 * head and remove it as long as it applies on the remote netdevice. The
380 * implicit rule for TX redirection is not removed, as isolate concerns only
381 * incoming traffic.
382 */
383
384 static struct remote_rule implicit_rte_flows[TAP_REMOTE_MAX_IDX] = {
385 [TAP_REMOTE_LOCAL_MAC] = {
386 .attr = {
387 .group = MAX_GROUP,
388 .priority = PRIORITY_MASK - TAP_REMOTE_LOCAL_MAC,
389 .ingress = 1,
390 },
391 .items[0] = {
392 .type = RTE_FLOW_ITEM_TYPE_ETH,
393 .mask = &(const struct rte_flow_item_eth){
394 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
395 },
396 },
397 .items[1] = {
398 .type = RTE_FLOW_ITEM_TYPE_END,
399 },
400 .mirred = TCA_EGRESS_REDIR,
401 },
402 [TAP_REMOTE_BROADCAST] = {
403 .attr = {
404 .group = MAX_GROUP,
405 .priority = PRIORITY_MASK - TAP_REMOTE_BROADCAST,
406 .ingress = 1,
407 },
408 .items[0] = {
409 .type = RTE_FLOW_ITEM_TYPE_ETH,
410 .mask = &(const struct rte_flow_item_eth){
411 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
412 },
413 .spec = &(const struct rte_flow_item_eth){
414 .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
415 },
416 },
417 .items[1] = {
418 .type = RTE_FLOW_ITEM_TYPE_END,
419 },
420 .mirred = TCA_EGRESS_MIRROR,
421 },
422 [TAP_REMOTE_BROADCASTV6] = {
423 .attr = {
424 .group = MAX_GROUP,
425 .priority = PRIORITY_MASK - TAP_REMOTE_BROADCASTV6,
426 .ingress = 1,
427 },
428 .items[0] = {
429 .type = RTE_FLOW_ITEM_TYPE_ETH,
430 .mask = &(const struct rte_flow_item_eth){
431 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00",
432 },
433 .spec = &(const struct rte_flow_item_eth){
434 .dst.addr_bytes = "\x33\x33\x00\x00\x00\x00",
435 },
436 },
437 .items[1] = {
438 .type = RTE_FLOW_ITEM_TYPE_END,
439 },
440 .mirred = TCA_EGRESS_MIRROR,
441 },
442 [TAP_REMOTE_PROMISC] = {
443 .attr = {
444 .group = MAX_GROUP,
445 .priority = PRIORITY_MASK - TAP_REMOTE_PROMISC,
446 .ingress = 1,
447 },
448 .items[0] = {
449 .type = RTE_FLOW_ITEM_TYPE_VOID,
450 },
451 .items[1] = {
452 .type = RTE_FLOW_ITEM_TYPE_END,
453 },
454 .mirred = TCA_EGRESS_MIRROR,
455 },
456 [TAP_REMOTE_ALLMULTI] = {
457 .attr = {
458 .group = MAX_GROUP,
459 .priority = PRIORITY_MASK - TAP_REMOTE_ALLMULTI,
460 .ingress = 1,
461 },
462 .items[0] = {
463 .type = RTE_FLOW_ITEM_TYPE_ETH,
464 .mask = &(const struct rte_flow_item_eth){
465 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
466 },
467 .spec = &(const struct rte_flow_item_eth){
468 .dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
469 },
470 },
471 .items[1] = {
472 .type = RTE_FLOW_ITEM_TYPE_END,
473 },
474 .mirred = TCA_EGRESS_MIRROR,
475 },
476 [TAP_REMOTE_TX] = {
477 .attr = {
478 .group = 0,
479 .priority = TAP_REMOTE_TX,
480 .egress = 1,
481 },
482 .items[0] = {
483 .type = RTE_FLOW_ITEM_TYPE_VOID,
484 },
485 .items[1] = {
486 .type = RTE_FLOW_ITEM_TYPE_END,
487 },
488 .mirred = TCA_EGRESS_MIRROR,
489 },
490 [TAP_ISOLATE] = {
491 .attr = {
492 .group = MAX_GROUP,
493 .priority = PRIORITY_MASK - TAP_ISOLATE,
494 .ingress = 1,
495 },
496 .items[0] = {
497 .type = RTE_FLOW_ITEM_TYPE_VOID,
498 },
499 .items[1] = {
500 .type = RTE_FLOW_ITEM_TYPE_END,
501 },
502 },
503 };
504
505 /**
506 * Make as much checks as possible on an Ethernet item, and if a flow is
507 * provided, fill it appropriately with Ethernet info.
508 *
509 * @param[in] item
510 * Item specification.
511 * @param[in, out] data
512 * Additional data structure to tell next layers we've been here.
513 *
514 * @return
515 * 0 if checks are alright, -1 otherwise.
516 */
517 static int
518 tap_flow_create_eth(const struct rte_flow_item *item, void *data)
519 {
520 struct convert_data *info = (struct convert_data *)data;
521 const struct rte_flow_item_eth *spec = item->spec;
522 const struct rte_flow_item_eth *mask = item->mask;
523 struct rte_flow *flow = info->flow;
524 struct nlmsg *msg;
525
526 /* use default mask if none provided */
527 if (!mask)
528 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_ETH].default_mask;
529 /* TC does not support eth_type masking. Only accept if exact match. */
530 if (mask->type && mask->type != 0xffff)
531 return -1;
532 if (!spec)
533 return 0;
534 /* store eth_type for consistency if ipv4/6 pattern item comes next */
535 if (spec->type & mask->type)
536 info->eth_type = spec->type;
537 if (!flow)
538 return 0;
539 msg = &flow->msg;
540 if (!rte_is_zero_ether_addr(&mask->dst)) {
541 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_DST,
542 RTE_ETHER_ADDR_LEN,
543 &spec->dst.addr_bytes);
544 tap_nlattr_add(&msg->nh,
545 TCA_FLOWER_KEY_ETH_DST_MASK, RTE_ETHER_ADDR_LEN,
546 &mask->dst.addr_bytes);
547 }
548 if (!rte_is_zero_ether_addr(&mask->src)) {
549 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_SRC,
550 RTE_ETHER_ADDR_LEN,
551 &spec->src.addr_bytes);
552 tap_nlattr_add(&msg->nh,
553 TCA_FLOWER_KEY_ETH_SRC_MASK, RTE_ETHER_ADDR_LEN,
554 &mask->src.addr_bytes);
555 }
556 return 0;
557 }
558
559 /**
560 * Make as much checks as possible on a VLAN item, and if a flow is provided,
561 * fill it appropriately with VLAN info.
562 *
563 * @param[in] item
564 * Item specification.
565 * @param[in, out] data
566 * Additional data structure to tell next layers we've been here.
567 *
568 * @return
569 * 0 if checks are alright, -1 otherwise.
570 */
571 static int
572 tap_flow_create_vlan(const struct rte_flow_item *item, void *data)
573 {
574 struct convert_data *info = (struct convert_data *)data;
575 const struct rte_flow_item_vlan *spec = item->spec;
576 const struct rte_flow_item_vlan *mask = item->mask;
577 struct rte_flow *flow = info->flow;
578 struct nlmsg *msg;
579
580 /* use default mask if none provided */
581 if (!mask)
582 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_VLAN].default_mask;
583 /* Outer TPID cannot be matched. */
584 if (info->eth_type)
585 return -1;
586 /* Double-tagging not supported. */
587 if (info->vlan)
588 return -1;
589 info->vlan = 1;
590 if (mask->inner_type) {
591 /* TC does not support partial eth_type masking */
592 if (mask->inner_type != RTE_BE16(0xffff))
593 return -1;
594 info->eth_type = spec->inner_type;
595 }
596 if (!flow)
597 return 0;
598 msg = &flow->msg;
599 msg->t.tcm_info = TC_H_MAKE(msg->t.tcm_info, htons(ETH_P_8021Q));
600 #define VLAN_PRIO(tci) ((tci) >> 13)
601 #define VLAN_ID(tci) ((tci) & 0xfff)
602 if (!spec)
603 return 0;
604 if (spec->tci) {
605 uint16_t tci = ntohs(spec->tci) & mask->tci;
606 uint16_t prio = VLAN_PRIO(tci);
607 uint8_t vid = VLAN_ID(tci);
608
609 if (prio)
610 tap_nlattr_add8(&msg->nh,
611 TCA_FLOWER_KEY_VLAN_PRIO, prio);
612 if (vid)
613 tap_nlattr_add16(&msg->nh,
614 TCA_FLOWER_KEY_VLAN_ID, vid);
615 }
616 return 0;
617 }
618
619 /**
620 * Make as much checks as possible on an IPv4 item, and if a flow is provided,
621 * fill it appropriately with IPv4 info.
622 *
623 * @param[in] item
624 * Item specification.
625 * @param[in, out] data
626 * Additional data structure to tell next layers we've been here.
627 *
628 * @return
629 * 0 if checks are alright, -1 otherwise.
630 */
631 static int
632 tap_flow_create_ipv4(const struct rte_flow_item *item, void *data)
633 {
634 struct convert_data *info = (struct convert_data *)data;
635 const struct rte_flow_item_ipv4 *spec = item->spec;
636 const struct rte_flow_item_ipv4 *mask = item->mask;
637 struct rte_flow *flow = info->flow;
638 struct nlmsg *msg;
639
640 /* use default mask if none provided */
641 if (!mask)
642 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_IPV4].default_mask;
643 /* check that previous eth type is compatible with ipv4 */
644 if (info->eth_type && info->eth_type != htons(ETH_P_IP))
645 return -1;
646 /* store ip_proto for consistency if udp/tcp pattern item comes next */
647 if (spec)
648 info->ip_proto = spec->hdr.next_proto_id;
649 if (!flow)
650 return 0;
651 msg = &flow->msg;
652 if (!info->eth_type)
653 info->eth_type = htons(ETH_P_IP);
654 if (!spec)
655 return 0;
656 if (mask->hdr.dst_addr) {
657 tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST,
658 spec->hdr.dst_addr);
659 tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST_MASK,
660 mask->hdr.dst_addr);
661 }
662 if (mask->hdr.src_addr) {
663 tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC,
664 spec->hdr.src_addr);
665 tap_nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC_MASK,
666 mask->hdr.src_addr);
667 }
668 if (spec->hdr.next_proto_id)
669 tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO,
670 spec->hdr.next_proto_id);
671 return 0;
672 }
673
674 /**
675 * Make as much checks as possible on an IPv6 item, and if a flow is provided,
676 * fill it appropriately with IPv6 info.
677 *
678 * @param[in] item
679 * Item specification.
680 * @param[in, out] data
681 * Additional data structure to tell next layers we've been here.
682 *
683 * @return
684 * 0 if checks are alright, -1 otherwise.
685 */
686 static int
687 tap_flow_create_ipv6(const struct rte_flow_item *item, void *data)
688 {
689 struct convert_data *info = (struct convert_data *)data;
690 const struct rte_flow_item_ipv6 *spec = item->spec;
691 const struct rte_flow_item_ipv6 *mask = item->mask;
692 struct rte_flow *flow = info->flow;
693 uint8_t empty_addr[16] = { 0 };
694 struct nlmsg *msg;
695
696 /* use default mask if none provided */
697 if (!mask)
698 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_IPV6].default_mask;
699 /* check that previous eth type is compatible with ipv6 */
700 if (info->eth_type && info->eth_type != htons(ETH_P_IPV6))
701 return -1;
702 /* store ip_proto for consistency if udp/tcp pattern item comes next */
703 if (spec)
704 info->ip_proto = spec->hdr.proto;
705 if (!flow)
706 return 0;
707 msg = &flow->msg;
708 if (!info->eth_type)
709 info->eth_type = htons(ETH_P_IPV6);
710 if (!spec)
711 return 0;
712 if (memcmp(mask->hdr.dst_addr, empty_addr, 16)) {
713 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST,
714 sizeof(spec->hdr.dst_addr), &spec->hdr.dst_addr);
715 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST_MASK,
716 sizeof(mask->hdr.dst_addr), &mask->hdr.dst_addr);
717 }
718 if (memcmp(mask->hdr.src_addr, empty_addr, 16)) {
719 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC,
720 sizeof(spec->hdr.src_addr), &spec->hdr.src_addr);
721 tap_nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC_MASK,
722 sizeof(mask->hdr.src_addr), &mask->hdr.src_addr);
723 }
724 if (spec->hdr.proto)
725 tap_nlattr_add8(&msg->nh,
726 TCA_FLOWER_KEY_IP_PROTO, spec->hdr.proto);
727 return 0;
728 }
729
730 /**
731 * Make as much checks as possible on a UDP item, and if a flow is provided,
732 * fill it appropriately with UDP info.
733 *
734 * @param[in] item
735 * Item specification.
736 * @param[in, out] data
737 * Additional data structure to tell next layers we've been here.
738 *
739 * @return
740 * 0 if checks are alright, -1 otherwise.
741 */
742 static int
743 tap_flow_create_udp(const struct rte_flow_item *item, void *data)
744 {
745 struct convert_data *info = (struct convert_data *)data;
746 const struct rte_flow_item_udp *spec = item->spec;
747 const struct rte_flow_item_udp *mask = item->mask;
748 struct rte_flow *flow = info->flow;
749 struct nlmsg *msg;
750
751 /* use default mask if none provided */
752 if (!mask)
753 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_UDP].default_mask;
754 /* check that previous ip_proto is compatible with udp */
755 if (info->ip_proto && info->ip_proto != IPPROTO_UDP)
756 return -1;
757 /* TC does not support UDP port masking. Only accept if exact match. */
758 if ((mask->hdr.src_port && mask->hdr.src_port != 0xffff) ||
759 (mask->hdr.dst_port && mask->hdr.dst_port != 0xffff))
760 return -1;
761 if (!flow)
762 return 0;
763 msg = &flow->msg;
764 tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_UDP);
765 if (!spec)
766 return 0;
767 if (mask->hdr.dst_port)
768 tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_DST,
769 spec->hdr.dst_port);
770 if (mask->hdr.src_port)
771 tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_SRC,
772 spec->hdr.src_port);
773 return 0;
774 }
775
776 /**
777 * Make as much checks as possible on a TCP item, and if a flow is provided,
778 * fill it appropriately with TCP info.
779 *
780 * @param[in] item
781 * Item specification.
782 * @param[in, out] data
783 * Additional data structure to tell next layers we've been here.
784 *
785 * @return
786 * 0 if checks are alright, -1 otherwise.
787 */
788 static int
789 tap_flow_create_tcp(const struct rte_flow_item *item, void *data)
790 {
791 struct convert_data *info = (struct convert_data *)data;
792 const struct rte_flow_item_tcp *spec = item->spec;
793 const struct rte_flow_item_tcp *mask = item->mask;
794 struct rte_flow *flow = info->flow;
795 struct nlmsg *msg;
796
797 /* use default mask if none provided */
798 if (!mask)
799 mask = tap_flow_items[RTE_FLOW_ITEM_TYPE_TCP].default_mask;
800 /* check that previous ip_proto is compatible with tcp */
801 if (info->ip_proto && info->ip_proto != IPPROTO_TCP)
802 return -1;
803 /* TC does not support TCP port masking. Only accept if exact match. */
804 if ((mask->hdr.src_port && mask->hdr.src_port != 0xffff) ||
805 (mask->hdr.dst_port && mask->hdr.dst_port != 0xffff))
806 return -1;
807 if (!flow)
808 return 0;
809 msg = &flow->msg;
810 tap_nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_TCP);
811 if (!spec)
812 return 0;
813 if (mask->hdr.dst_port)
814 tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_DST,
815 spec->hdr.dst_port);
816 if (mask->hdr.src_port)
817 tap_nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_SRC,
818 spec->hdr.src_port);
819 return 0;
820 }
821
822 /**
823 * Check support for a given item.
824 *
825 * @param[in] item
826 * Item specification.
827 * @param size
828 * Bit-Mask size in bytes.
829 * @param[in] supported_mask
830 * Bit-mask covering supported fields to compare with spec, last and mask in
831 * \item.
832 * @param[in] default_mask
833 * Bit-mask default mask if none is provided in \item.
834 *
835 * @return
836 * 0 on success.
837 */
838 static int
839 tap_flow_item_validate(const struct rte_flow_item *item,
840 unsigned int size,
841 const uint8_t *supported_mask,
842 const uint8_t *default_mask)
843 {
844 int ret = 0;
845
846 /* An empty layer is allowed, as long as all fields are NULL */
847 if (!item->spec && (item->mask || item->last))
848 return -1;
849 /* Is the item spec compatible with what the NIC supports? */
850 if (item->spec && !item->mask) {
851 unsigned int i;
852 const uint8_t *spec = item->spec;
853
854 for (i = 0; i < size; ++i)
855 if ((spec[i] | supported_mask[i]) != supported_mask[i])
856 return -1;
857 /* Is the default mask compatible with what the NIC supports? */
858 for (i = 0; i < size; i++)
859 if ((default_mask[i] | supported_mask[i]) !=
860 supported_mask[i])
861 return -1;
862 }
863 /* Is the item last compatible with what the NIC supports? */
864 if (item->last && !item->mask) {
865 unsigned int i;
866 const uint8_t *spec = item->last;
867
868 for (i = 0; i < size; ++i)
869 if ((spec[i] | supported_mask[i]) != supported_mask[i])
870 return -1;
871 }
872 /* Is the item mask compatible with what the NIC supports? */
873 if (item->mask) {
874 unsigned int i;
875 const uint8_t *spec = item->mask;
876
877 for (i = 0; i < size; ++i)
878 if ((spec[i] | supported_mask[i]) != supported_mask[i])
879 return -1;
880 }
881 /**
882 * Once masked, Are item spec and item last equal?
883 * TC does not support range so anything else is invalid.
884 */
885 if (item->spec && item->last) {
886 uint8_t spec[size];
887 uint8_t last[size];
888 const uint8_t *apply = default_mask;
889 unsigned int i;
890
891 if (item->mask)
892 apply = item->mask;
893 for (i = 0; i < size; ++i) {
894 spec[i] = ((const uint8_t *)item->spec)[i] & apply[i];
895 last[i] = ((const uint8_t *)item->last)[i] & apply[i];
896 }
897 ret = memcmp(spec, last, size);
898 }
899 return ret;
900 }
901
902 /**
903 * Configure the kernel with a TC action and its configured parameters
904 * Handled actions: "gact", "mirred", "skbedit", "bpf"
905 *
906 * @param[in] flow
907 * Pointer to rte flow containing the netlink message
908 *
909 * @param[in, out] act_index
910 * Pointer to action sequence number in the TC command
911 *
912 * @param[in] adata
913 * Pointer to struct holding the action parameters
914 *
915 * @return
916 * -1 on failure, 0 on success
917 */
918 static int
919 add_action(struct rte_flow *flow, size_t *act_index, struct action_data *adata)
920 {
921 struct nlmsg *msg = &flow->msg;
922
923 if (tap_nlattr_nested_start(msg, (*act_index)++) < 0)
924 return -1;
925
926 tap_nlattr_add(&msg->nh, TCA_ACT_KIND,
927 strlen(adata->id) + 1, adata->id);
928 if (tap_nlattr_nested_start(msg, TCA_ACT_OPTIONS) < 0)
929 return -1;
930 if (strcmp("gact", adata->id) == 0) {
931 tap_nlattr_add(&msg->nh, TCA_GACT_PARMS, sizeof(adata->gact),
932 &adata->gact);
933 } else if (strcmp("mirred", adata->id) == 0) {
934 if (adata->mirred.eaction == TCA_EGRESS_MIRROR)
935 adata->mirred.action = TC_ACT_PIPE;
936 else /* REDIRECT */
937 adata->mirred.action = TC_ACT_STOLEN;
938 tap_nlattr_add(&msg->nh, TCA_MIRRED_PARMS,
939 sizeof(adata->mirred),
940 &adata->mirred);
941 } else if (strcmp("skbedit", adata->id) == 0) {
942 tap_nlattr_add(&msg->nh, TCA_SKBEDIT_PARMS,
943 sizeof(adata->skbedit.skbedit),
944 &adata->skbedit.skbedit);
945 tap_nlattr_add16(&msg->nh, TCA_SKBEDIT_QUEUE_MAPPING,
946 adata->skbedit.queue);
947 } else if (strcmp("bpf", adata->id) == 0) {
948 tap_nlattr_add32(&msg->nh, TCA_ACT_BPF_FD, adata->bpf.bpf_fd);
949 tap_nlattr_add(&msg->nh, TCA_ACT_BPF_NAME,
950 strlen(adata->bpf.annotation) + 1,
951 adata->bpf.annotation);
952 tap_nlattr_add(&msg->nh, TCA_ACT_BPF_PARMS,
953 sizeof(adata->bpf.bpf),
954 &adata->bpf.bpf);
955 } else {
956 return -1;
957 }
958 tap_nlattr_nested_finish(msg); /* nested TCA_ACT_OPTIONS */
959 tap_nlattr_nested_finish(msg); /* nested act_index */
960 return 0;
961 }
962
963 /**
964 * Helper function to send a serie of TC actions to the kernel
965 *
966 * @param[in] flow
967 * Pointer to rte flow containing the netlink message
968 *
969 * @param[in] nb_actions
970 * Number of actions in an array of action structs
971 *
972 * @param[in] data
973 * Pointer to an array of action structs
974 *
975 * @param[in] classifier_actions
976 * The classifier on behave of which the actions are configured
977 *
978 * @return
979 * -1 on failure, 0 on success
980 */
981 static int
982 add_actions(struct rte_flow *flow, int nb_actions, struct action_data *data,
983 int classifier_action)
984 {
985 struct nlmsg *msg = &flow->msg;
986 size_t act_index = 1;
987 int i;
988
989 if (tap_nlattr_nested_start(msg, classifier_action) < 0)
990 return -1;
991 for (i = 0; i < nb_actions; i++)
992 if (add_action(flow, &act_index, data + i) < 0)
993 return -1;
994 tap_nlattr_nested_finish(msg); /* nested TCA_FLOWER_ACT */
995 return 0;
996 }
997
998 /**
999 * Validate a flow supported by TC.
1000 * If flow param is not NULL, then also fill the netlink message inside.
1001 *
1002 * @param pmd
1003 * Pointer to private structure.
1004 * @param[in] attr
1005 * Flow rule attributes.
1006 * @param[in] pattern
1007 * Pattern specification (list terminated by the END pattern item).
1008 * @param[in] actions
1009 * Associated actions (list terminated by the END action).
1010 * @param[out] error
1011 * Perform verbose error reporting if not NULL.
1012 * @param[in, out] flow
1013 * Flow structure to update.
1014 * @param[in] mirred
1015 * If set to TCA_EGRESS_REDIR, provided actions will be replaced with a
1016 * redirection to the tap netdevice, and the TC rule will be configured
1017 * on the remote netdevice in pmd.
1018 * If set to TCA_EGRESS_MIRROR, provided actions will be replaced with a
1019 * mirroring to the tap netdevice, and the TC rule will be configured
1020 * on the remote netdevice in pmd. Matching packets will thus be duplicated.
1021 * If set to 0, the standard behavior is to be used: set correct actions for
1022 * the TC rule, and apply it on the tap netdevice.
1023 *
1024 * @return
1025 * 0 on success, a negative errno value otherwise and rte_errno is set.
1026 */
1027 static int
1028 priv_flow_process(struct pmd_internals *pmd,
1029 const struct rte_flow_attr *attr,
1030 const struct rte_flow_item items[],
1031 const struct rte_flow_action actions[],
1032 struct rte_flow_error *error,
1033 struct rte_flow *flow,
1034 int mirred)
1035 {
1036 const struct tap_flow_items *cur_item = tap_flow_items;
1037 struct convert_data data = {
1038 .eth_type = 0,
1039 .ip_proto = 0,
1040 .flow = flow,
1041 };
1042 int action = 0; /* Only one action authorized for now */
1043
1044 if (attr->transfer) {
1045 rte_flow_error_set(
1046 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
1047 NULL, "transfer is not supported");
1048 return -rte_errno;
1049 }
1050 if (attr->group > MAX_GROUP) {
1051 rte_flow_error_set(
1052 error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
1053 NULL, "group value too big: cannot exceed 15");
1054 return -rte_errno;
1055 }
1056 if (attr->priority > MAX_PRIORITY) {
1057 rte_flow_error_set(
1058 error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1059 NULL, "priority value too big");
1060 return -rte_errno;
1061 } else if (flow) {
1062 uint16_t group = attr->group << GROUP_SHIFT;
1063 uint16_t prio = group | (attr->priority +
1064 RSS_PRIORITY_OFFSET + PRIORITY_OFFSET);
1065 flow->msg.t.tcm_info = TC_H_MAKE(prio << 16,
1066 flow->msg.t.tcm_info);
1067 }
1068 if (flow) {
1069 if (mirred) {
1070 /*
1071 * If attr->ingress, the rule applies on remote ingress
1072 * to match incoming packets
1073 * If attr->egress, the rule applies on tap ingress (as
1074 * seen from the kernel) to deal with packets going out
1075 * from the DPDK app.
1076 */
1077 flow->msg.t.tcm_parent = TC_H_MAKE(TC_H_INGRESS, 0);
1078 } else {
1079 /* Standard rule on tap egress (kernel standpoint). */
1080 flow->msg.t.tcm_parent =
1081 TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0);
1082 }
1083 /* use flower filter type */
1084 tap_nlattr_add(&flow->msg.nh, TCA_KIND, sizeof("flower"), "flower");
1085 if (tap_nlattr_nested_start(&flow->msg, TCA_OPTIONS) < 0)
1086 goto exit_item_not_supported;
1087 }
1088 for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) {
1089 const struct tap_flow_items *token = NULL;
1090 unsigned int i;
1091 int err = 0;
1092
1093 if (items->type == RTE_FLOW_ITEM_TYPE_VOID)
1094 continue;
1095 for (i = 0;
1096 cur_item->items &&
1097 cur_item->items[i] != RTE_FLOW_ITEM_TYPE_END;
1098 ++i) {
1099 if (cur_item->items[i] == items->type) {
1100 token = &tap_flow_items[items->type];
1101 break;
1102 }
1103 }
1104 if (!token)
1105 goto exit_item_not_supported;
1106 cur_item = token;
1107 err = tap_flow_item_validate(
1108 items, cur_item->mask_sz,
1109 (const uint8_t *)cur_item->mask,
1110 (const uint8_t *)cur_item->default_mask);
1111 if (err)
1112 goto exit_item_not_supported;
1113 if (flow && cur_item->convert) {
1114 err = cur_item->convert(items, &data);
1115 if (err)
1116 goto exit_item_not_supported;
1117 }
1118 }
1119 if (flow) {
1120 if (data.vlan) {
1121 tap_nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE,
1122 htons(ETH_P_8021Q));
1123 tap_nlattr_add16(&flow->msg.nh,
1124 TCA_FLOWER_KEY_VLAN_ETH_TYPE,
1125 data.eth_type ?
1126 data.eth_type : htons(ETH_P_ALL));
1127 } else if (data.eth_type) {
1128 tap_nlattr_add16(&flow->msg.nh, TCA_FLOWER_KEY_ETH_TYPE,
1129 data.eth_type);
1130 }
1131 }
1132 if (mirred && flow) {
1133 struct action_data adata = {
1134 .id = "mirred",
1135 .mirred = {
1136 .eaction = mirred,
1137 },
1138 };
1139
1140 /*
1141 * If attr->egress && mirred, then this is a special
1142 * case where the rule must be applied on the tap, to
1143 * redirect packets coming from the DPDK App, out
1144 * through the remote netdevice.
1145 */
1146 adata.mirred.ifindex = attr->ingress ? pmd->if_index :
1147 pmd->remote_if_index;
1148 if (mirred == TCA_EGRESS_MIRROR)
1149 adata.mirred.action = TC_ACT_PIPE;
1150 else
1151 adata.mirred.action = TC_ACT_STOLEN;
1152 if (add_actions(flow, 1, &adata, TCA_FLOWER_ACT) < 0)
1153 goto exit_action_not_supported;
1154 else
1155 goto end;
1156 }
1157 actions:
1158 for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
1159 int err = 0;
1160
1161 if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) {
1162 continue;
1163 } else if (actions->type == RTE_FLOW_ACTION_TYPE_DROP) {
1164 if (action)
1165 goto exit_action_not_supported;
1166 action = 1;
1167 if (flow) {
1168 struct action_data adata = {
1169 .id = "gact",
1170 .gact = {
1171 .action = TC_ACT_SHOT,
1172 },
1173 };
1174
1175 err = add_actions(flow, 1, &adata,
1176 TCA_FLOWER_ACT);
1177 }
1178 } else if (actions->type == RTE_FLOW_ACTION_TYPE_PASSTHRU) {
1179 if (action)
1180 goto exit_action_not_supported;
1181 action = 1;
1182 if (flow) {
1183 struct action_data adata = {
1184 .id = "gact",
1185 .gact = {
1186 /* continue */
1187 .action = TC_ACT_UNSPEC,
1188 },
1189 };
1190
1191 err = add_actions(flow, 1, &adata,
1192 TCA_FLOWER_ACT);
1193 }
1194 } else if (actions->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
1195 const struct rte_flow_action_queue *queue =
1196 (const struct rte_flow_action_queue *)
1197 actions->conf;
1198
1199 if (action)
1200 goto exit_action_not_supported;
1201 action = 1;
1202 if (!queue ||
1203 (queue->index > pmd->dev->data->nb_rx_queues - 1))
1204 goto exit_action_not_supported;
1205 if (flow) {
1206 struct action_data adata = {
1207 .id = "skbedit",
1208 .skbedit = {
1209 .skbedit = {
1210 .action = TC_ACT_PIPE,
1211 },
1212 .queue = queue->index,
1213 },
1214 };
1215
1216 err = add_actions(flow, 1, &adata,
1217 TCA_FLOWER_ACT);
1218 }
1219 } else if (actions->type == RTE_FLOW_ACTION_TYPE_RSS) {
1220 const struct rte_flow_action_rss *rss =
1221 (const struct rte_flow_action_rss *)
1222 actions->conf;
1223
1224 if (action++)
1225 goto exit_action_not_supported;
1226
1227 if (!pmd->rss_enabled) {
1228 err = rss_enable(pmd, attr, error);
1229 if (err)
1230 goto exit_action_not_supported;
1231 }
1232 if (flow)
1233 err = rss_add_actions(flow, pmd, rss, error);
1234 } else {
1235 goto exit_action_not_supported;
1236 }
1237 if (err)
1238 goto exit_action_not_supported;
1239 }
1240 /* When fate is unknown, drop traffic. */
1241 if (!action) {
1242 static const struct rte_flow_action drop[] = {
1243 { .type = RTE_FLOW_ACTION_TYPE_DROP, },
1244 { .type = RTE_FLOW_ACTION_TYPE_END, },
1245 };
1246
1247 actions = drop;
1248 goto actions;
1249 }
1250 end:
1251 if (flow)
1252 tap_nlattr_nested_finish(&flow->msg); /* nested TCA_OPTIONS */
1253 return 0;
1254 exit_item_not_supported:
1255 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM,
1256 items, "item not supported");
1257 return -rte_errno;
1258 exit_action_not_supported:
1259 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
1260 actions, "action not supported");
1261 return -rte_errno;
1262 }
1263
1264
1265
1266 /**
1267 * Validate a flow.
1268 *
1269 * @see rte_flow_validate()
1270 * @see rte_flow_ops
1271 */
1272 static int
1273 tap_flow_validate(struct rte_eth_dev *dev,
1274 const struct rte_flow_attr *attr,
1275 const struct rte_flow_item items[],
1276 const struct rte_flow_action actions[],
1277 struct rte_flow_error *error)
1278 {
1279 struct pmd_internals *pmd = dev->data->dev_private;
1280
1281 return priv_flow_process(pmd, attr, items, actions, error, NULL, 0);
1282 }
1283
1284 /**
1285 * Set a unique handle in a flow.
1286 *
1287 * The kernel supports TC rules with equal priority, as long as they use the
1288 * same matching fields (e.g.: dst mac and ipv4) with different values (and
1289 * full mask to ensure no collision is possible).
1290 * In those rules, the handle (uint32_t) is the part that would identify
1291 * specifically each rule.
1292 *
1293 * On 32-bit architectures, the handle can simply be the flow's pointer address.
1294 * On 64-bit architectures, we rely on jhash(flow) to find a (sufficiently)
1295 * unique handle.
1296 *
1297 * @param[in, out] flow
1298 * The flow that needs its handle set.
1299 */
1300 static void
1301 tap_flow_set_handle(struct rte_flow *flow)
1302 {
1303 uint32_t handle = 0;
1304
1305 if (sizeof(flow) > 4)
1306 handle = rte_jhash(&flow, sizeof(flow), 1);
1307 else
1308 handle = (uintptr_t)flow;
1309 /* must be at least 1 to avoid letting the kernel choose one for us */
1310 if (!handle)
1311 handle = 1;
1312 flow->msg.t.tcm_handle = handle;
1313 }
1314
1315 /**
1316 * Free the flow opened file descriptors and allocated memory
1317 *
1318 * @param[in] flow
1319 * Pointer to the flow to free
1320 *
1321 */
1322 static void
1323 tap_flow_free(struct pmd_internals *pmd, struct rte_flow *flow)
1324 {
1325 int i;
1326
1327 if (!flow)
1328 return;
1329
1330 if (pmd->rss_enabled) {
1331 /* Close flow BPF file descriptors */
1332 for (i = 0; i < SEC_MAX; i++)
1333 if (flow->bpf_fd[i] != 0) {
1334 close(flow->bpf_fd[i]);
1335 flow->bpf_fd[i] = 0;
1336 }
1337
1338 /* Release the map key for this RSS rule */
1339 bpf_rss_key(KEY_CMD_RELEASE, &flow->key_idx);
1340 flow->key_idx = 0;
1341 }
1342
1343 /* Free flow allocated memory */
1344 rte_free(flow);
1345 }
1346
1347 /**
1348 * Create a flow.
1349 *
1350 * @see rte_flow_create()
1351 * @see rte_flow_ops
1352 */
1353 static struct rte_flow *
1354 tap_flow_create(struct rte_eth_dev *dev,
1355 const struct rte_flow_attr *attr,
1356 const struct rte_flow_item items[],
1357 const struct rte_flow_action actions[],
1358 struct rte_flow_error *error)
1359 {
1360 struct pmd_internals *pmd = dev->data->dev_private;
1361 struct rte_flow *remote_flow = NULL;
1362 struct rte_flow *flow = NULL;
1363 struct nlmsg *msg = NULL;
1364 int err;
1365
1366 if (!pmd->if_index) {
1367 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1368 NULL,
1369 "can't create rule, ifindex not found");
1370 goto fail;
1371 }
1372 /*
1373 * No rules configured through standard rte_flow should be set on the
1374 * priorities used by implicit rules.
1375 */
1376 if ((attr->group == MAX_GROUP) &&
1377 attr->priority > (MAX_PRIORITY - TAP_REMOTE_MAX_IDX)) {
1378 rte_flow_error_set(
1379 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY,
1380 NULL, "priority value too big");
1381 goto fail;
1382 }
1383 flow = rte_zmalloc(__func__, sizeof(struct rte_flow), 0);
1384 if (!flow) {
1385 rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1386 NULL, "cannot allocate memory for rte_flow");
1387 goto fail;
1388 }
1389 msg = &flow->msg;
1390 tc_init_msg(msg, pmd->if_index, RTM_NEWTFILTER,
1391 NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE);
1392 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1393 tap_flow_set_handle(flow);
1394 if (priv_flow_process(pmd, attr, items, actions, error, flow, 0))
1395 goto fail;
1396 err = tap_nl_send(pmd->nlsk_fd, &msg->nh);
1397 if (err < 0) {
1398 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1399 NULL, "couldn't send request to kernel");
1400 goto fail;
1401 }
1402 err = tap_nl_recv_ack(pmd->nlsk_fd);
1403 if (err < 0) {
1404 TAP_LOG(ERR,
1405 "Kernel refused TC filter rule creation (%d): %s",
1406 errno, strerror(errno));
1407 rte_flow_error_set(error, EEXIST, RTE_FLOW_ERROR_TYPE_HANDLE,
1408 NULL,
1409 "overlapping rules or Kernel too old for flower support");
1410 goto fail;
1411 }
1412 LIST_INSERT_HEAD(&pmd->flows, flow, next);
1413 /**
1414 * If a remote device is configured, a TC rule with identical items for
1415 * matching must be set on that device, with a single action: redirect
1416 * to the local pmd->if_index.
1417 */
1418 if (pmd->remote_if_index) {
1419 remote_flow = rte_zmalloc(__func__, sizeof(struct rte_flow), 0);
1420 if (!remote_flow) {
1421 rte_flow_error_set(
1422 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1423 "cannot allocate memory for rte_flow");
1424 goto fail;
1425 }
1426 msg = &remote_flow->msg;
1427 /* set the rule if_index for the remote netdevice */
1428 tc_init_msg(
1429 msg, pmd->remote_if_index, RTM_NEWTFILTER,
1430 NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE);
1431 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1432 tap_flow_set_handle(remote_flow);
1433 if (priv_flow_process(pmd, attr, items, NULL,
1434 error, remote_flow, TCA_EGRESS_REDIR)) {
1435 rte_flow_error_set(
1436 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1437 NULL, "rte flow rule validation failed");
1438 goto fail;
1439 }
1440 err = tap_nl_send(pmd->nlsk_fd, &msg->nh);
1441 if (err < 0) {
1442 rte_flow_error_set(
1443 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1444 NULL, "Failure sending nl request");
1445 goto fail;
1446 }
1447 err = tap_nl_recv_ack(pmd->nlsk_fd);
1448 if (err < 0) {
1449 TAP_LOG(ERR,
1450 "Kernel refused TC filter rule creation (%d): %s",
1451 errno, strerror(errno));
1452 rte_flow_error_set(
1453 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1454 NULL,
1455 "overlapping rules or Kernel too old for flower support");
1456 goto fail;
1457 }
1458 flow->remote_flow = remote_flow;
1459 }
1460 return flow;
1461 fail:
1462 if (remote_flow)
1463 rte_free(remote_flow);
1464 if (flow)
1465 tap_flow_free(pmd, flow);
1466 return NULL;
1467 }
1468
1469 /**
1470 * Destroy a flow using pointer to pmd_internal.
1471 *
1472 * @param[in, out] pmd
1473 * Pointer to private structure.
1474 * @param[in] flow
1475 * Pointer to the flow to destroy.
1476 * @param[in, out] error
1477 * Pointer to the flow error handler
1478 *
1479 * @return 0 if the flow could be destroyed, -1 otherwise.
1480 */
1481 static int
1482 tap_flow_destroy_pmd(struct pmd_internals *pmd,
1483 struct rte_flow *flow,
1484 struct rte_flow_error *error)
1485 {
1486 struct rte_flow *remote_flow = flow->remote_flow;
1487 int ret = 0;
1488
1489 LIST_REMOVE(flow, next);
1490 flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
1491 flow->msg.nh.nlmsg_type = RTM_DELTFILTER;
1492
1493 ret = tap_nl_send(pmd->nlsk_fd, &flow->msg.nh);
1494 if (ret < 0) {
1495 rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1496 NULL, "couldn't send request to kernel");
1497 goto end;
1498 }
1499 ret = tap_nl_recv_ack(pmd->nlsk_fd);
1500 /* If errno is ENOENT, the rule is already no longer in the kernel. */
1501 if (ret < 0 && errno == ENOENT)
1502 ret = 0;
1503 if (ret < 0) {
1504 TAP_LOG(ERR,
1505 "Kernel refused TC filter rule deletion (%d): %s",
1506 errno, strerror(errno));
1507 rte_flow_error_set(
1508 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1509 "couldn't receive kernel ack to our request");
1510 goto end;
1511 }
1512
1513 if (remote_flow) {
1514 remote_flow->msg.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK;
1515 remote_flow->msg.nh.nlmsg_type = RTM_DELTFILTER;
1516
1517 ret = tap_nl_send(pmd->nlsk_fd, &remote_flow->msg.nh);
1518 if (ret < 0) {
1519 rte_flow_error_set(
1520 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1521 NULL, "Failure sending nl request");
1522 goto end;
1523 }
1524 ret = tap_nl_recv_ack(pmd->nlsk_fd);
1525 if (ret < 0 && errno == ENOENT)
1526 ret = 0;
1527 if (ret < 0) {
1528 TAP_LOG(ERR,
1529 "Kernel refused TC filter rule deletion (%d): %s",
1530 errno, strerror(errno));
1531 rte_flow_error_set(
1532 error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE,
1533 NULL, "Failure trying to receive nl ack");
1534 goto end;
1535 }
1536 }
1537 end:
1538 if (remote_flow)
1539 rte_free(remote_flow);
1540 tap_flow_free(pmd, flow);
1541 return ret;
1542 }
1543
1544 /**
1545 * Destroy a flow.
1546 *
1547 * @see rte_flow_destroy()
1548 * @see rte_flow_ops
1549 */
1550 static int
1551 tap_flow_destroy(struct rte_eth_dev *dev,
1552 struct rte_flow *flow,
1553 struct rte_flow_error *error)
1554 {
1555 struct pmd_internals *pmd = dev->data->dev_private;
1556
1557 return tap_flow_destroy_pmd(pmd, flow, error);
1558 }
1559
1560 /**
1561 * Enable/disable flow isolation.
1562 *
1563 * @see rte_flow_isolate()
1564 * @see rte_flow_ops
1565 */
1566 static int
1567 tap_flow_isolate(struct rte_eth_dev *dev,
1568 int set,
1569 struct rte_flow_error *error __rte_unused)
1570 {
1571 struct pmd_internals *pmd = dev->data->dev_private;
1572 struct pmd_process_private *process_private = dev->process_private;
1573
1574 /* normalize 'set' variable to contain 0 or 1 values */
1575 if (set)
1576 set = 1;
1577 /* if already in the right isolation mode - nothing to do */
1578 if ((set ^ pmd->flow_isolate) == 0)
1579 return 0;
1580 /* mark the isolation mode for tap_flow_implicit_create() */
1581 pmd->flow_isolate = set;
1582 /*
1583 * If netdevice is there, setup appropriate flow rules immediately.
1584 * Otherwise it will be set when bringing up the netdevice (tun_alloc).
1585 */
1586 if (!process_private->rxq_fds[0])
1587 return 0;
1588 if (set) {
1589 struct rte_flow *remote_flow;
1590
1591 while (1) {
1592 remote_flow = LIST_FIRST(&pmd->implicit_flows);
1593 if (!remote_flow)
1594 break;
1595 /*
1596 * Remove all implicit rules on the remote.
1597 * Keep the local rule to redirect packets on TX.
1598 * Keep also the last implicit local rule: ISOLATE.
1599 */
1600 if (remote_flow->msg.t.tcm_ifindex == pmd->if_index)
1601 break;
1602 if (tap_flow_destroy_pmd(pmd, remote_flow, NULL) < 0)
1603 goto error;
1604 }
1605 /* Switch the TC rule according to pmd->flow_isolate */
1606 if (tap_flow_implicit_create(pmd, TAP_ISOLATE) == -1)
1607 goto error;
1608 } else {
1609 /* Switch the TC rule according to pmd->flow_isolate */
1610 if (tap_flow_implicit_create(pmd, TAP_ISOLATE) == -1)
1611 goto error;
1612 if (!pmd->remote_if_index)
1613 return 0;
1614 if (tap_flow_implicit_create(pmd, TAP_REMOTE_TX) < 0)
1615 goto error;
1616 if (tap_flow_implicit_create(pmd, TAP_REMOTE_LOCAL_MAC) < 0)
1617 goto error;
1618 if (tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCAST) < 0)
1619 goto error;
1620 if (tap_flow_implicit_create(pmd, TAP_REMOTE_BROADCASTV6) < 0)
1621 goto error;
1622 if (dev->data->promiscuous &&
1623 tap_flow_implicit_create(pmd, TAP_REMOTE_PROMISC) < 0)
1624 goto error;
1625 if (dev->data->all_multicast &&
1626 tap_flow_implicit_create(pmd, TAP_REMOTE_ALLMULTI) < 0)
1627 goto error;
1628 }
1629 return 0;
1630 error:
1631 pmd->flow_isolate = 0;
1632 return rte_flow_error_set(
1633 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
1634 "TC rule creation failed");
1635 }
1636
1637 /**
1638 * Destroy all flows.
1639 *
1640 * @see rte_flow_flush()
1641 * @see rte_flow_ops
1642 */
1643 int
1644 tap_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error)
1645 {
1646 struct pmd_internals *pmd = dev->data->dev_private;
1647 struct rte_flow *flow;
1648
1649 while (!LIST_EMPTY(&pmd->flows)) {
1650 flow = LIST_FIRST(&pmd->flows);
1651 if (tap_flow_destroy(dev, flow, error) < 0)
1652 return -1;
1653 }
1654 return 0;
1655 }
1656
1657 /**
1658 * Add an implicit flow rule on the remote device to make sure traffic gets to
1659 * the tap netdevice from there.
1660 *
1661 * @param pmd
1662 * Pointer to private structure.
1663 * @param[in] idx
1664 * The idx in the implicit_rte_flows array specifying which rule to apply.
1665 *
1666 * @return -1 if the rule couldn't be applied, 0 otherwise.
1667 */
1668 int tap_flow_implicit_create(struct pmd_internals *pmd,
1669 enum implicit_rule_index idx)
1670 {
1671 uint16_t flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE;
1672 struct rte_flow_action *actions = implicit_rte_flows[idx].actions;
1673 struct rte_flow_action isolate_actions[2] = {
1674 [1] = {
1675 .type = RTE_FLOW_ACTION_TYPE_END,
1676 },
1677 };
1678 struct rte_flow_item *items = implicit_rte_flows[idx].items;
1679 struct rte_flow_attr *attr = &implicit_rte_flows[idx].attr;
1680 struct rte_flow_item_eth eth_local = { .type = 0 };
1681 uint16_t if_index = pmd->remote_if_index;
1682 struct rte_flow *remote_flow = NULL;
1683 struct nlmsg *msg = NULL;
1684 int err = 0;
1685 struct rte_flow_item items_local[2] = {
1686 [0] = {
1687 .type = items[0].type,
1688 .spec = &eth_local,
1689 .mask = items[0].mask,
1690 },
1691 [1] = {
1692 .type = items[1].type,
1693 }
1694 };
1695
1696 remote_flow = rte_zmalloc(__func__, sizeof(struct rte_flow), 0);
1697 if (!remote_flow) {
1698 TAP_LOG(ERR, "Cannot allocate memory for rte_flow");
1699 goto fail;
1700 }
1701 msg = &remote_flow->msg;
1702 if (idx == TAP_REMOTE_TX) {
1703 if_index = pmd->if_index;
1704 } else if (idx == TAP_ISOLATE) {
1705 if_index = pmd->if_index;
1706 /* Don't be exclusive for this rule, it can be changed later. */
1707 flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE;
1708 isolate_actions[0].type = pmd->flow_isolate ?
1709 RTE_FLOW_ACTION_TYPE_DROP :
1710 RTE_FLOW_ACTION_TYPE_PASSTHRU;
1711 actions = isolate_actions;
1712 } else if (idx == TAP_REMOTE_LOCAL_MAC) {
1713 /*
1714 * eth addr couldn't be set in implicit_rte_flows[] as it is not
1715 * known at compile time.
1716 */
1717 memcpy(&eth_local.dst, &pmd->eth_addr, sizeof(pmd->eth_addr));
1718 items = items_local;
1719 }
1720 tc_init_msg(msg, if_index, RTM_NEWTFILTER, flags);
1721 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1722 /*
1723 * The ISOLATE rule is always present and must have a static handle, as
1724 * the action is changed whether the feature is enabled (DROP) or
1725 * disabled (PASSTHRU).
1726 * There is just one REMOTE_PROMISCUOUS rule in all cases. It should
1727 * have a static handle such that adding it twice will fail with EEXIST
1728 * with any kernel version. Remark: old kernels may falsely accept the
1729 * same REMOTE_PROMISCUOUS rules if they had different handles.
1730 */
1731 if (idx == TAP_ISOLATE)
1732 remote_flow->msg.t.tcm_handle = ISOLATE_HANDLE;
1733 else if (idx == TAP_REMOTE_PROMISC)
1734 remote_flow->msg.t.tcm_handle = REMOTE_PROMISCUOUS_HANDLE;
1735 else
1736 tap_flow_set_handle(remote_flow);
1737 if (priv_flow_process(pmd, attr, items, actions, NULL,
1738 remote_flow, implicit_rte_flows[idx].mirred)) {
1739 TAP_LOG(ERR, "rte flow rule validation failed");
1740 goto fail;
1741 }
1742 err = tap_nl_send(pmd->nlsk_fd, &msg->nh);
1743 if (err < 0) {
1744 TAP_LOG(ERR, "Failure sending nl request");
1745 goto fail;
1746 }
1747 err = tap_nl_recv_ack(pmd->nlsk_fd);
1748 if (err < 0) {
1749 /* Silently ignore re-entering existing rule */
1750 if (errno == EEXIST)
1751 goto success;
1752 TAP_LOG(ERR,
1753 "Kernel refused TC filter rule creation (%d): %s",
1754 errno, strerror(errno));
1755 goto fail;
1756 }
1757 LIST_INSERT_HEAD(&pmd->implicit_flows, remote_flow, next);
1758 success:
1759 return 0;
1760 fail:
1761 if (remote_flow)
1762 rte_free(remote_flow);
1763 return -1;
1764 }
1765
1766 /**
1767 * Remove specific implicit flow rule on the remote device.
1768 *
1769 * @param[in, out] pmd
1770 * Pointer to private structure.
1771 * @param[in] idx
1772 * The idx in the implicit_rte_flows array specifying which rule to remove.
1773 *
1774 * @return -1 if one of the implicit rules couldn't be created, 0 otherwise.
1775 */
1776 int tap_flow_implicit_destroy(struct pmd_internals *pmd,
1777 enum implicit_rule_index idx)
1778 {
1779 struct rte_flow *remote_flow;
1780 int cur_prio = -1;
1781 int idx_prio = implicit_rte_flows[idx].attr.priority + PRIORITY_OFFSET;
1782
1783 for (remote_flow = LIST_FIRST(&pmd->implicit_flows);
1784 remote_flow;
1785 remote_flow = LIST_NEXT(remote_flow, next)) {
1786 cur_prio = (remote_flow->msg.t.tcm_info >> 16) & PRIORITY_MASK;
1787 if (cur_prio != idx_prio)
1788 continue;
1789 return tap_flow_destroy_pmd(pmd, remote_flow, NULL);
1790 }
1791 return 0;
1792 }
1793
1794 /**
1795 * Destroy all implicit flows.
1796 *
1797 * @see rte_flow_flush()
1798 */
1799 int
1800 tap_flow_implicit_flush(struct pmd_internals *pmd, struct rte_flow_error *error)
1801 {
1802 struct rte_flow *remote_flow;
1803
1804 while (!LIST_EMPTY(&pmd->implicit_flows)) {
1805 remote_flow = LIST_FIRST(&pmd->implicit_flows);
1806 if (tap_flow_destroy_pmd(pmd, remote_flow, error) < 0)
1807 return -1;
1808 }
1809 return 0;
1810 }
1811
1812 #define MAX_RSS_KEYS 256
1813 #define KEY_IDX_OFFSET (3 * MAX_RSS_KEYS)
1814 #define SEC_NAME_CLS_Q "cls_q"
1815
1816 static const char *sec_name[SEC_MAX] = {
1817 [SEC_L3_L4] = "l3_l4",
1818 };
1819
1820 /**
1821 * Enable RSS on tap: create TC rules for queuing.
1822 *
1823 * @param[in, out] pmd
1824 * Pointer to private structure.
1825 *
1826 * @param[in] attr
1827 * Pointer to rte_flow to get flow group
1828 *
1829 * @param[out] error
1830 * Pointer to error reporting if not NULL.
1831 *
1832 * @return 0 on success, negative value on failure.
1833 */
1834 static int rss_enable(struct pmd_internals *pmd,
1835 const struct rte_flow_attr *attr,
1836 struct rte_flow_error *error)
1837 {
1838 struct rte_flow *rss_flow = NULL;
1839 struct nlmsg *msg = NULL;
1840 /* 4096 is the maximum number of instructions for a BPF program */
1841 char annotation[64];
1842 int i;
1843 int err = 0;
1844
1845 /* unlimit locked memory */
1846 struct rlimit memlock_limit = {
1847 .rlim_cur = RLIM_INFINITY,
1848 .rlim_max = RLIM_INFINITY,
1849 };
1850 setrlimit(RLIMIT_MEMLOCK, &memlock_limit);
1851
1852 /* Get a new map key for a new RSS rule */
1853 err = bpf_rss_key(KEY_CMD_INIT, NULL);
1854 if (err < 0) {
1855 rte_flow_error_set(
1856 error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1857 "Failed to initialize BPF RSS keys");
1858
1859 return -1;
1860 }
1861
1862 /*
1863 * Create BPF RSS MAP
1864 */
1865 pmd->map_fd = tap_flow_bpf_rss_map_create(sizeof(__u32), /* key size */
1866 sizeof(struct rss_key),
1867 MAX_RSS_KEYS);
1868 if (pmd->map_fd < 0) {
1869 TAP_LOG(ERR,
1870 "Failed to create BPF map (%d): %s",
1871 errno, strerror(errno));
1872 rte_flow_error_set(
1873 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
1874 "Kernel too old or not configured "
1875 "to support BPF maps");
1876
1877 return -ENOTSUP;
1878 }
1879
1880 /*
1881 * Add a rule per queue to match reclassified packets and direct them to
1882 * the correct queue.
1883 */
1884 for (i = 0; i < pmd->dev->data->nb_rx_queues; i++) {
1885 pmd->bpf_fd[i] = tap_flow_bpf_cls_q(i);
1886 if (pmd->bpf_fd[i] < 0) {
1887 TAP_LOG(ERR,
1888 "Failed to load BPF section %s for queue %d",
1889 SEC_NAME_CLS_Q, i);
1890 rte_flow_error_set(
1891 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE,
1892 NULL,
1893 "Kernel too old or not configured "
1894 "to support BPF programs loading");
1895
1896 return -ENOTSUP;
1897 }
1898
1899 rss_flow = rte_zmalloc(__func__, sizeof(struct rte_flow), 0);
1900 if (!rss_flow) {
1901 TAP_LOG(ERR,
1902 "Cannot allocate memory for rte_flow");
1903 return -1;
1904 }
1905 msg = &rss_flow->msg;
1906 tc_init_msg(msg, pmd->if_index, RTM_NEWTFILTER, NLM_F_REQUEST |
1907 NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE);
1908 msg->t.tcm_info = TC_H_MAKE(0, htons(ETH_P_ALL));
1909 tap_flow_set_handle(rss_flow);
1910 uint16_t group = attr->group << GROUP_SHIFT;
1911 uint16_t prio = group | (i + PRIORITY_OFFSET);
1912 msg->t.tcm_info = TC_H_MAKE(prio << 16, msg->t.tcm_info);
1913 msg->t.tcm_parent = TC_H_MAKE(MULTIQ_MAJOR_HANDLE, 0);
1914
1915 tap_nlattr_add(&msg->nh, TCA_KIND, sizeof("bpf"), "bpf");
1916 if (tap_nlattr_nested_start(msg, TCA_OPTIONS) < 0)
1917 return -1;
1918 tap_nlattr_add32(&msg->nh, TCA_BPF_FD, pmd->bpf_fd[i]);
1919 snprintf(annotation, sizeof(annotation), "[%s%d]",
1920 SEC_NAME_CLS_Q, i);
1921 tap_nlattr_add(&msg->nh, TCA_BPF_NAME, strlen(annotation) + 1,
1922 annotation);
1923 /* Actions */
1924 {
1925 struct action_data adata = {
1926 .id = "skbedit",
1927 .skbedit = {
1928 .skbedit = {
1929 .action = TC_ACT_PIPE,
1930 },
1931 .queue = i,
1932 },
1933 };
1934 if (add_actions(rss_flow, 1, &adata, TCA_BPF_ACT) < 0)
1935 return -1;
1936 }
1937 tap_nlattr_nested_finish(msg); /* nested TCA_OPTIONS */
1938
1939 /* Netlink message is now ready to be sent */
1940 if (tap_nl_send(pmd->nlsk_fd, &msg->nh) < 0)
1941 return -1;
1942 err = tap_nl_recv_ack(pmd->nlsk_fd);
1943 if (err < 0) {
1944 TAP_LOG(ERR,
1945 "Kernel refused TC filter rule creation (%d): %s",
1946 errno, strerror(errno));
1947 return err;
1948 }
1949 LIST_INSERT_HEAD(&pmd->rss_flows, rss_flow, next);
1950 }
1951
1952 pmd->rss_enabled = 1;
1953 return err;
1954 }
1955
1956 /**
1957 * Manage bpf RSS keys repository with operations: init, get, release
1958 *
1959 * @param[in] cmd
1960 * Command on RSS keys: init, get, release
1961 *
1962 * @param[in, out] key_idx
1963 * Pointer to RSS Key index (out for get command, in for release command)
1964 *
1965 * @return -1 if couldn't get, release or init the RSS keys, 0 otherwise.
1966 */
1967 static int bpf_rss_key(enum bpf_rss_key_e cmd, __u32 *key_idx)
1968 {
1969 __u32 i;
1970 int err = 0;
1971 static __u32 num_used_keys;
1972 static __u32 rss_keys[MAX_RSS_KEYS] = {KEY_STAT_UNSPEC};
1973 static __u32 rss_keys_initialized;
1974 __u32 key;
1975
1976 switch (cmd) {
1977 case KEY_CMD_GET:
1978 if (!rss_keys_initialized) {
1979 err = -1;
1980 break;
1981 }
1982
1983 if (num_used_keys == RTE_DIM(rss_keys)) {
1984 err = -1;
1985 break;
1986 }
1987
1988 *key_idx = num_used_keys % RTE_DIM(rss_keys);
1989 while (rss_keys[*key_idx] == KEY_STAT_USED)
1990 *key_idx = (*key_idx + 1) % RTE_DIM(rss_keys);
1991
1992 rss_keys[*key_idx] = KEY_STAT_USED;
1993
1994 /*
1995 * Add an offset to key_idx in order to handle a case of
1996 * RSS and non RSS flows mixture.
1997 * If a non RSS flow is destroyed it has an eBPF map
1998 * index 0 (initialized on flow creation) and might
1999 * unintentionally remove RSS entry 0 from eBPF map.
2000 * To avoid this issue, add an offset to the real index
2001 * during a KEY_CMD_GET operation and subtract this offset
2002 * during a KEY_CMD_RELEASE operation in order to restore
2003 * the real index.
2004 */
2005 *key_idx += KEY_IDX_OFFSET;
2006 num_used_keys++;
2007 break;
2008
2009 case KEY_CMD_RELEASE:
2010 if (!rss_keys_initialized)
2011 break;
2012
2013 /*
2014 * Subtract offest to restore real key index
2015 * If a non RSS flow is falsely trying to release map
2016 * entry 0 - the offset subtraction will calculate the real
2017 * map index as an out-of-range value and the release operation
2018 * will be silently ignored.
2019 */
2020 key = *key_idx - KEY_IDX_OFFSET;
2021 if (key >= RTE_DIM(rss_keys))
2022 break;
2023
2024 if (rss_keys[key] == KEY_STAT_USED) {
2025 rss_keys[key] = KEY_STAT_AVAILABLE;
2026 num_used_keys--;
2027 }
2028 break;
2029
2030 case KEY_CMD_INIT:
2031 for (i = 0; i < RTE_DIM(rss_keys); i++)
2032 rss_keys[i] = KEY_STAT_AVAILABLE;
2033
2034 rss_keys_initialized = 1;
2035 num_used_keys = 0;
2036 break;
2037
2038 case KEY_CMD_DEINIT:
2039 for (i = 0; i < RTE_DIM(rss_keys); i++)
2040 rss_keys[i] = KEY_STAT_UNSPEC;
2041
2042 rss_keys_initialized = 0;
2043 num_used_keys = 0;
2044 break;
2045
2046 default:
2047 break;
2048 }
2049
2050 return err;
2051 }
2052
2053 /**
2054 * Add RSS hash calculations and queue selection
2055 *
2056 * @param[in, out] pmd
2057 * Pointer to internal structure. Used to set/get RSS map fd
2058 *
2059 * @param[in] rss
2060 * Pointer to RSS flow actions
2061 *
2062 * @param[out] error
2063 * Pointer to error reporting if not NULL.
2064 *
2065 * @return 0 on success, negative value on failure
2066 */
2067 static int rss_add_actions(struct rte_flow *flow, struct pmd_internals *pmd,
2068 const struct rte_flow_action_rss *rss,
2069 struct rte_flow_error *error)
2070 {
2071 /* 4096 is the maximum number of instructions for a BPF program */
2072 unsigned int i;
2073 int err;
2074 struct rss_key rss_entry = { .hash_fields = 0,
2075 .key_size = 0 };
2076
2077 /* Check supported RSS features */
2078 if (rss->func != RTE_ETH_HASH_FUNCTION_DEFAULT)
2079 return rte_flow_error_set
2080 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2081 "non-default RSS hash functions are not supported");
2082 if (rss->level)
2083 return rte_flow_error_set
2084 (error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
2085 "a nonzero RSS encapsulation level is not supported");
2086
2087 /* Get a new map key for a new RSS rule */
2088 err = bpf_rss_key(KEY_CMD_GET, &flow->key_idx);
2089 if (err < 0) {
2090 rte_flow_error_set(
2091 error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
2092 "Failed to get BPF RSS key");
2093
2094 return -1;
2095 }
2096
2097 /* Update RSS map entry with queues */
2098 rss_entry.nb_queues = rss->queue_num;
2099 for (i = 0; i < rss->queue_num; i++)
2100 rss_entry.queues[i] = rss->queue[i];
2101 rss_entry.hash_fields =
2102 (1 << HASH_FIELD_IPV4_L3_L4) | (1 << HASH_FIELD_IPV6_L3_L4);
2103
2104 /* Add this RSS entry to map */
2105 err = tap_flow_bpf_update_rss_elem(pmd->map_fd,
2106 &flow->key_idx, &rss_entry);
2107
2108 if (err) {
2109 TAP_LOG(ERR,
2110 "Failed to update BPF map entry #%u (%d): %s",
2111 flow->key_idx, errno, strerror(errno));
2112 rte_flow_error_set(
2113 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
2114 "Kernel too old or not configured "
2115 "to support BPF maps updates");
2116
2117 return -ENOTSUP;
2118 }
2119
2120
2121 /*
2122 * Load bpf rules to calculate hash for this key_idx
2123 */
2124
2125 flow->bpf_fd[SEC_L3_L4] =
2126 tap_flow_bpf_calc_l3_l4_hash(flow->key_idx, pmd->map_fd);
2127 if (flow->bpf_fd[SEC_L3_L4] < 0) {
2128 TAP_LOG(ERR,
2129 "Failed to load BPF section %s (%d): %s",
2130 sec_name[SEC_L3_L4], errno, strerror(errno));
2131 rte_flow_error_set(
2132 error, ENOTSUP, RTE_FLOW_ERROR_TYPE_HANDLE, NULL,
2133 "Kernel too old or not configured "
2134 "to support BPF program loading");
2135
2136 return -ENOTSUP;
2137 }
2138
2139 /* Actions */
2140 {
2141 struct action_data adata[] = {
2142 {
2143 .id = "bpf",
2144 .bpf = {
2145 .bpf_fd = flow->bpf_fd[SEC_L3_L4],
2146 .annotation = sec_name[SEC_L3_L4],
2147 .bpf = {
2148 .action = TC_ACT_PIPE,
2149 },
2150 },
2151 },
2152 };
2153
2154 if (add_actions(flow, RTE_DIM(adata), adata,
2155 TCA_FLOWER_ACT) < 0)
2156 return -1;
2157 }
2158
2159 return 0;
2160 }
2161
2162 /**
2163 * Manage filter operations.
2164 *
2165 * @param dev
2166 * Pointer to Ethernet device structure.
2167 * @param filter_type
2168 * Filter type.
2169 * @param filter_op
2170 * Operation to perform.
2171 * @param arg
2172 * Pointer to operation-specific structure.
2173 *
2174 * @return
2175 * 0 on success, negative errno value on failure.
2176 */
2177 int
2178 tap_dev_filter_ctrl(struct rte_eth_dev *dev,
2179 enum rte_filter_type filter_type,
2180 enum rte_filter_op filter_op,
2181 void *arg)
2182 {
2183 switch (filter_type) {
2184 case RTE_ETH_FILTER_GENERIC:
2185 if (filter_op != RTE_ETH_FILTER_GET)
2186 return -EINVAL;
2187 *(const void **)arg = &tap_flow_ops;
2188 return 0;
2189 default:
2190 TAP_LOG(ERR, "%p: filter type (%d) not supported",
2191 dev, filter_type);
2192 }
2193 return -EINVAL;
2194 }
Ceph